DrupalorgProjectPackageRelease.class.php

<?php

class DrupalorgProjectPackageRelease implements ProjectReleasePackagerInterface {
  /**
   * Configuration settings.
   */

  /// Protected data members of the class
  protected $release_node;
  protected $release_node_wrapper;
  protected $release_version = '';
  protected $release_file_id = '';
  protected $release_node_view_link = '';
  protected $project_node;
  protected $project_short_name = '';
  protected $filenames = array();
  protected $file_destination_root = '';
  protected $temp_directory = '';
  protected $git_label = '';
  protected $git_url = '';
  protected $git_checkout_dir = '';
  protected $export_to = '';
  protected $export = '';
  protected $license = '';
  protected $build_type = 'success';

  public function __construct($release_node, $temp_directory) {
    $this->license = realpath(drupal_get_path('module', 'drupalorg_project') . '/plugins/release_packager/LICENSE.txt');

    // Stash the release node this packager is going to be working on.
    $this->release_node = $release_node;
    $this->release_node_wrapper = entity_metadata_wrapper('node', $release_node);

    // Save all the directory information.
    $this->temp_directory = $temp_directory;

    // Load the project for this release, using our static node_load() cache.
    $this->project_node = node_load(project_release_get_release_project_nid($release_node));

    // We use all of these a lot in a number of functions, so initialize them
    // once here so we can just reuse them whenever we need them.
    $this->project_short_name = $this->project_node->field_project_machine_name[LANGUAGE_NONE][0]['value'];
    $this->release_version = $release_node->field_release_version[LANGUAGE_NONE][0]['value'];
    $this->release_file_id = $this->project_short_name . '-' . $this->release_version;
    $this->release_node_view_link = l(t('view'), 'node/' . $this->release_node->nid);

    if (!empty($this->release_node_wrapper->field_release_vcs_label->value())) {
      $this->git_label = $this->release_node_wrapper->field_release_vcs_label->value();
    }

    $this->export_to = $this->project_short_name;
    // For core, we want to checkout into a directory named via the version,
    // e.g. "drupal-7.0".
    if ($this->project_node->type == 'project_core') {
      $this->export_to = $this->release_file_id;
    }
    // Full path to the Git clone and exported archive.
    $this->git_checkout_dir = $this->temp_directory . '/clone';
    $this->export = $this->temp_directory . '/' . $this->export_to;
    if ($repo = versioncontrol_project_repository_load($this->project_node->nid)) {
      $this->git_url = $repo->remoteUrl();
    }

    // Figure out the filenames we're going to be using for our packages.
    $field = field_info_field('field_release_file');
    $instance = field_info_instance('field_collection_item', 'field_release_file', 'field_release_files');
    $this->file_destination_root = file_field_widget_uri($field, $instance);
    $this->filenames['path_tgz'] = $this->file_destination_root . '/' . $this->release_file_id . '.tar.gz';
    $this->filenames['full_dest_tgz'] = file_stream_wrapper_get_instance_by_uri($this->filenames['path_tgz'])->realpath();
    $this->filenames['path_zip'] = $this->file_destination_root . '/' . $this->release_file_id . '.zip';
    $this->filenames['full_dest_zip'] = file_stream_wrapper_get_instance_by_uri($this->filenames['path_zip'])->realpath();
  }

  public function createPackage(&$files) {

    try {
      return $this->_createPackage($files);
    }
    catch (Exception $e) {
      watchdog('package_error', 'Packaging Error processing @id: @message', [
        '@id' => $this->project_short_name,
        '@message' => $e->getMessage(),
      ], WATCHDOG_ERROR);
      if (!empty($this->git_checkout_dir)) {
        drush_shell_exec('rm -rf %s', $this->git_checkout_dir);
      }
      return 'error';
    }
  }

  /**
   * Fix the given .info file with the specified version string.
   */
  protected function fixInfoFileVersion($file) {
    $site_name = variable_get('site_name', 'Drupal.org');

    $info = "\n; Information added by $site_name packaging script on " . gmdate('Y-m-d') . "\n";
    $info .= 'version = "' . $this->release_version . "\"\n";

    // .info files started with 5.x, so we don't have to worry about version
    // strings like "4.7.x-1.0" in this regular expression. If we can't parse
    // the version (also from an old "HEAD" release), or the version isn't at
    // least 6.x, don't add any "core" attribute at all.
    $matches = array();
    if (preg_match('/^((\d+)\.x)-.*/', $this->release_version, $matches) && $matches[2] >= 6) {
      $info .= "core = \"$matches[1]\"\n";
    }
    $info .= 'project = "' . $this->project_short_name . "\"\n";
    $info .= 'datestamp = "' . time() . "\"\n";

    if (!chmod($file, 0644)) {
      watchdog('package_error', 'chmod(@file, 0644) failed', array('@file' => $file), WATCHDOG_ERROR);
      return FALSE;
    }
    if (!$info_fd = fopen($file, 'ab')) {
      watchdog('package_error', "fopen(@file, 'ab') failed", array('@file' => $file), WATCHDOG_ERROR);
      return FALSE;
    }
    if (!fwrite($info_fd, $info)) {
      watchdog('package_error', 'fwrite(@file) failed', array('@file' => $file), WATCHDOG_ERROR);
      return FALSE;
    }
    return TRUE;
  }

  /**
   * Fix the given .info.yml file with the specified version string.
   */
  protected function fixInfoYmlFileVersion($file) {
    $site_name = variable_get('site_name', 'Drupal.org');

    $doc = file_get_contents($file);

    // Comment out the keys we're going to add.
    $doc = preg_replace('/^((?:version|project|datestamp)\s*:.*)$/m', '# $1', $doc);

    // Add Drupal.org packaging keys.
    $doc .= "\n# Information added by $site_name packaging script on " . gmdate('Y-m-d') . "\n";
    $doc .= "version: '" . $this->release_version . "'\n";
    $doc .= "project: '" . $this->project_short_name . "'\n";
    $doc .= "datestamp: " . time() . "\n";

    if (!chmod($file, 0644)) {
      watchdog('package_error', 'chmod(@file, 0644) failed.', array('@file' => $file), WATCHDOG_ERROR);
      return FALSE;
    }
    if (file_put_contents($file, $doc) === FALSE) {
      watchdog('package_error', 'Writing @file failed.', array('@file' => $file), WATCHDOG_ERROR);
      return FALSE;
    }
    return TRUE;
  }

  /**
   * @param $files
   *
   * @return string
   * @throws \Exception
   */
  protected function _createPackage(&$files) {
    // Remember if the tar.gz version of this release file already exists.
    if (is_file($this->filenames['full_dest_tgz'])) {
      $this->build_type = 'rebuild';
    }

    // If we are packaging a branch that we have packaged in the past,
    // Check to make sure we're not redundantly packaging a commit hash
    // That we have already packaged.
    if ($this->release_node_wrapper->field_release_build_type->value() === 'dynamic' && $this->build_type == 'rebuild' && $this->release_node_wrapper->field_packaged_git_sha1->value()) {
      $head_commit = versioncontrol_gitlab_get_client()->api('repositories')->commit(versioncontrol_project_repository_load($this->project_node->nid)->gitlab_project_id, $this->release_node_wrapper->field_release_vcs_label->value())['id'];
      if ($head_commit === $this->release_node_wrapper->field_packaged_git_sha1->value()) {
        drush_log(dt('Commit @field_packaged_git_sha1 already packaged.', ['@field_packaged_git_sha1' => $this->release_node_wrapper->field_packaged_git_sha1->value()]), 'notice');
        return 'no-op';
      }
    }

    if (empty($this->git_url)) {
      throw new Exception(format_string('%project_title does not have a VCS repository defined', ['%project_title' => $this->project_node->title]));
    }
    if (empty($this->git_label)) {
      throw new Exception(format_string('%release_title does not have a VCS repository defined', ['%release_title' => $this->release_node->title]));
    }

    // Clone this release from Git
    if (!drush_shell_exec('git clone --branch=%s %s %s', $this->git_label, $this->git_url, $this->git_checkout_dir)) {
      throw new Exception(format_string('Git clone failed: <pre>@output</pre>', ['@output' => implode("\n", drush_shell_exec_output())]));
    }
    if ($this->release_node_wrapper->field_release_build_type->value() === 'static') {
      // In case there is a branch with the same name as the tag, make sure we
      // have the tag.
      if (!drush_shell_cd_and_exec($this->git_checkout_dir, 'git fetch --tags')) {
        throw new Exception(format_string('Git fetch tags failed: <pre>@output</pre>', ['@output' => implode("\n", drush_shell_exec_output())]));
      }
      if (!drush_shell_cd_and_exec($this->git_checkout_dir, 'git checkout %s', 'refs/tags/' . $this->git_label)) {
        throw new Exception(format_string('Git checkout tag failed: <pre>@output</pre>', ['@output' => implode("\n", drush_shell_exec_output())]));
      }
    }

    if ($this->project_node->type === 'project_core' && (($this->release_node_wrapper->field_release_version_major->value() == 8 && $this->release_node_wrapper->field_release_version_minor->value() >= 8) || ($this->release_node_wrapper->field_release_version_major->value() >= 9))) {
      // Build the tarball with composer create project.
      $this->buildComposerPackage();
    } else {
      $this->buildGitPackage();
    }

    // Get the commit hash for the tag or branch being packaged.
    drush_shell_cd_and_exec($this->git_checkout_dir, 'git rev-list --topo-order --max-count=1 %s 2>&1', $this->git_label);
    if (($last_tag_hash = drush_shell_exec_output()) && preg_match('/^[0-9a-f]{40}$/', $last_tag_hash[0])) {
      drush_log(dt('Using commit @last_tag_hash', ['@last_tag_hash' => $last_tag_hash[0]]), 'notice');
      $this->release_node->field_packaged_git_sha1[LANGUAGE_NONE][0]['value'] = $last_tag_hash[0];
    }

    // If this is a -dev release, do some magic to determine a spiffy
    // "rebuild_version" string which we'll put into any .info files and
    // save in the DB for other uses.
    if ($this->release_node_wrapper->field_release_build_type->value() === 'dynamic') {
      if ($last_tag_hash) {
        drush_shell_cd_and_exec($this->git_checkout_dir, "git describe --tags %s 2>&1", $last_tag_hash[0]);
        if ($last_tag = drush_shell_exec_output()) {
          // Make sure the tag starts as Drupal formatted (for eg.
          // 7.x-1.0-alpha1 or 1.0.0) and if we are on a release branch then
          // it’s on that branch.
          $version_regex = preg_quote(substr($this->git_label, 0, -1), '/');
          if (preg_match('/^\d+\.x$/', $this->git_label)) {
            // Major-only semantic versions branch, like 1.x or 2.x. The tagged
            // releases will have an additional minor version component.
            $version_regex .= '\d+\.';
          }
          $matches = [];
          if (preg_match('/^(?<drupalversion>' . $version_regex . '\d+(?:-[^-]+)?)(?<gitextra>-(?<numberofcommits>\d+-)g[0-9a-f]{7,})?$/', $last_tag[0], $matches)) {
            // If we found additional git metadata (in particular, number of
            // commits) then use that info to build the version string.
            if (isset($matches['gitextra'])) {
              $this->release_version = $matches['drupalversion'] . '+' . $matches['numberofcommits'] . 'dev';
            }
            // Otherwise, the branch tip is pointing to the same commit as the
            // last tag on the branch, in which case we use the prior tag and
            // add '+0-dev' to indicate we're still on a -dev branch.
            else {
              $this->release_version = $last_tag[0] . '+0-dev';
            }
          }
        }
      }
      project_release_record_rebuild_metadata($this->release_node->nid, $this->release_version);
    }

    // Update any .info files with packaging metadata.
    foreach (array_keys(file_scan_directory($this->export, '/^.+\.info(\.yml)?$/')) as $file) {
      switch (strrchr($file, '.')) {
        case '.info':
          if (!$this->fixInfoFileVersion($file)) {
            throw new Exception(format_string('Failed to update version in %file, aborting packaging.', ['%file' => $file]));
          }
          break;

        case '.yml':
          if (!$this->fixInfoYmlFileVersion($file)) {
            throw new Exception(format_string('Failed to update version in %file, aborting packaging.', ['%file' => $file]));
          }
          break;
      }
    }

    // Link not copy, since we want to preserve the date...
    @unlink($this->export . '/LICENSE.txt');
    if (!symlink($this->license, $this->export . '/LICENSE.txt')) {
      throw new Exception('Unable to link LICENSE.txt, for some reason. Maybe
      look for something that has broken.');
    }

    // 'h' is for dereference, we want to include the files, not the links
    if (!drush_shell_cd_and_exec($this->temp_directory, "/bin/tar -ch --owner=0 --group=0 --file=- %s | /bin/gzip -9 --no-name > %s", $this->export_to, $this->filenames['full_dest_tgz'])) {
      throw new Exception(format_string('Archiving failed: <pre>@output</pre>', ['@output' => implode("\n", drush_shell_exec_output())]));

    }
    $files[$this->filenames['path_tgz']] = 0;

    // If we're rebuilding, make sure the previous .zip is gone, since just
    // running zip again with the same zip archive won't give us the semantics
    // we want. For example, files that are removed in CVS will still be left
    // in the .zip archive.
    @unlink($this->filenames['full_dest_zip']);
    if (!drush_shell_cd_and_exec($this->temp_directory, "/usr/bin/zip -rq %s %s", $this->filenames['full_dest_zip'], $this->export_to)) {
      throw new Exception(format_string('Archiving failed: <pre>@output</pre>', ['@output' => implode("\n", drush_shell_exec_output())]));
    }
    $files[$this->filenames['path_zip']] = 1;

    // We must remove the link before Drush runs drush_delete_dir_contents.
    // Drush cleanup will briefly set all files to 777, including the file
    // LICENSE.txt is linked to. Remove when
    // https://github.com/drush-ops/drush/issues/672 is fixed.
    @unlink($this->export . '/LICENSE.txt');


    // Clean up the clone because drush_delete_tmp_dir() is slow, and disk use
    // can pile up as multiple releases are packaged.
    drush_shell_exec('rm -rf %s', $this->git_checkout_dir);

    return $this->build_type;
  }

  protected function buildGitPackage() {
    // Archive and expand to preserve timestamps.
    if (!drush_shell_cd_and_exec($this->temp_directory, 'git --git-dir=%s archive --format=tar --prefix=%s/ %s | /bin/tar x', $this->git_checkout_dir . '/.git', $this->export_to, $this->git_label)) {
      throw new Exception(format_string('Git archive failed: <pre>@output</pre>', ['@output' => implode("\n", drush_shell_exec_output())]));
    }
    if (!is_dir($this->export)) {
      throw new Exception(format_string('%export does not exist after clone and archive.', ['%export' => $this->export]));
    }

    // Install core dependencies with composer for Drupal 8 and above.
    if ($this->project_node->type === 'project_core' && $this->release_node_wrapper->field_release_version_major->value() >= 8 && file_exists($this->export . '/composer.json')) {
      $composer_options = [
        '--working-dir=%s',
        '--prefer-dist',
        '--no-interaction',
        '--ignore-platform-reqs',
        '--no-suggest',
      ];
      // If we're packaging a tagged release, exclude all dev dependencies.
      if ($this->release_node_wrapper->field_release_build_type->value() === 'static') {
        $composer_options[] = '--no-dev';
      }
      if (!drush_shell_cd_and_exec($this->temp_directory, '/usr/bin/php7.3 /usr/local/bin/composer install ' . implode(' ', $composer_options), $this->export)) {
        throw new Exception(format_string('Installing core dependencies with composer failed: <pre>@output</pre>', ['@output' => implode("\n", drush_shell_exec_output())]));
      }
    }
  }

  protected function buildComposerPackage() {
    $composer_options = [
      '--prefer-dist',
      '--no-interaction',
      '--no-progress',
      '--ignore-platform-reqs',
    ];
    // If we're packaging a tagged release, exclude all dev dependencies.
    if ($this->release_node_wrapper->field_release_build_type->value() === 'static') {
      $composer_options[] = '--no-dev';
    }

    // clone the legacy project that we plan on making tarballs out of.
    drush_shell_cd_and_exec($this->temp_directory, 'git clone --depth 1 --branch %s %s %s', $this->git_label, variable_get('drupalorg_subtree_vaults_url', '/var/lib/subtree-splits/subtree-vaults') . '/drupal/legacy-project', "subtrees/legacy-project");
    $legacy_dir = "{$this->temp_directory}/subtrees/legacy-project";

    $local_dependencies = ['core-composer-scaffold','core-project-message','core-recommended','core-vendor-hardening','recommended-project','core','core-dev-pinned', 'core-dev'];

    // Clone all the important local repos
    foreach ($local_dependencies as $dependency){
      drush_shell_cd_and_exec($this->temp_directory, 'git clone --depth 1 --branch %s %s %s', $this->git_label, variable_get('drupalorg_subtree_vaults_url', '/var/lib/subtree-splits/subtree-vaults') . '/drupal/' . $dependency, "subtrees/{$dependency}");
      // update the composer path repos
      drush_shell_cd_and_exec($this->temp_directory, "/usr/bin/php7.3 /usr/local/bin/composer --working-dir=%s config repositories.%s %s", $legacy_dir, $dependency, "{\"type\": \"path\", \"url\": \"{$this->temp_directory}/subtrees/{$dependency}\"}");
    }
    // Point the lock file at the local path repos
    $this->fixLockFile($legacy_dir, $local_dependencies);
    // Composer create project
    drush_shell_cd_and_exec($this->temp_directory, "COMPOSER_MIRROR_PATH_REPOS=1 /usr/bin/php7.3 /usr/local/bin/composer -vvv --working-dir=%s " . implode(' ', $composer_options) . " create-project", $legacy_dir);
    drush_shell_cd_and_exec($this->temp_directory, "git -C %s -c advice.detachedHead=false checkout -- composer.json composer.lock", $legacy_dir);
    // Update installed.json to act as if we installed from github
    $this->fixInstalledFile($legacy_dir, $local_dependencies);
    // Cleanup the git repo
    drush_shell_cd_and_exec($legacy_dir, "rm -rf {$legacy_dir}/.git");
    // Put it into the location that the rest of packaging expects.
    drush_shell_cd_and_exec($this->temp_directory, "mv ./subtrees/legacy-project %s", $this->release_file_id);

  }
  /**
   * @param $directory
   * @param array $dep_list
   */
  protected function fixLockFile($directory, array $dep_names) {
    // Add core to the list that we fix.
    $dep_list = array_map(
      static function ($package_name) {
        return 'drupal/' . $package_name;
      },
      $dep_names
    );

    $composer_lock = json_decode(file_get_contents("$directory/composer.lock"), TRUE);
    $composer_lock['packages'] = $this->fixLockedPackages($composer_lock['packages'], $dep_list);
    $composer_lock['packages-dev'] = $this->fixLockedPackages($composer_lock['packages-dev'], $dep_list);
    file_put_contents("$directory/composer.lock", json_encode($composer_lock, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE));
  }

  protected function fixLockedPackages($package_list, $dep_names) {
    $tmpdir = $this->temp_directory;
    $package_list = array_map(
      static function ($package) use ($dep_names, $tmpdir) {
        if (in_array($package['name'], $dep_names)) {
          $name = str_replace('drupal/','', $package['name']);
          $package['source']['type'] = "git";
          $package['source']['url'] = "/{$tmpdir}/subtrees/{$name}";
          $package['source']['reference'] = $package['dist']['reference'];
          $package['dist']['type'] = "path";
          $package['dist']['url'] = "/{$tmpdir}/subtrees/{$name}";
          $package['dist']['shasum'] = "";
        }
        return $package;
      },
      $package_list
    );
    return $package_list;
  }

  /**
   * @param $directory
   * @param array $dep_list
   */
  protected function fixInstalledFile($directory, array $dep_names) {
    // Add core to the list that we fix.
    $dep_list = array_map(
      static function ($package_name) {
        return 'drupal/' . $package_name;
      },
      $dep_names
    );

    $installed = json_decode(file_get_contents("$directory/vendor/composer/installed.json"), TRUE);
    if (isset($installed['packages'])) {
      $composer2 = TRUE;
      // installed.json was generated by Composer 2.
      $installed = $installed['packages'];
    }
    $installed = $this->fixPathPackages($installed, $dep_list);
    if ($composer2) {
      $installed_c2['packages'] = $installed;
      $installed = $installed_c2;
    }

    file_put_contents("$directory/vendor/composer/installed.json", json_encode($installed, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE));
  }

  protected function fixPathPackages($package_list, $dep_names){
    $package_list = array_map(
      static function ($package) use ($dep_names) {
        if (in_array($package['name'], $dep_names)) {
          $package['source']['type'] = "git";
          $package['source']['url'] = "https://github.com/{$package['name']}.git";
          $package['source']['reference'] = $package['dist']['reference'];
          $package['dist']['type'] = "zip";
          $package['dist']['url'] = "https://api.github.com/repos/{$package['name']}/zipball/{$package['dist']['reference']}";
          $package['dist']['shasum'] = "";
        }
        return $package;
      },
      $package_list
    );
    return $package_list;
  }

}