Merged 8.6.6.

8 files changed, 283 insertions, 68 deletions

diff --git a/core/composer.json b/core/composer.json
index 37120ae..34fed39 100644
--- a/core/composer.json
+++ b/core/composer.json
@@ -31,6 +31,7 @@
         "symfony/process": "~3.4.0",
         "symfony/polyfill-iconv": "^1.0",
         "symfony/yaml": "~3.4.5",
+        "typo3/phar-stream-wrapper": "^2.0.1",
         "twig/twig": "^1.35.0",
         "doctrine/common": "^2.5",
         "doctrine/annotations": "^1.2",
diff --git a/core/lib/Drupal/Core/Archiver/ArchiveTar.php b/core/lib/Drupal/Core/Archiver/ArchiveTar.php
index 716511e..abee870 100644
--- a/core/lib/Drupal/Core/Archiver/ArchiveTar.php
+++ b/core/lib/Drupal/Core/Archiver/ArchiveTar.php
@@ -42,7 +42,7 @@
 
  /**
  * Note on Drupal 8 porting.
- * This file origin is Tar.php, release 1.4.0 (stable) with some code
+ * This file origin is Tar.php, release 1.4.5 (stable) with some code
  * from PEAR.php, release 1.9.5 (stable) both at http://pear.php.net.
  * To simplify future porting from pear of this file, you should not
  * do cosmetic or other non significant changes to this file.
@@ -152,6 +152,13 @@ class ArchiveTar
     public $error_object = null;
 
     /**
+     * Format for data extraction
+     *
+     * @var string
+     */
+    public $_fmt ='';
+
+    /**
      * Archive_Tar Class constructor. This flavour of the constructor only
      * declare a new Archive_Tar object, identifying it by the name of the
      * tar file.
@@ -257,6 +264,18 @@ class ArchiveTar
                 return false;
             }
         }
+
+        if (version_compare(PHP_VERSION, "5.5.0-dev") < 0) {
+            $this->_fmt = "a100filename/a8mode/a8uid/a8gid/a12size/a12mtime/" .
+                   "a8checksum/a1typeflag/a100link/a6magic/a2version/" .
+                   "a32uname/a32gname/a8devmajor/a8devminor/a131prefix";
+        } else {
+            $this->_fmt = "Z100filename/Z8mode/Z8uid/Z8gid/Z12size/Z12mtime/" .
+                   "Z8checksum/Z1typeflag/Z100link/Z6magic/Z2version/" .
+                   "Z32uname/Z32gname/Z8devmajor/Z8devminor/Z131prefix";
+        }
+
+
     }
 
     public function __destruct()
@@ -712,7 +731,7 @@ class ArchiveTar
         }
 
         // ----- Get the arguments
-        $v_att_list = & func_get_args();
+        $v_att_list = func_get_args();
 
         // ----- Read the attributes
         $i = 0;
@@ -1392,10 +1411,20 @@ class ArchiveTar
         if ($p_stored_filename == '') {
             $p_stored_filename = $p_filename;
         }
-        $v_reduce_filename = $this->_pathReduction($p_stored_filename);
+        $v_reduced_filename = $this->_pathReduction($p_stored_filename);
 
-        if (strlen($v_reduce_filename) > 99) {
-            if (!$this->_writeLongHeader($v_reduce_filename)) {
+        if (strlen($v_reduced_filename) > 99) {
+            if (!$this->_writeLongHeader($v_reduced_filename, false)) {
+                return false;
+            }
+        }
+
+        $v_linkname = '';
+        if (@is_link($p_filename)) {
+            $v_linkname = readlink($p_filename);
+        }
+        if (strlen($v_linkname) > 99) {
+            if (!$this->_writeLongHeader($v_linkname, true)) {
                 return false;
             }
         }
@@ -1404,14 +1433,10 @@ class ArchiveTar
         $v_uid = sprintf("%07s", DecOct($v_info[4]));
         $v_gid = sprintf("%07s", DecOct($v_info[5]));
         $v_perms = sprintf("%07s", DecOct($v_info['mode'] & 000777));
-
         $v_mtime = sprintf("%011s", DecOct($v_info['mtime']));
 
-        $v_linkname = '';
-
         if (@is_link($p_filename)) {
             $v_typeflag = '2';
-            $v_linkname = readlink($p_filename);
             $v_size = sprintf("%011s", DecOct(0));
         } elseif (@is_dir($p_filename)) {
             $v_typeflag = "5";
@@ -1423,7 +1448,6 @@ class ArchiveTar
         }
 
         $v_magic = 'ustar ';
-
         $v_version = ' ';
 
         if (function_exists('posix_getpwuid')) {
@@ -1438,14 +1462,12 @@ class ArchiveTar
         }
 
         $v_devmajor = '';
-
         $v_devminor = '';
 
         $v_prefix = '';
 
         $v_binary_data_first = pack(
             "a100a8a8a8a12a12",
-            $v_reduce_filename,
             $v_perms,
             $v_uid,
             $v_gid,
@@ -1485,7 +1507,7 @@ class ArchiveTar
         $this->_writeBlock($v_binary_data_first, 148);
 
         // ----- Write the calculated checksum
-        $v_checksum = sprintf("%06s ", DecOct($v_checksum));
+        $v_checksum = sprintf("%06s\0 ", DecOct($v_checksum));
         $v_binary_data = pack("a8", $v_checksum);
         $this->_writeBlock($v_binary_data, 8);
 
@@ -1517,7 +1539,7 @@ class ArchiveTar
         $p_filename = $this->_pathReduction($p_filename);
 
         if (strlen($p_filename) > 99) {
-            if (!$this->_writeLongHeader($p_filename)) {
+            if (!$this->_writeLongHeader($p_filename, false)) {
                 return false;
             }
         }
@@ -1613,36 +1635,31 @@ class ArchiveTar
      * @param string $p_filename
      * @return bool
      */
-    public function _writeLongHeader($p_filename)
+    public function _writeLongHeader($p_filename, $is_link = false)
     {
-        $v_size = sprintf("%11s ", DecOct(strlen($p_filename)));
-
-        $v_typeflag = 'L';
-
+        $v_uid = sprintf("%07s", 0);
+        $v_gid = sprintf("%07s", 0);
+        $v_perms = sprintf("%07s", 0);
+        $v_size = sprintf("%'011s", DecOct(strlen($p_filename)));
+        $v_mtime = sprintf("%011s", 0);
+        $v_typeflag = ($is_link ? 'K' : 'L');
         $v_linkname = '';
-
-        $v_magic = '';
-
-        $v_version = '';
-
+        $v_magic = 'ustar ';
+        $v_version = ' ';
         $v_uname = '';
-
         $v_gname = '';
-
         $v_devmajor = '';
-
         $v_devminor = '';
-
         $v_prefix = '';
 
         $v_binary_data_first = pack(
             "a100a8a8a8a12a12",
             '././@LongLink',
-            0,
-            0,
-            0,
+            $v_perms,
+            $v_uid,
+            $v_gid,
             $v_size,
-            0
+            $v_mtime
         );
         $v_binary_data_last = pack(
             "a1a100a6a2a32a32a8a8a155a12",
@@ -1677,7 +1694,7 @@ class ArchiveTar
         $this->_writeBlock($v_binary_data_first, 148);
 
         // ----- Write the calculated checksum
-        $v_checksum = sprintf("%06s ", DecOct($v_checksum));
+        $v_checksum = sprintf("%06s\0 ", DecOct($v_checksum));
         $v_binary_data = pack("a8", $v_checksum);
         $this->_writeBlock($v_binary_data, 8);
 
@@ -1718,28 +1735,12 @@ class ArchiveTar
         // ----- Calculate the checksum
         $v_checksum = 0;
         // ..... First part of the header
-        for ($i = 0; $i < 148; $i++) {
-            $v_checksum += ord(substr($v_binary_data, $i, 1));
-        }
-        // ..... Ignore the checksum value and replace it by ' ' (space)
-        for ($i = 148; $i < 156; $i++) {
-            $v_checksum += ord(' ');
-        }
-        // ..... Last part of the header
-        for ($i = 156; $i < 512; $i++) {
-            $v_checksum += ord(substr($v_binary_data, $i, 1));
-        }
+        $v_binary_split = str_split($v_binary_data);
+        $v_checksum += array_sum(array_map('ord', array_slice($v_binary_split, 0, 148)));
+        $v_checksum += array_sum(array_map('ord', array(' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ',)));
+        $v_checksum += array_sum(array_map('ord', array_slice($v_binary_split, 156, 512)));
 
-        if (version_compare(PHP_VERSION, "5.5.0-dev") < 0) {
-            $fmt = "a100filename/a8mode/a8uid/a8gid/a12size/a12mtime/" .
-                "a8checksum/a1typeflag/a100link/a6magic/a2version/" .
-                "a32uname/a32gname/a8devmajor/a8devminor/a131prefix";
-        } else {
-            $fmt = "Z100filename/Z8mode/Z8uid/Z8gid/Z12size/Z12mtime/" .
-                "Z8checksum/Z1typeflag/Z100link/Z6magic/Z2version/" .
-                "Z32uname/Z32gname/Z8devmajor/Z8devminor/Z131prefix";
-        }
-        $v_data = unpack($fmt, $v_binary_data);
+        $v_data = unpack($this->_fmt, $v_binary_data);
 
         if (strlen($v_data["prefix"]) > 0) {
             $v_data["filename"] = "$v_data[prefix]/$v_data[filename]";
@@ -1775,7 +1776,7 @@ class ArchiveTar
         $v_header['mode'] = OctDec(trim($v_data['mode']));
         $v_header['uid'] = OctDec(trim($v_data['uid']));
         $v_header['gid'] = OctDec(trim($v_data['gid']));
-        $v_header['size'] = OctDec(trim($v_data['size']));
+        $v_header['size'] = $this->_tarRecToSize($v_data['size']);
         $v_header['mtime'] = OctDec(trim($v_data['mtime']));
         if (($v_header['typeflag'] = $v_data['typeflag']) == "5") {
             $v_header['size'] = 0;
@@ -1795,6 +1796,41 @@ class ArchiveTar
     }
 
     /**
+     * Convert Tar record size to actual size
+     *
+     * @param string $tar_size
+     * @return size of tar record in bytes
+     */
+    private function _tarRecToSize($tar_size)
+    {
+        /*
+         * First byte of size has a special meaning if bit 7 is set.
+         *
+         * Bit 7 indicates base-256 encoding if set.
+         * Bit 6 is the sign bit.
+         * Bits 5:0 are most significant value bits.
+         */
+        $ch = ord($tar_size[0]);
+        if ($ch & 0x80) {
+            // Full 12-bytes record is required.
+            $rec_str = $tar_size . "\x00";
+
+            $size = ($ch & 0x40) ? -1 : 0;
+            $size = ($size << 6) | ($ch & 0x3f);
+
+            for ($num_ch = 1; $num_ch < 12; ++$num_ch) {
+                $size = ($size * 256) + ord($rec_str[$num_ch]);
+            }
+
+            return $size;
+
+        } else {
+            return OctDec(trim($tar_size));
+        }
+    }
+
+
+    /**
      * Detect and report a malicious file name
      *
      * @param string $file
@@ -1803,10 +1839,13 @@ class ArchiveTar
      */
     private function _maliciousFilename($file)
     {
-        if (strpos($file, '/../') !== false) {
+        if (strpos($file, 'phar://') === 0) {
             return true;
         }
-        if (strpos($file, '../') === 0) {
+        if (strpos($file, DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR) !== false) {
+            return true;
+        }
+        if (strpos($file, '..' . DIRECTORY_SEPARATOR) === 0) {
             return true;
         }
         return false;
@@ -1871,11 +1910,20 @@ class ArchiveTar
                 continue;
             }
 
-            // ----- Look for long filename
-            if ($v_header['typeflag'] == 'L') {
-                if (!$this->_readLongHeader($v_header)) {
-                    return null;
-                }
+            switch ($v_header['typeflag']) {
+                case 'L': {
+                    if (!$this->_readLongHeader($v_header)) {
+                        return null;
+                    }
+                } break;
+
+                case 'K': {
+                    $v_link_header = $v_header;
+                    if (!$this->_readLongHeader($v_link_header)) {
+                        return null;
+                    }
+                    $v_header['link'] = $v_link_header['filename'];
+                } break;
             }
 
             if ($v_header['filename'] == $p_filename) {
@@ -1976,11 +2024,20 @@ class ArchiveTar
                 continue;
             }
 
-            // ----- Look for long filename
-            if ($v_header['typeflag'] == 'L') {
-                if (!$this->_readLongHeader($v_header)) {
-                    return false;
-                }
+            switch ($v_header['typeflag']) {
+                case 'L': {
+                    if (!$this->_readLongHeader($v_header)) {
+                        return null;
+                    }
+                } break;
+
+                case 'K': {
+                    $v_link_header = $v_header;
+                    if (!$this->_readLongHeader($v_link_header)) {
+                        return null;
+                    }
+                    $v_header['link'] = $v_link_header['filename'];
+                } break;
             }
 
             // ignore extended / pax headers
diff --git a/core/lib/Drupal/Core/DrupalKernel.php b/core/lib/Drupal/Core/DrupalKernel.php
index a099327..3b14d4d 100644
--- a/core/lib/Drupal/Core/DrupalKernel.php
+++ b/core/lib/Drupal/Core/DrupalKernel.php
@@ -19,6 +19,7 @@ use Drupal\Core\File\MimeType\MimeTypeGuesser;
 use Drupal\Core\Http\TrustedHostsRequestFactory;
 use Drupal\Core\Installer\InstallerRedirectTrait;
 use Drupal\Core\Language\Language;
+use Drupal\Core\Security\PharExtensionInterceptor;
 use Drupal\Core\Security\RequestSanitizer;
 use Drupal\Core\Site\Settings;
 use Drupal\Core\Test\TestDatabase;
@@ -35,6 +36,9 @@ use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
 use Symfony\Component\HttpKernel\Exception\HttpExceptionInterface;
 use Symfony\Component\HttpKernel\TerminableInterface;
 use Symfony\Component\Routing\Route;
+use TYPO3\PharStreamWrapper\Manager as PharStreamWrapperManager;
+use TYPO3\PharStreamWrapper\Behavior as PharStreamWrapperBehavior;
+use TYPO3\PharStreamWrapper\PharStreamWrapper;
 
 /**
  * The DrupalKernel class is the core of Drupal itself.
@@ -471,6 +475,26 @@ class DrupalKernel implements DrupalKernelInterface, TerminableInterface {
     // Initialize the container.
     $this->initializeContainer();
 
+    if (in_array('phar', stream_get_wrappers(), TRUE)) {
+      // Set up a stream wrapper to handle insecurities due to PHP's builtin
+      // phar stream wrapper. This is not registered as a regular stream wrapper
+      // to prevent \Drupal\Core\File\FileSystem::validScheme() treating "phar"
+      // as a valid scheme.
+      try {
+        $behavior = new PharStreamWrapperBehavior();
+        PharStreamWrapperManager::initialize(
+          $behavior->withAssertion(new PharExtensionInterceptor())
+        );
+      }
+      catch (\LogicException $e) {
+        // Continue if the PharStreamWrapperManager is already initialized. For
+        // example, this occurs during a module install.
+        // @see \Drupal\Core\Extension\ModuleInstaller::install()
+      };
+      stream_wrapper_unregister('phar');
+      stream_wrapper_register('phar', PharStreamWrapper::class);
+    }
+
     $this->booted = TRUE;
 
     return $this;
diff --git a/core/lib/Drupal/Core/Security/PharExtensionInterceptor.php b/core/lib/Drupal/Core/Security/PharExtensionInterceptor.php
new file mode 100644
index 0000000..a77e9f8
--- /dev/null
+++ b/core/lib/Drupal/Core/Security/PharExtensionInterceptor.php
@@ -0,0 +1,73 @@
+<?php
+
+namespace Drupal\Core\Security;
+
+use TYPO3\PharStreamWrapper\Assertable;
+use TYPO3\PharStreamWrapper\Helper;
+use TYPO3\PharStreamWrapper\Exception;
+
+/**
+ * An alternate PharExtensionInterceptor to support phar-based CLI tools.
+ *
+ * @see \TYPO3\PharStreamWrapper\Interceptor\PharExtensionInterceptor
+ */
+class PharExtensionInterceptor implements Assertable {
+
+  /**
+   * Determines whether phar file is allowed to execute.
+   *
+   * The phar file is allowed to execute if:
+   * - the base file name has a ".phar" suffix.
+   * - it is the CLI tool that has invoked the interceptor.
+   *
+   * @param string $path
+   *   The path of the phar file to check.
+   *
+   * @param string $command
+   *   The command being carried out.
+   *
+   * @return bool
+   *   TRUE if the phar file is allowed to execute.
+   *
+   * @throws Exception
+   *   Thrown when the file is not allowed to execute.
+   */
+  public function assert($path, $command) {
+    if ($this->baseFileContainsPharExtension($path)) {
+      return TRUE;
+    }
+    throw new Exception(
+      sprintf(
+        'Unexpected file extension in "%s"',
+        $path
+      ),
+      1535198703
+    );
+  }
+
+  /**
+   * @param string $path
+   *   The path of the phar file to check.
+   *
+   * @return bool
+   *   TRUE if the file has a .phar extension or if the execution has been
+   *   invoked by the phar file.
+   */
+  private function baseFileContainsPharExtension($path) {
+    $baseFile = Helper::determineBaseFile($path);
+    if ($baseFile === NULL) {
+      return FALSE;
+    }
+    // If the stream wrapper is registered by invoking a phar file that does
+    // not not have .phar extension then this should be allowed. For
+    // example, some CLI tools recommend removing the extension.
+    $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS);
+    $caller = array_pop($backtrace);
+    if (isset($caller['file']) && $baseFile === $caller['file']) {
+      return TRUE;
+    }
+    $fileExtension = pathinfo($baseFile, PATHINFO_EXTENSION);
+    return strtolower($fileExtension) === 'phar';
+  }
+
+}
diff --git a/core/modules/file/file.module b/core/modules/file/file.module
index a6f6809..0eb9a64 100644
--- a/core/modules/file/file.module
+++ b/core/modules/file/file.module
@@ -23,7 +23,7 @@ use Drupal\Core\Template\Attribute;
 /**
  * The regex pattern used when checking for insecure file types.
  */
-define('FILE_INSECURE_EXTENSION_REGEX', '/\.(php|pl|py|cgi|asp|js)(\.|$)/i');
+define('FILE_INSECURE_EXTENSION_REGEX', '/\.(phar|php|pl|py|cgi|asp|js)(\.|$)/i');
 
 // Load all Field module hooks for File.
 require_once __DIR__ . '/file.field.inc';
diff --git a/core/modules/simpletest/files/phar-1.phar b/core/modules/simpletest/files/phar-1.phar
new file mode 100644
index 0000000..8d25e6d
--- /dev/null
+++ b/core/modules/simpletest/files/phar-1.phar
diff --git a/core/tests/Drupal/KernelTests/Core/File/PharWrapperTest.php b/core/tests/Drupal/KernelTests/Core/File/PharWrapperTest.php
new file mode 100644
index 0000000..2cd2ab1
--- /dev/null
+++ b/core/tests/Drupal/KernelTests/Core/File/PharWrapperTest.php
@@ -0,0 +1,33 @@
+<?php
+
+namespace Drupal\KernelTests\Core\File;
+
+use Drupal\KernelTests\KernelTestBase;
+
+/**
+ * Tests that the phar stream wrapper works.
+ *
+ * @group File
+ */
+class PharWrapperTest extends KernelTestBase {
+
+  /**
+   * Tests that only valid phar files can be used.
+   */
+  public function testPharFile() {
+    $base = $this->getDrupalRoot() . '/core/modules/simpletest/files';
+    // Ensure that file operations via the phar:// stream wrapper work for phar
+    // files with the .phar extension.
+    $this->assertFalse(file_exists("phar://$base/phar-1.phar/no-such-file.php"));
+    $this->assertTrue(file_exists("phar://$base/phar-1.phar/index.php"));
+    $file_contents = file_get_contents("phar://$base/phar-1.phar/index.php");
+    $expected_hash = 'c7e7904ea573c5ebea3ef00bb08c1f86af1a45961fbfbeb1892ff4a98fd73ad5';
+    $this->assertSame($expected_hash, hash('sha256', $file_contents));
+
+    // Ensure that file operations via the phar:// stream wrapper throw an
+    // exception for files without the .phar extension.
+    $this->setExpectedException('TYPO3\PharStreamWrapper\Exception');
+    file_exists("phar://$base/image-2.jpg/index.php");
+  }
+
+}
diff --git a/core/tests/Drupal/KernelTests/Core/File/StreamWrapperTest.php b/core/tests/Drupal/KernelTests/Core/File/StreamWrapperTest.php
index 6c18b66..070782f 100644
--- a/core/tests/Drupal/KernelTests/Core/File/StreamWrapperTest.php
+++ b/core/tests/Drupal/KernelTests/Core/File/StreamWrapperTest.php
@@ -144,4 +144,31 @@ class StreamWrapperTest extends FileTestBase {
     $this->assertFalse(file_stream_wrapper_valid_scheme(file_uri_scheme('foo://asdf')), 'Did not get a valid stream scheme from foo://asdf');
   }
 
+  /**
+   * Tests that phar stream wrapper is registered as expected.
+   *
+   * @see \Drupal\Core\StreamWrapper\StreamWrapperManager::register()
+   */
+  public function testPharStreamWrapperRegistration() {
+    if (!in_array('phar', stream_get_wrappers(), TRUE)) {
+      $this->markTestSkipped('There is no phar stream wrapper registered. PHP is probably compiled without phar support.');
+    }
+    // Ensure that phar is not treated as a valid scheme.
+    $stream_wrapper_manager = $this->container->get('stream_wrapper_manager');
+    $this->assertFalse($stream_wrapper_manager->getViaScheme('phar'));
+
+    // Ensure that calling register again and unregister do not create errors
+    // due to the PharStreamWrapperManager singleton.
+    $stream_wrapper_manager->register();
+    $this->assertContains('public', stream_get_wrappers());
+    $this->assertContains('phar', stream_get_wrappers());
+    $stream_wrapper_manager->unregister();
+    $this->assertNotContains('public', stream_get_wrappers());
+    // This will have reverted to the builtin phar stream wrapper.
+    $this->assertContains('phar', stream_get_wrappers());
+    $stream_wrapper_manager->register();
+    $this->assertContains('public', stream_get_wrappers());
+    $this->assertContains('phar', stream_get_wrappers());
+  }
+
 }