summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlex Pott2015-05-21 19:08:57 (GMT)
committerAlex Pott2015-05-21 19:08:57 (GMT)
commit10c6e1da5119301d84ed929c83376a793bbf1dd6 (patch)
treeda4f5c53f31d62acd8b567fa31e3e7eef19eb81a
parente0505159c7670d7d0bca22a0dbb9bec1be4efa63 (diff)
Issue #1833356 by grendzy: CSS files encoded in UTF-8 with BOM break the design when enabling CSS aggregation
-rw-r--r--core/lib/Drupal/Component/Utility/Unicode.php32
-rw-r--r--core/lib/Drupal/Core/Asset/CssOptimizer.php14
-rw-r--r--core/lib/Drupal/Core/Asset/JsOptimizer.php14
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/CssOptimizerUnitTest.php56
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/JsOptimizerUnitTest.php48
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_bom.css3
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_bom_and_charset.css4
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_charset.css4
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_utf16_bom.cssbin0 -> 96 bytes
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/js_test_files/latin_9.js1
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/js_test_files/latin_9.js.optimized.js1
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/js_test_files/utf16_bom.jsbin0 -> 40 bytes
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/js_test_files/utf16_bom.js.optimized.js1
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/js_test_files/utf8_bom.js1
-rw-r--r--core/tests/Drupal/Tests/Core/Asset/js_test_files/utf8_bom.js.optimized.js1
15 files changed, 179 insertions, 1 deletions
diff --git a/core/lib/Drupal/Component/Utility/Unicode.php b/core/lib/Drupal/Component/Utility/Unicode.php
index 934b745..7691e00 100644
--- a/core/lib/Drupal/Component/Utility/Unicode.php
+++ b/core/lib/Drupal/Component/Utility/Unicode.php
@@ -184,6 +184,38 @@ EOD;
}
/**
+ * Decodes UTF byte-order mark (BOM) into the encoding's name.
+ *
+ * @param string $data
+ * The data possibly containing a BOM. This can be the entire contents of
+ * a file, or just a fragment containing at least the first five bytes.
+ *
+ * @return string|bool
+ * The name of the encoding, or FALSE if no byte order mark was present.
+ */
+ public static function encodingFromBOM($data) {
+ static $bomMap = array(
+ "\xEF\xBB\xBF" => 'UTF-8',
+ "\xFE\xFF" => 'UTF-16BE',
+ "\xFF\xFE" => 'UTF-16LE',
+ "\x00\x00\xFE\xFF" => 'UTF-32BE',
+ "\xFF\xFE\x00\x00" => 'UTF-32LE',
+ "\x2B\x2F\x76\x38" => 'UTF-7',
+ "\x2B\x2F\x76\x39" => 'UTF-7',
+ "\x2B\x2F\x76\x2B" => 'UTF-7',
+ "\x2B\x2F\x76\x2F" => 'UTF-7',
+ "\x2B\x2F\x76\x38\x2D" => 'UTF-7',
+ );
+
+ foreach ($bomMap as $bom => $encoding) {
+ if (strpos($data, $bom) === 0) {
+ return $encoding;
+ }
+ }
+ return FALSE;
+ }
+
+ /**
* Converts data to UTF-8.
*
* Requires the iconv, GNU recode or mbstring PHP extension.
diff --git a/core/lib/Drupal/Core/Asset/CssOptimizer.php b/core/lib/Drupal/Core/Asset/CssOptimizer.php
index 673012b..4b96888 100644
--- a/core/lib/Drupal/Core/Asset/CssOptimizer.php
+++ b/core/lib/Drupal/Core/Asset/CssOptimizer.php
@@ -7,6 +7,7 @@
namespace Drupal\Core\Asset;
use Drupal\Core\Asset\AssetOptimizerInterface;
+use Drupal\Component\Utility\Unicode;
/**
* Optimizes a CSS asset.
@@ -125,6 +126,19 @@ class CssOptimizer implements AssetOptimizerInterface {
// but are merely there to disable certain module CSS files.
$content = '';
if ($contents = @file_get_contents($file)) {
+ // If a BOM is found, convert the file to UTF-8, then use substr() to
+ // remove the BOM from the result.
+ if ($encoding = (Unicode::encodingFromBOM($contents))) {
+ $contents = Unicode::substr(Unicode::convertToUtf8($contents, $encoding), 1);
+ }
+ // If no BOM, check for fallback encoding. Per CSS spec the regex is very strict.
+ elseif (preg_match('/^@charset "([^"]+)";/', $contents, $matches)) {
+ if ($matches[1] !== 'utf-8' && $matches[1] !== 'UTF-8') {
+ $contents = substr($contents, strlen($matches[0]));
+ $contents = Unicode::convertToUtf8($contents, $matches[1]);
+ }
+ }
+
// Return the processed stylesheet.
$content = $this->processCss($contents, $_optimize);
}
diff --git a/core/lib/Drupal/Core/Asset/JsOptimizer.php b/core/lib/Drupal/Core/Asset/JsOptimizer.php
index bd004a5..33aca69 100644
--- a/core/lib/Drupal/Core/Asset/JsOptimizer.php
+++ b/core/lib/Drupal/Core/Asset/JsOptimizer.php
@@ -7,6 +7,7 @@
namespace Drupal\Core\Asset;
use Drupal\Core\Asset\AssetOptimizerInterface;
+use Drupal\Component\Utility\Unicode;
/**
* Optimizes a JavaScript asset.
@@ -24,8 +25,19 @@ class JsOptimizer implements AssetOptimizerInterface {
throw new \Exception('Only file JavaScript assets with preprocessing enabled can be optimized.');
}
+ // If a BOM is found, convert the file to UTF-8, then use substr() to
+ // remove the BOM from the result.
+ $data = file_get_contents($js_asset['data']);
+ if ($encoding = (Unicode::encodingFromBOM($data))) {
+ $data = Unicode::substr(Unicode::convertToUtf8($data, $encoding), 1);
+ }
+ // If no BOM is found, check for the charset attribute.
+ elseif (isset($js_asset['attributes']['charset'])) {
+ $data = Unicode::convertToUtf8($data, $js_asset['attributes']['charset']);
+ }
+
// No-op optimizer: no optimizations are applied to JavaScript assets.
- return file_get_contents($js_asset['data']);
+ return $data;
}
/**
diff --git a/core/tests/Drupal/Tests/Core/Asset/CssOptimizerUnitTest.php b/core/tests/Drupal/Tests/Core/Asset/CssOptimizerUnitTest.php
index f4d083c..a805e85 100644
--- a/core/tests/Drupal/Tests/Core/Asset/CssOptimizerUnitTest.php
+++ b/core/tests/Drupal/Tests/Core/Asset/CssOptimizerUnitTest.php
@@ -170,6 +170,62 @@ class CssOptimizerUnitTest extends UnitTestCase {
),
file_get_contents($path . 'charset.css.optimized.css'),
),
+ 6 => array(
+ array(
+ 'group' => -100,
+ 'every_page' => TRUE,
+ 'type' => 'file',
+ 'weight' => 0.013,
+ 'media' => 'all',
+ 'preprocess' => TRUE,
+ 'data' => $path . 'css_input_with_bom.css',
+ 'browsers' => array('IE' => TRUE, '!IE' => TRUE),
+ 'basename' => 'css_input_with_bom.css',
+ ),
+ '.byte-order-mark-test{content:"☃";}'. "\n",
+ ),
+ 7 => array(
+ array(
+ 'group' => -100,
+ 'every_page' => TRUE,
+ 'type' => 'file',
+ 'weight' => 0.013,
+ 'media' => 'all',
+ 'preprocess' => TRUE,
+ 'data' => $path . 'css_input_with_charset.css',
+ 'browsers' => array('IE' => TRUE, '!IE' => TRUE),
+ 'basename' => 'css_input_with_charset.css',
+ ),
+ '.charset-test{content:"€";}' . "\n",
+ ),
+ 8 => array(
+ array(
+ 'group' => -100,
+ 'every_page' => TRUE,
+ 'type' => 'file',
+ 'weight' => 0.013,
+ 'media' => 'all',
+ 'preprocess' => TRUE,
+ 'data' => $path . 'css_input_with_bom_and_charset.css',
+ 'browsers' => array('IE' => TRUE, '!IE' => TRUE),
+ 'basename' => 'css_input_with_bom_and_charset.css',
+ ),
+ '.byte-order-mark-charset-test{content:"☃";}' . "\n",
+ ),
+ 9 => array(
+ array(
+ 'group' => -100,
+ 'every_page' => TRUE,
+ 'type' => 'file',
+ 'weight' => 0.013,
+ 'media' => 'all',
+ 'preprocess' => TRUE,
+ 'data' => $path . 'css_input_with_utf16_bom.css',
+ 'browsers' => array('IE' => TRUE, '!IE' => TRUE),
+ 'basename' => 'css_input_with_utf16_bom.css',
+ ),
+ '.utf16-byte-order-mark-test{content:"☃";}' . "\n",
+ ),
);
}
diff --git a/core/tests/Drupal/Tests/Core/Asset/JsOptimizerUnitTest.php b/core/tests/Drupal/Tests/Core/Asset/JsOptimizerUnitTest.php
index 13bd77c..c7490b9 100644
--- a/core/tests/Drupal/Tests/Core/Asset/JsOptimizerUnitTest.php
+++ b/core/tests/Drupal/Tests/Core/Asset/JsOptimizerUnitTest.php
@@ -80,4 +80,52 @@ class JsOptimizerUnitTest extends UnitTestCase {
$this->assertEquals($expected, $this->optimizer->clean($js_asset));
}
+ /**
+ * Provides data for the JS asset optimize test.
+ *
+ * @see \Drupal\Core\Asset\JsOptimizer::optimize().
+ *
+ * @returns array
+ * An array of test data.
+ */
+ function providerTestOptimize() {
+ $path = dirname(__FILE__) . '/js_test_files/';
+ return array(
+ 0 => array(
+ array(
+ 'type' => 'file',
+ 'preprocess' => TRUE,
+ 'data' => $path . 'utf8_bom.js',
+ ),
+ file_get_contents($path . 'utf8_bom.js.optimized.js'),
+ ),
+ 1 => array(
+ array(
+ 'type' => 'file',
+ 'preprocess' => TRUE,
+ 'data' => $path . 'utf16_bom.js',
+ ),
+ file_get_contents($path . 'utf16_bom.js.optimized.js'),
+ ),
+ 2 => array(
+ array(
+ 'type' => 'file',
+ 'preprocess' => TRUE,
+ 'data' => $path . 'latin_9.js',
+ 'attributes' => array('charset' => 'ISO-8859-15'),
+ ),
+ file_get_contents($path . 'latin_9.js.optimized.js'),
+ ),
+ );
+ }
+
+ /**
+ * Tests cleaning of a JS asset group containing 'type' => 'file'.
+ *
+ * @dataProvider providerTestOptimize
+ */
+ function testOptimize($js_asset, $expected) {
+ $this->assertEquals($expected, $this->optimizer->optimize($js_asset));
+ }
+
}
diff --git a/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_bom.css b/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_bom.css
new file mode 100644
index 0000000..bc82185
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_bom.css
@@ -0,0 +1,3 @@
+.byte-order-mark-test {
+ content: "☃";
+}
diff --git a/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_bom_and_charset.css b/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_bom_and_charset.css
new file mode 100644
index 0000000..9ffae69
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_bom_and_charset.css
@@ -0,0 +1,4 @@
+@charset "utf-8";
+.byte-order-mark-charset-test {
+ content: "☃";
+}
diff --git a/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_charset.css b/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_charset.css
new file mode 100644
index 0000000..949f6ca
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_charset.css
@@ -0,0 +1,4 @@
+@charset "iso-8859-15";
+.charset-test {
+ content: "";
+}
diff --git a/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_utf16_bom.css b/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_utf16_bom.css
new file mode 100644
index 0000000..dcbd5c8
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Asset/css_test_files/css_input_with_utf16_bom.css
Binary files differ
diff --git a/core/tests/Drupal/Tests/Core/Asset/js_test_files/latin_9.js b/core/tests/Drupal/Tests/Core/Asset/js_test_files/latin_9.js
new file mode 100644
index 0000000..a488429
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Asset/js_test_files/latin_9.js
@@ -0,0 +1 @@
+var latin9Char = '';
diff --git a/core/tests/Drupal/Tests/Core/Asset/js_test_files/latin_9.js.optimized.js b/core/tests/Drupal/Tests/Core/Asset/js_test_files/latin_9.js.optimized.js
new file mode 100644
index 0000000..f158382
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Asset/js_test_files/latin_9.js.optimized.js
@@ -0,0 +1 @@
+var latin9Char = '€';
diff --git a/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf16_bom.js b/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf16_bom.js
new file mode 100644
index 0000000..ad50d5f
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf16_bom.js
Binary files differ
diff --git a/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf16_bom.js.optimized.js b/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf16_bom.js.optimized.js
new file mode 100644
index 0000000..6a0d6a6
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf16_bom.js.optimized.js
@@ -0,0 +1 @@
+var utf8BOM = '☃';
diff --git a/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf8_bom.js b/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf8_bom.js
new file mode 100644
index 0000000..f4eb31a
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf8_bom.js
@@ -0,0 +1 @@
+var utf8BOM = '☃';
diff --git a/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf8_bom.js.optimized.js b/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf8_bom.js.optimized.js
new file mode 100644
index 0000000..6a0d6a6
--- /dev/null
+++ b/core/tests/Drupal/Tests/Core/Asset/js_test_files/utf8_bom.js.optimized.js
@@ -0,0 +1 @@
+var utf8BOM = '☃';