Select ZHH, ZHS and ZHT for a Chinese region in the lang attribute (#191)

src/Shaper/OtlTags.php

@@ -16,11 +16,6 @@ class OtlTags
 	/**
 	 * The v2 Indic and Myanmar tags, each with the original tag that a font made to the earlier
 	 * specification offers instead.
-	 *
-	 * Order is load-bearing. A v2 tag the font does not offer is tried against its own original tag
-	 * and then against the original tags of every entry after it, not only its own: a Bengali run in
-	 * a font offering neither bng2 nor beng is laid out as deva if the font has that. That is #192,
-	 * kept here as it was.
 	 */
 	private static $originalIndicTags = [
 		'bng2' => 'beng',
@@ -60,14 +55,11 @@ public static function script(array $ScriptLang, $scripttag, $scriptblock, $shap
 			return [$scripttag, false];
 		}
 
-		if ($shaper) {
-			$reached = false;
-			foreach (self::$originalIndicTags as $v2 => $original) {
-				$reached = $reached || $v2 === $scripttag;
-				if ($reached && isset($ScriptLang[$original])) {
-					return [$original, true];
-				}
-			}
+		// Only the run's own original tag: another Indic script's lookups and reordering are no fit
+		// for it, and a font offering nothing for the script is laid out by its default entry
+		$original = isset(self::$originalIndicTags[$scripttag]) ? self::$originalIndicTags[$scripttag] : '';
+		if ($shaper && $original && isset($ScriptLang[$original])) {
+			return [$original, true];
 		}
 
 		foreach (['DFLT', 'dflt', 'latn'] as $fallback) {
@@ -107,11 +99,12 @@ public static function language($ietf, $available)
 			$country = strtolower($tags[2]);
 		}
 
+		$region = $lang . '-' . $country;
+
 		if ($lang != '' && isset(Ucdn::$ot_languages[$lang])) {
 			$langsys = Ucdn::$ot_languages[$lang];
-		} elseif ($lang != '' && $country != '' && isset(Ucdn::$ot_languages[$lang . $country])) {
-			// Never matches: the region keys are written 'zh-hk' (#191)
-			$langsys = Ucdn::$ot_languages[$lang . $country];
+		} elseif ($lang != '' && $country != '' && isset(Ucdn::$ot_languages[$region])) {
+			$langsys = Ucdn::$ot_languages[$region];
 		} else {
 			$langsys = 'DFLT';
 		}

tests/Mpdf/ChineseRegionLangSysTest.php

@@ -0,0 +1,55 @@
+<?php
+
+namespace Mpdf;
+
+/**
+ * A Chinese region in the lang attribute selects the font's language system for it: ZHH for Hong
+ * Kong, ZHT for Taiwan and Macao, ZHS for mainland China and Singapore.
+ *
+ * No font in the corpus has a Chinese language system, so NotoSansTC-RegionLangSys-Synthetic is a
+ * subset of Noto Sans TC 2.004 (OFL 1.1) with its GSUB replaced. Its hani script has no features by
+ * default, and one 'locl' lookup under each of ZHH, ZHS and ZHT, which gives 骨 the glyph of 三, 一
+ * and 二 in turn. `hb-shape --language` draws the same glyph for each of these tags.
+ */
+class ChineseRegionLangSysTest extends \Yoast\PHPUnitPolyfills\TestCases\TestCase
+{
+
+	const BONE = 0x9AA8;
+
+	const ZHS = 0x4E00;
+
+	const ZHT = 0x4E8C;
+
+	const ZHH = 0x4E09;
+
+	public function regions()
+	{
+		return [
+			'zh-HK' => ['zh-HK', self::ZHH],
+			'zh-TW' => ['zh-TW', self::ZHT],
+			'zh-CN' => ['zh-CN', self::ZHS],
+		];
+	}
+
+	/**
+	 * @dataProvider regions
+	 */
+	public function testTheRegionSelectsTheLanguageSystem($lang, $expected)
+	{
+		$mpdf = new TextRecordingMpdf([
+			'mode' => 'utf-8',
+			'fontDir' => [__DIR__ . '/../data/ttf'],
+			'fontdata' => ['notosanstcregionlangsyssynthetic' => [
+				'R' => 'NotoSansTC-RegionLangSys-Synthetic.ttf',
+				'useOTL' => 0xFF,
+			]],
+			'default_font' => 'notosanstcregionlangsyssynthetic',
+		]);
+		$mpdf->WriteHTML(sprintf('<p lang="%s">&#x%04X;</p>', $lang, self::BONE));
+
+		$drawn = array_values(unpack('N*', mb_convert_encoding($mpdf->drawnText[0], 'UTF-32BE', 'UTF-8')));
+
+		$this->assertSame([$expected], $drawn);
+	}
+
+}

tests/Mpdf/IndicScriptFallbackTest.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace Mpdf;
+
+/**
+ * An Indic run in a font offering neither its v2 tag nor its original one is laid out with the
+ * font's default script, not with another Indic script's original tag.
+ *
+ * No font in the corpus offers another script's original tag without the run's own and has glyphs
+ * for the run, so NotoSansBengali-DevaScript-Synthetic is a subset of Noto Sans Bengali 3.011 (OFL
+ * 1.1) with its GSUB replaced. It offers DFLT and deva and nothing for Bengali, and each has one
+ * 'locl' lookup: DFLT's gives KA the glyph of GA, deva's the glyph of KHA. `hb-shape` draws GA.
+ */
+class IndicScriptFallbackTest extends \Yoast\PHPUnitPolyfills\TestCases\TestCase
+{
+
+	const KA = 0x0995;
+
+	const GA = 0x0997;
+
+	public function testABengaliRunIsLaidOutWithTheDefaultScriptRatherThanDevanagari()
+	{
+		$mpdf = new TextRecordingMpdf([
+			'mode' => 'utf-8',
+			'fontDir' => [__DIR__ . '/../data/ttf'],
+			'fontdata' => ['notosansbengalidevascriptsynthetic' => [
+				'R' => 'NotoSansBengali-DevaScript-Synthetic.ttf',
+				'useOTL' => 0xFF,
+			]],
+			'default_font' => 'notosansbengalidevascriptsynthetic',
+		]);
+		$mpdf->WriteHTML(sprintf('<p>&#x%04X;</p>', self::KA));
+
+		$drawn = array_values(unpack('N*', mb_convert_encoding($mpdf->drawnText[0], 'UTF-32BE', 'UTF-8')));
+
+		$this->assertSame([self::GA], $drawn);
+	}
+
+}

tests/Mpdf/Shaper/OtlTagsTest.php

@@ -55,14 +55,22 @@ public function testTheOriginalIndicTagIsNotTriedWithoutAShaper()
 	}
 
 	/**
-	 * Pins #192 as it stands: a v2 tag reaches the original tags listed after its own, but never
-	 * those before it.
+	 * @dataProvider originalTagFallbacks
 	 */
-	public function testAV2TagFallsThroughToTheOriginalTagsListedAfterItsOwn()
+	public function testAV2TagFallsBackToItsOwnOriginalTagAndNoOtherScriptsOriginalTag(array $offered, $scripttag, $scriptblock, array $expected)
 	{
-		$this->assertSame(['deva', true], OtlTags::script(['deva' => 'DFLT', 'DFLT' => 'DFLT'], 'bng2', Ucdn::SCRIPT_BENGALI, 'I', 0xFF));
-		$this->assertSame(['mymr', true], OtlTags::script(['mymr' => 'DFLT'], 'tml2', Ucdn::SCRIPT_TAMIL, 'I', 0xFF));
-		$this->assertSame(['DFLT', false], OtlTags::script(['beng' => 'DFLT', 'DFLT' => 'DFLT'], 'dev2', Ucdn::SCRIPT_DEVANAGARI, 'I', 0xFF));
+		$this->assertSame($expected, OtlTags::script($offered, $scripttag, $scriptblock, 'I', 0xFF));
+	}
+
+	public function originalTagFallbacks()
+	{
+		return [
+			'Bengali, where the font offers deva' => [['deva' => 'DFLT', 'DFLT' => 'DFLT'], 'bng2', Ucdn::SCRIPT_BENGALI, ['DFLT', false]],
+			'Devanagari, where the font offers gujr' => [['gujr' => 'DFLT', 'DFLT' => 'DFLT'], 'dev2', Ucdn::SCRIPT_DEVANAGARI, ['DFLT', false]],
+			'Tamil, where the font offers only mymr' => [['mymr' => 'DFLT'], 'tml2', Ucdn::SCRIPT_TAMIL, ['', false]],
+			'Devanagari, where the font offers beng' => [['beng' => 'DFLT', 'DFLT' => 'DFLT'], 'dev2', Ucdn::SCRIPT_DEVANAGARI, ['DFLT', false]],
+			'Bengali, where the font offers beng and deva' => [['deva' => 'DFLT', 'beng' => 'DFLT'], 'bng2', Ucdn::SCRIPT_BENGALI, ['beng', true]],
+		];
 	}
 
 	public function testATagThatIsNotAV2IndicTagHasNoOriginalToFallBackTo()
@@ -115,4 +123,37 @@ public function languages()
 		];
 	}
 
+	/**
+	 * @dataProvider chineseRegions
+	 */
+	public function testAChineseRegionSelectsItsLanguageSystem($ietf, $expected)
+	{
+		$this->assertSame($expected, OtlTags::language($ietf, 'DFLT ZHH ZHS ZHT '));
+	}
+
+	public function chineseRegions()
+	{
+		return [
+			'Hong Kong' => ['zh-HK', 'ZHH '],
+			'Hong Kong, after the script' => ['zh-Hant-HK', 'ZHH '],
+			'Taiwan' => ['zh-TW', 'ZHT '],
+			'Macao' => ['zh-MO', 'ZHT '],
+			'China' => ['zh-CN', 'ZHS '],
+			'Singapore' => ['zh-SG', 'ZHS '],
+			'lower case' => ['zh-tw', 'ZHT '],
+			'a region the table has no entry for' => ['zh-US', 'DFLT'],
+		];
+	}
+
+	/**
+	 * Pins #201 as it stands: without a region, Chinese has no language system, where HarfBuzz
+	 * takes ZHS for zh and zh-Hans and ZHT for zh-Hant.
+	 */
+	public function testChineseWithoutARegionHasNoLanguageSystem()
+	{
+		$this->assertSame('DFLT', OtlTags::language('zh', 'DFLT ZHH ZHS ZHT '));
+		$this->assertSame('DFLT', OtlTags::language('zh-Hant', 'DFLT ZHH ZHS ZHT '));
+		$this->assertSame('DFLT', OtlTags::language('zh-Hans', 'DFLT ZHH ZHS ZHT '));
+	}
+
 }

tests/data/fontcache/NotoSansBengali-DevaScript-Synthetic.json

@@ -0,0 +1,92 @@
+{
+    "_": "Generated by composer fontcache:update. See tests/Mpdf/Fonts/ParserGoldenMaster.php.",
+    "fullName": "NotoSansBengali-DevaScript-Synthetic",
+    "mtx": {
+        "GSUBScriptLang": {
+            "DFLT": "DFLT ",
+            "deva": "DFLT "
+        },
+        "GSUBFeatures": {
+            "DFLT": {
+                "DFLT": {
+                    "locl": [
+                        0
+                    ]
+                }
+            },
+            "deva": {
+                "DFLT": {
+                    "locl": [
+                        1
+                    ]
+                }
+            }
+        },
+        "GSUBLookups": [
+            {
+                "Type": 1,
+                "Flag": 0,
+                "SubtableCount": 1,
+                "Subtables": [
+                    88
+                ],
+                "MarkFilteringSet": ""
+            },
+            {
+                "Type": 1,
+                "Flag": 0,
+                "SubtableCount": 1,
+                "Subtables": [
+                    102
+                ],
+                "MarkFilteringSet": ""
+            }
+        ],
+        "GPOSScriptLang": [],
+        "GPOSFeatures": [],
+        "GPOSLookups": [],
+        "MarkGlyphSets": [
+            " 009CD",
+            " 009CD",
+            " 009CD",
+            " 009CD",
+            " 009CD",
+            " 009CD",
+            " 009CD"
+        ],
+        "rtlPUAstr": "",
+        "haskernGPOS": false,
+        "hassmallcapsGSUB": false
+    },
+    "cache": {
+        "GDEFdata.json": {
+            "GlyphClassBases": " 00995| 00996| 00997| 009B7| 009BF| 025CC",
+            "GlyphClassMarks": " 009CD",
+            "GlyphClassLigatures": "",
+            "GlyphClassComponents": "",
+            "MarkGlyphSets": [
+                " 009CD",
+                " 009CD",
+                " 009CD",
+                " 009CD",
+                " 009CD",
+                " 009CD",
+                " 009CD"
+            ],
+            "MarkAttachmentType": []
+        },
+        "GSUB.dat": "114 bytes, sha256 db414921ffcf0b37c47e2d31afc271c25f46f9c9037a44e957346ab2cb8d203c",
+        "GSUBdata.json": [
+            [
+                {
+                    "2453": 0
+                }
+            ],
+            [
+                {
+                    "2453": 0
+                }
+            ]
+        ]
+    }
+}

tests/data/fontcache/NotoSansTC-RegionLangSys-Synthetic.json

@@ -0,0 +1,92 @@
+{
+    "_": "Generated by composer fontcache:update. See tests/Mpdf/Fonts/ParserGoldenMaster.php.",
+    "fullName": "NotoSansTC-RegionLangSys-Synthetic",
+    "mtx": {
+        "GSUBScriptLang": {
+            "hani": "DFLT ZHH  ZHS  ZHT  "
+        },
+        "GSUBFeatures": {
+            "hani": {
+                "ZHH ": {
+                    "locl": [
+                        0
+                    ]
+                },
+                "ZHS ": {
+                    "locl": [
+                        1
+                    ]
+                },
+                "ZHT ": {
+                    "locl": [
+                        2
+                    ]
+                }
+            }
+        },
+        "GSUBLookups": [
+            {
+                "Type": 1,
+                "Flag": 0,
+                "SubtableCount": 1,
+                "Subtables": [
+                    124
+                ],
+                "MarkFilteringSet": ""
+            },
+            {
+                "Type": 1,
+                "Flag": 0,
+                "SubtableCount": 1,
+                "Subtables": [
+                    138
+                ],
+                "MarkFilteringSet": ""
+            },
+            {
+                "Type": 1,
+                "Flag": 0,
+                "SubtableCount": 1,
+                "Subtables": [
+                    152
+                ],
+                "MarkFilteringSet": ""
+            }
+        ],
+        "GPOSScriptLang": [],
+        "GPOSFeatures": [],
+        "GPOSLookups": [],
+        "MarkGlyphSets": [],
+        "rtlPUAstr": "",
+        "haskernGPOS": false,
+        "hassmallcapsGSUB": false
+    },
+    "cache": {
+        "GDEFdata.json": {
+            "GlyphClassBases": " 04E00| 04E09| 04E8C| 09AA8",
+            "GlyphClassMarks": "",
+            "GlyphClassLigatures": "",
+            "GlyphClassComponents": "",
+            "MarkGlyphSets": [],
+            "MarkAttachmentType": []
+        },
+        "GSUB.dat": "164 bytes, sha256 806298eba96771ffa947e91279efff39c5b091473cd2d52e8caf0c74bf95a2a0",
+        "GSUBdata.json": [
+            [
+                {
+                    "39592": 0
+                }
+            ],
+            [
+                {
+                    "39592": 0
+                }
+            ],
+            [
+                {
+                    "39592": 0
+                }
+            ]
+        ]
+    }
+}