From 85ec38efb2eeeb5246c337621e58c50778d1f3aa Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Thu, 17 Sep 2026 10:03:56 +1000 Subject: [PATCH 1/2] Select HarfBuzz's language system for a variant, script, region or extended language subtag, and for Chinese other than zh (#206) OtlTags::language() looked a tag up by its first subtag, with Chinese the one exception. HarfBuzz 14.3.1 reads more of it, and mPDF now tries the candidates in HarfBuzz's order (hb_ot_tags_from_language() and hb_ot_tags_from_complex_language()): 1. -fonnapa, -polyton, -arevmda, -provenc, -fonipa, -geok, -syre, -syrj or -syrn after any language 2. a retired tag read whole: art-lojban, i-hak, i-lux, i-navajo, no-bok, no-nyn, zh-min, zh-min-nan 3. ga-Latg IRT, mnw-TH MONT, ro-MD MOL then ROM 4. Chinese scripts and regions, for every Chinese language 5. an extended language subtag, zh-yue ZHH 6. the language subtag, trying each of its tags in turn Only subtags before the first singleton count, as in HarfBuzz. Ucdn::$ot_languages gains zh and the nineteen other Chinese languages HarfBuzz maps (yue ZHH, lzh ZHT, the rest ZHS), gives ga IRI then IRT, and adds nv as NAV then ATH. Hans makes yue and lzh ZHS; any other script or region leaves them their own tag, where zh and its other members take zh's rules. The five zh- region keys are removed: nothing has read them since #201, and zh-mo's ZHT was wrong. An extended language Ucdn has no key for keeps the language subtag's tags, so ar-afb stays ARA. HarfBuzz's table gives most extended languages their macrolanguage's tag and the rest their ISO 639-3 code; syncing the table is left to its own issue. A font offering only IRT now lays out lang="ga" with it, one offering MOL lays out ro-MD with it rather than ROM (DejaVu Sans and Serif offer both under latn), and NotoSans-Regular's NAV, IPPH and APPH now serve nv, -fonipa and -fonnapa. No golden master or snapshot sets a language, and no fixture moves. Noto-LanguageTags-Synthetic merges subsets of Noto Sans TC, Georgian, Myanmar and Syriac with a 'locl' lookup under each of PGR, IRT, MOL, MONT, PRO, SYRE, IPPH, KGE, ATH, ZHS, ZHT and ZHH. For every row of the issue's table mPDF now draws the glyph hb-shape --language draws, where before it drew the unsubstituted one or ZHS's. Test font: Noto-LanguageTags-Synthetic (Noto Sans TC 2.004, Noto Sans Georgian 2.005, Noto Sans Myanmar 2.107, Noto Sans Syriac 3.000, all OFL 1.1), built in fontTools 4.59.2. Co-Authored-By: Claude Opus 5 (1M context) --- src/Shaper/OtlTags.php | 163 ++++++++-- src/Ucdn.php | 28 +- tests/Mpdf/LanguageTagLangSysTest.php | 64 ++++ tests/Mpdf/Shaper/OtlTagsTest.php | 106 +++++++ .../Noto-LanguageTags-Synthetic.json | 278 ++++++++++++++++++ .../otldump/Noto-LanguageTags-Synthetic.txt | 17 ++ .../shaping/Noto-LanguageTags-Synthetic.txt | 100 +++++++ .../subset/Noto-LanguageTags-Synthetic.json | 123 ++++++++ .../data/ttf/Noto-LanguageTags-Synthetic.ttf | Bin 0 -> 4384 bytes 9 files changed, 851 insertions(+), 28 deletions(-) create mode 100644 tests/Mpdf/LanguageTagLangSysTest.php create mode 100644 tests/data/fontcache/Noto-LanguageTags-Synthetic.json create mode 100644 tests/data/otldump/Noto-LanguageTags-Synthetic.txt create mode 100644 tests/data/shaping/Noto-LanguageTags-Synthetic.txt create mode 100644 tests/data/subset/Noto-LanguageTags-Synthetic.json create mode 100644 tests/data/ttf/Noto-LanguageTags-Synthetic.ttf diff --git a/src/Shaper/OtlTags.php b/src/Shaper/OtlTags.php index 07c8c57e4..84ef52547 100644 --- a/src/Shaper/OtlTags.php +++ b/src/Shaper/OtlTags.php @@ -30,6 +30,46 @@ class OtlTags 'mym2' => 'mymr', ]; + /** + * Variant and script subtags with a language system of their own whatever the language, in the + * order HarfBuzz tests them. + */ + private static $variants = [ + 'fonnapa' => ['APPH'], + 'polyton' => ['PGR '], + 'arevmda' => ['HYE '], + 'provenc' => ['PRO '], + 'fonipa' => ['IPPH'], + 'geok' => ['KGE '], + 'syre' => ['SYRE'], + 'syrj' => ['SYRJ'], + 'syrn' => ['SYRN'], + ]; + + /** + * Retired tags HarfBuzz reads whole, before the extended language rule would read no-nyn as + * Nkole. + */ + private static $retired = [ + 'art-lojban' => ['JBO '], + 'i-hak' => ['ZHS '], + 'i-lux' => ['LTZ '], + 'i-navajo' => ['NAV ', 'ATH '], + 'no-bok' => ['NOR '], + 'no-nyn' => ['NYN '], + 'zh-min' => ['ZHS '], + 'zh-min-nan' => ['ZHS '], + ]; + + /** + * Languages that take another language system for one script or region. + */ + private static $languageSubtags = [ + 'ga' => ['latg' => ['IRT ']], + 'mnw' => ['th' => ['MONT']], + 'ro' => ['md' => ['MOL ', 'ROM ']], + ]; + /** * The Chinese regions with language systems of their own, in the order HarfBuzz looks for them. */ @@ -95,16 +135,7 @@ public static function language($ietf, $available) return ''; } - $subtags = $ietf ? explode('-', strtolower($ietf)) : []; - - $candidates = []; - if (isset($subtags[0]) && $subtags[0] == 'zh') { - $candidates = self::chinese($subtags); - } elseif (isset($subtags[0]) && isset(Ucdn::$ot_languages[$subtags[0]])) { - $candidates = [Ucdn::$ot_languages[$subtags[0]]]; - } - - foreach ($candidates as $langsys) { + foreach (self::languageSystems($ietf) as $langsys) { if (strpos($available, $langsys) !== false) { return $langsys; } @@ -114,40 +145,128 @@ public static function language($ietf, $available) } /** - * The language systems for Chinese, in the order HarfBuzz tries them - * (hb_ot_tags_from_complex_language() in hb-ot-tag-table.hh). + * The language systems for an IETF tag, in the order HarfBuzz 14.3.1 tries them + * (hb_ot_tags_from_language() in hb-ot-tag.cc and hb_ot_tags_from_complex_language() in + * hb-ot-tag-table.hh): * - * The script subtag outranks the region, so zh-Hans-HK is Simplified, except that Traditional - * Chinese in Hong Kong or Macao keeps its region's tag. Macao has its own ZHTM, and where a font - * lacks it takes Hong Kong's ZHH before the default. A tag with neither is Simplified. + * 1. a variant or script that has its own language system in any language, e.g. -polyton + * 2. a retired tag, read whole + * 3. a script or region with its own language system in this language, e.g. ro-MD + * 4. Chinese scripts and regions + * 5. an extended language subtag, e.g. zh-yue + * 6. the language subtag + * + * Only the subtags before the first singleton count, so an extension or private use subtag + * selects nothing. A script is the subtag after the language; a region may be anywhere. * - * The zh- keys of Ucdn::$ot_languages are not read: they hold one tag per region, and nothing for - * the script or for zh alone. + * An extended language Ucdn::$ot_languages has no key for keeps the language subtag's tags, so + * ar-afb stays ARA. HarfBuzz's far larger table gives most extended languages their + * macrolanguage's tag, and takes the rest as their own ISO 639-3 code. * - * @param string[] $subtags The lower-cased subtags of a tag whose language is zh + * @param string $ietf * * @return string[] */ - private static function chinese(array $subtags) + private static function languageSystems($ietf) { + $tag = strtolower((string) $ietf); + $subtags = explode('-', $tag); + + foreach ($subtags as $i => $subtag) { + if ($i > 0 && strlen($subtag) == 1 && isset($subtags[$i + 1])) { + $subtags = array_slice($subtags, 0, $i); + break; + } + } + if ($subtags[0] == 'x') { + return []; + } + + $following = array_slice($subtags, 1); + foreach (self::$variants as $variant => $langsys) { + if (in_array($variant, $following, true)) { + return $langsys; + } + } + + if (isset(self::$retired[$tag])) { + return self::$retired[$tag]; + } + + $language = $subtags[0]; $script = isset($subtags[1]) ? $subtags[1] : ''; + if (isset(self::$languageSubtags[$language])) { + foreach (self::$languageSubtags[$language] as $subtag => $langsys) { + if (strlen($subtag) == 4 ? $script == $subtag : in_array($subtag, $following, true)) { + return $langsys; + } + } + } + + $own = self::tags($language); + + if ($own && in_array($own[0], ['ZHS ', 'ZHT ', 'ZHH '], true)) { + $chinese = self::chinese($own[0], $script, $following); + if ($chinese) { + return $chinese; + } + } + + if (preg_match('/^[a-z]{3}$/', $script) && self::tags($script)) { + return self::tags($script); + } + + return $own; + } + + /** + * The Chinese language systems a script or region gives, in the order HarfBuzz tries them. + * + * The script subtag outranks the region, so zh-Hans-HK is Simplified, except that Traditional + * Chinese in Hong Kong or Macao keeps its region's tag. Macao has its own ZHTM, and where a font + * lacks it takes Hong Kong's ZHH before the default. + * + * Only Hans moves Cantonese (yue, ZHH) and Literary Chinese (lzh, ZHT) off their own tags. Every + * other Chinese language is Simplified by default and takes zh's rules. + * + * @param string $own The language's own tag + * @param string $script The subtag after the language + * @param string[] $following The subtags after the language + * + * @return string[] The language systems, or none where the language's own tag stands + */ + private static function chinese($own, $script, array $following) + { if ($script == 'hans') { return ['ZHS ']; } + if ($own != 'ZHS ') { + return []; + } if ($script == 'hant') { - $region = isset($subtags[2]) ? $subtags[2] : ''; + $region = isset($following[1]) ? $following[1] : ''; return $region == 'hk' || $region == 'mo' ? self::$chineseRegions[$region] : ['ZHT ']; } foreach (self::$chineseRegions as $region => $langsys) { - if (in_array($region, $subtags, true)) { + if (in_array($region, $following, true)) { return $langsys; } } - return ['ZHS ']; + return []; + } + + /** + * @param string $language A language subtag + * + * @return string[] Its language systems in Ucdn::$ot_languages, or none + */ + private static function tags($language) + { + return isset(Ucdn::$ot_languages[$language]) ? (array) Ucdn::$ot_languages[$language] : []; } /** diff --git a/src/Ucdn.php b/src/Ucdn.php index 9378db229..103f7f741 100644 --- a/src/Ucdn.php +++ b/src/Ucdn.php @@ -471,12 +471,17 @@ public static function get_script($code) 'bxr' => 'RBU ', /* Russian Buriat */ 'byn' => 'BIL ', /* Bilen */ 'ca' => 'CAT ', /* Catalan */ + 'cdo' => 'ZHS ', /* Min Dong Chinese */ 'ce' => 'CHE ', /* Chechen */ 'ceb' => 'CEB ', /* Cebuano */ 'chp' => 'CHP ', /* Chipewyan */ 'chr' => 'CHR ', /* Cherokee */ + 'cjy' => 'ZHS ', /* Jinyu Chinese */ 'ckt' => 'CHK ', /* Chukchi */ + 'cmn' => 'ZHS ', /* Mandarin Chinese */ + 'cnp' => 'ZHS ', /* Northern Ping Chinese */ 'cop' => 'COP ', /* Coptic */ + 'cpx' => 'ZHS ', /* Pu-Xian Chinese */ 'cr' => 'CRE ', /* Cree */ 'crh' => 'CRT ', /* Crimean Tatar */ 'crj' => 'ECR ', /* [Southern] East Cree */ @@ -484,10 +489,13 @@ public static function get_script($code) 'crm' => 'MCR ', /* Moose Cree */ 'crx' => 'CRR ', /* Carrier */ 'cs' => 'CSY ', /* Czech */ + 'csp' => 'ZHS ', /* Southern Ping Chinese */ 'cu' => 'CSL ', /* Church Slavic */ 'cv' => 'CHU ', /* Chuvash */ 'cwd' => 'DCR ', /* Woods Cree */ 'cy' => 'WEL ', /* Welsh */ + 'czh' => 'ZHS ', /* Huizhou Chinese */ + 'czo' => 'ZHS ', /* Min Zhong Chinese */ 'da' => 'DAN ', /* Danish */ 'dap' => 'NIS ', /* Nisi (India) */ 'dar' => 'DAR ', /* Dargwa */ @@ -522,9 +530,10 @@ public static function get_script($code) 'fr' => 'FRA ', /* French */ 'fur' => 'FRL ', /* Friulian */ 'fy' => 'FRI ', /* Western Frisian */ - 'ga' => 'IRI ', /* Irish */ + 'ga' => ['IRI ', 'IRT '], /* Irish */ 'gaa' => 'GAD ', /* Ga */ 'gag' => 'GAG ', /* Gagauz */ + 'gan' => 'ZHS ', /* Gan Chinese */ 'gbm' => 'GAW ', /* Garhwali */ 'gd' => 'GAE ', /* Scottish Gaelic */ 'gez' => 'GEZ ', /* Ge'ez */ @@ -538,6 +547,7 @@ public static function get_script($code) 'guk' => 'GMZ ', /* Gumuz */ 'gv' => 'MNX ', /* Manx Gaelic */ 'ha' => 'HAU ', /* Hausa */ + 'hak' => 'ZHS ', /* Hakka Chinese */ 'har' => 'HRI ', /* Harari */ 'haw' => 'HAW ', /* Hawaiin */ 'he' => 'IWR ', /* Hebrew */ @@ -545,11 +555,13 @@ public static function get_script($code) 'hil' => 'HIL ', /* Hiligaynon */ 'hnd' => 'HND ', /* [Southern] Hindko */ 'hne' => 'CHH ', /* Chattisgarhi */ + 'hnm' => 'ZHS ', /* Hainanese */ 'hno' => 'HND ', /* [Northern] Hindko */ 'hoc' => 'HO ', /* Ho */ 'hoj' => 'HAR ', /* Harauti */ 'hr' => 'HRV ', /* Croatian */ 'hsb' => 'USB ', /* Upper Sorbian */ + 'hsn' => 'ZHS ', /* Xiang Chinese */ 'ht' => 'HAI ', /* Haitian */ 'hu' => 'HUN ', /* Hungarian */ 'hy' => 'HYE ', /* Armenian */ @@ -619,10 +631,12 @@ public static function get_script($code) 'lt' => 'LTH ', /* Lithuanian */ 'lu' => 'LUB ', /* Luba-Katanga */ 'lua' => 'LUB ', /* Luba-Kasai */ + 'luh' => 'ZHS ', /* Leizhou Chinese */ 'luo' => 'LUO ', /* Luo (Kenya and Tanzania) */ 'lus' => 'MIZ ', /* Mizo */ 'luy' => 'LUH ', /* Luhya [macrolanguage] */ 'lv' => 'LVI ', /* Latvian */ + 'lzh' => 'ZHT ', /* Literary Chinese */ 'lzz' => 'LAZ ', /* Laz */ 'mai' => 'MTH ', /* Maithili */ 'mdc' => 'MLE ', /* Male (Papua New Guinea) */ @@ -638,6 +652,7 @@ public static function get_script($code) 'mnc' => 'MCH ', /* Manchu */ 'mni' => 'MNI ', /* Manipuri */ 'mnk' => 'MND ', /* Mandinka */ + 'mnp' => 'ZHS ', /* Min Bei Chinese */ 'mns' => 'MAN ', /* Mansi */ 'mnw' => 'MON ', /* Mon */ 'mo' => 'MOL ', /* Moldavian */ @@ -652,6 +667,7 @@ public static function get_script($code) 'mym' => 'MEN ', /* Me'en */ 'myv' => 'ERZ ', /* Erzya */ 'nag' => 'NAG ', /* Naga-Assamese */ + 'nan' => 'ZHS ', /* Min Nan Chinese */ 'nb' => 'NOR ', /* Norwegian Bokmål */ 'nco' => 'SIB ', /* Sibe */ 'nd' => 'NDB ', /* [North] Ndebele */ @@ -670,6 +686,7 @@ public static function get_script($code) 'nr' => 'NDB ', /* [South] Ndebele */ 'nsk' => 'NAS ', /* Naskapi */ 'nso' => 'SOT ', /* [Northern] Sotho */ + 'nv' => ['NAV ', 'ATH '], /* Navajo */ 'ny' => 'CHI ', /* Nyanja */ 'nyn' => 'NKL ', /* Nkole */ 'oc' => 'OCI ', /* Occitan (post 1500) */ @@ -711,6 +728,7 @@ public static function get_script($code) 'shn' => 'SHN ', /* Shan */ 'si' => 'SNH ', /* Sinhala */ 'sid' => 'SID ', /* Sidamo */ + 'sjc' => 'ZHS ', /* Shaojiang Chinese */ 'sjd' => 'KSM ', /* Kildin Sami */ 'sk' => 'SKY ', /* Slovak */ 'skr' => 'SRK ', /* Seraiki */ @@ -765,6 +783,7 @@ public static function get_script($code) 'wbm' => 'WA ', /* Wa */ 'wbr' => 'WAG ', /* Wagdi */ 'wo' => 'WLF ', /* Wolof */ + 'wuu' => 'ZHS ', /* Wu Chinese */ 'xal' => 'KLM ', /* Kalmyk */ 'xh' => 'XHS ', /* Xhosa */ 'xom' => 'KMO ', /* Komo (Sudan) */ @@ -773,13 +792,10 @@ public static function get_script($code) 'yid' => 'JII ', /* Yiddish */ 'yo' => 'YBA ', /* Yoruba */ 'yso' => 'NIS ', /* Nisi (China) */ + 'yue' => 'ZHH ', /* Yue Chinese */ + 'zh' => 'ZHS ', /* Chinese */ 'zne' => 'ZND ', /* Zande */ 'zu' => 'ZUL ', /* Zulu */ - 'zh-cn' => 'ZHS ', /* Chinese (China) */ - 'zh-hk' => 'ZHH ', /* Chinese (Hong Kong) */ - 'zh-mo' => 'ZHT ', /* Chinese (Macao) */ - 'zh-sg' => 'ZHS ', /* Chinese (Singapore) */ - 'zh-tw' => 'ZHT ', /* Chinese (Taiwan) */ ]; // hb-unicode.h diff --git a/tests/Mpdf/LanguageTagLangSysTest.php b/tests/Mpdf/LanguageTagLangSysTest.php new file mode 100644 index 000000000..e5e549078 --- /dev/null +++ b/tests/Mpdf/LanguageTagLangSysTest.php @@ -0,0 +1,64 @@ + 'utf-8', + 'fontDir' => [__DIR__ . '/../data/ttf'], + 'fontdata' => ['notolanguagetagssynthetic' => [ + 'R' => 'Noto-LanguageTags-Synthetic.ttf', + 'useOTL' => 0xFF, + ]], + 'default_font' => 'notolanguagetagssynthetic', + ]); + $mpdf->WriteHTML(sprintf('

&#x%04X;

', $lang, $character)); + + $this->assertSame([$expected], array_values(unpack('N*', mb_convert_encoding($mpdf->drawnText[0], 'UTF-32BE', 'UTF-8')))); + } + + public function languages() + { + return [ + 'el-polyton, PGR' => ['el-polyton', 0x03B1, 0x03B2], + 'ga-Latg, IRT' => ['ga-Latg', 0x61, 0x69], + 'ro-MD, MOL' => ['ro-MD', 0x61, 0x6D], + 'mnw-TH, MONT' => ['mnw-TH', 0x1000, 0x1001], + 'oc-provenc, PRO' => ['oc-provenc', 0x61, 0x70], + 'syr-Syre, SYRE' => ['syr-Syre', 0x0710, 0x0712], + 'en-fonipa, IPPH' => ['en-fonipa', 0x61, 0x68], + 'ka-Geok, KGE' => ['ka-Geok', 0x10A0, 0x10A1], + 'yue, ZHH' => ['yue', 0x9AA8, 0x4E09], + 'yue-Hant-HK, ZHH' => ['yue-Hant-HK', 0x9AA8, 0x4E09], + 'cmn-Hans, ZHS' => ['cmn-Hans', 0x9AA8, 0x4E00], + 'lzh, ZHT' => ['lzh', 0x9AA8, 0x4E8C], + 'zh-yue, ZHH' => ['zh-yue', 0x9AA8, 0x4E09], + 'zh-lzh, ZHT' => ['zh-lzh', 0x9AA8, 0x4E8C], + 'ga, IRT after IRI' => ['ga', 0x61, 0x69], + 'nv, ATH after NAV' => ['nv', 0x61, 0x74], + 'cmn-Hant-TW, ZHT' => ['cmn-Hant-TW', 0x9AA8, 0x4E8C], + 'yue-Hans, ZHS' => ['yue-Hans', 0x9AA8, 0x4E00], + 'ro, ROM not offered' => ['ro', 0x61, 0x61], + ]; + } + +} diff --git a/tests/Mpdf/Shaper/OtlTagsTest.php b/tests/Mpdf/Shaper/OtlTagsTest.php index fbdc26eb2..5a7be3ed7 100644 --- a/tests/Mpdf/Shaper/OtlTagsTest.php +++ b/tests/Mpdf/Shaper/OtlTagsTest.php @@ -164,4 +164,110 @@ public function testMacaoFallsBackToHongKong() $this->assertSame('DFLT', OtlTags::language('zh-MO', 'DFLT ZHS ZHT ')); } + /** + * @dataProvider complexLanguages + */ + public function testSubtagsBeyondTheLanguageSelectTheLanguageSystemHarfBuzzDoes($ietf, $expected) + { + $offered = 'DFLT ATH ELL IPPH IRI IRT KAT KGE MOL MON MONT NAV OCI PGR PRO ROM SYR SYRE ZHH ZHS ZHT ZHTM '; + + $this->assertSame($expected, OtlTags::language($ietf, $offered)); + } + + public function complexLanguages() + { + return [ + 'el-polyton' => ['el-polyton', 'PGR '], + 'ga-Latg' => ['ga-Latg', 'IRT '], + 'ro-MD' => ['ro-MD', 'MOL '], + 'mnw-TH' => ['mnw-TH', 'MONT'], + 'oc-provenc' => ['oc-provenc', 'PRO '], + 'syr-Syre' => ['syr-Syre', 'SYRE'], + 'en-fonipa' => ['en-fonipa', 'IPPH'], + 'ka-Geok' => ['ka-Geok', 'KGE '], + 'yue' => ['yue', 'ZHH '], + 'yue-Hant-HK' => ['yue-Hant-HK', 'ZHH '], + 'cmn-Hans' => ['cmn-Hans', 'ZHS '], + 'lzh' => ['lzh', 'ZHT '], + 'zh-yue' => ['zh-yue', 'ZHH '], + 'zh-lzh' => ['zh-lzh', 'ZHT '], + 'a variant after a region' => ['el-GR-polyton', 'PGR '], + 'the region, after the script' => ['ro-Latn-MD', 'MOL '], + 'upper case' => ['RO-MD', 'MOL '], + 'Irish without Latg' => ['ga-IE', 'IRI '], + 'Mon outside Thailand' => ['mnw-MM', 'MON '], + 'a variant or region after a private use subtag' => ['ro-x-md', 'ROM '], + 'a variant after an extension' => ['el-u-polyton', 'ELL '], + 'a private use tag' => ['x-fonipa', 'DFLT'], + 'Cantonese, Traditional, in Taiwan' => ['yue-Hant-TW', 'ZHH '], + 'Cantonese, in Macao' => ['yue-MO', 'ZHH '], + 'Cantonese, Simplified' => ['yue-Hans', 'ZHS '], + 'Literary Chinese, in Hong Kong' => ['lzh-HK', 'ZHT '], + 'Literary Chinese, Simplified' => ['lzh-Hans', 'ZHS '], + 'Mandarin, Traditional, in Taiwan' => ['cmn-Hant-TW', 'ZHT '], + 'Mandarin, in Macao' => ['cmn-MO', 'ZHTM'], + 'Hakka, in Hong Kong' => ['hak-HK', 'ZHH '], + 'Min Nan, alone' => ['nan', 'ZHS '], + 'zh-yue, in Hong Kong, by the region' => ['zh-yue-HK', 'ZHH '], + 'zh-yue, Simplified, by the extended language' => ['zh-yue-Hans', 'ZHH '], + 'zh-cmn, Traditional, by the extended language' => ['zh-cmn-Hant', 'ZHS '], + 'Navajo' => ['nv', 'NAV '], + 'Navajo, retired' => ['i-navajo', 'NAV '], + ]; + } + + /** + * @dataProvider languageFallbacks + */ + public function testALanguageWithMoreThanOneTagTakesTheFirstTheScriptOffers($ietf, $offered, $expected) + { + $this->assertSame($expected, OtlTags::language($ietf, $offered)); + } + + public function languageFallbacks() + { + return [ + 'Moldova, with MOL' => ['ro-MD', 'DFLT MOL ROM ', 'MOL '], + 'Moldova, without MOL' => ['ro-MD', 'DFLT ROM ', 'ROM '], + 'Moldova, with neither' => ['ro-MD', 'DFLT ENG ', 'DFLT'], + 'Irish, with IRI' => ['ga', 'DFLT IRI IRT ', 'IRI '], + 'Irish, without IRI' => ['ga', 'DFLT IRT ', 'IRT '], + 'Irish, with neither' => ['ga', 'DFLT ENG ', 'DFLT'], + 'Navajo, without NAV' => ['nv', 'DFLT ATH ', 'ATH '], + 'Irish Traditional, without IRT' => ['ga-Latg', 'DFLT IRI ', 'DFLT'], + ]; + } + + /** + * HarfBuzz reads these whole. Read a subtag at a time, no-nyn would be Nkole. + * + * @dataProvider retiredTags + */ + public function testARetiredTagIsReadWhole($ietf, $expected) + { + $this->assertSame($expected, OtlTags::language($ietf, 'DFLT JBO LTZ NKL NOR NYN ZHS ')); + } + + public function retiredTags() + { + return [ + 'no-bok' => ['no-bok', 'NOR '], + 'no-nyn' => ['no-nyn', 'NYN '], + 'zh-min' => ['zh-min', 'ZHS '], + 'zh-min-nan' => ['zh-min-nan', 'ZHS '], + 'i-hak' => ['i-hak', 'ZHS '], + 'i-lux' => ['i-lux', 'LTZ '], + 'art-lojban' => ['art-lojban', 'JBO '], + ]; + } + + /** + * Ucdn::$ot_languages has no key for most extended languages. HarfBuzz's table gives most of them + * their macrolanguage's tag, as it gives Gulf Arabic ARA, so mPDF keeps the language subtag's. + */ + public function testAnExtendedLanguageWithNoTagKeepsTheLanguageSubtagsTag() + { + $this->assertSame('ARA ', OtlTags::language('ar-afb', 'DFLT ARA ')); + } + } diff --git a/tests/data/fontcache/Noto-LanguageTags-Synthetic.json b/tests/data/fontcache/Noto-LanguageTags-Synthetic.json new file mode 100644 index 000000000..d3981bec2 --- /dev/null +++ b/tests/data/fontcache/Noto-LanguageTags-Synthetic.json @@ -0,0 +1,278 @@ +{ + "_": "Generated by composer fontcache:update. See tests/Mpdf/Fonts/ParserGoldenMaster.php.", + "fullName": "Noto-LanguageTags-Synthetic", + "mtx": { + "GSUBScriptLang": { + "geor": "DFLT KGE ", + "grek": "DFLT PGR ", + "hani": "DFLT ZHH ZHS ZHT ", + "latn": "DFLT ATH IPPH IRT MOL PRO ", + "mym2": "DFLT MONT ", + "syrc": "DFLT SYRE " + }, + "GSUBFeatures": { + "geor": { + "KGE ": { + "locl": [ + 0 + ] + } + }, + "grek": { + "PGR ": { + "locl": [ + 1 + ] + } + }, + "hani": { + "ZHH ": { + "locl": [ + 2 + ] + }, + "ZHS ": { + "locl": [ + 3 + ] + }, + "ZHT ": { + "locl": [ + 4 + ] + } + }, + "latn": { + "ATH ": { + "locl": [ + 5 + ] + }, + "IPPH": { + "locl": [ + 6 + ] + }, + "IRT ": { + "locl": [ + 7 + ] + }, + "MOL ": { + "locl": [ + 8 + ] + }, + "PRO ": { + "locl": [ + 9 + ] + } + }, + "mym2": { + "MONT": { + "locl": [ + 10 + ] + } + }, + "syrc": { + "SYRE": { + "locl": [ + 11 + ] + } + } + }, + "GSUBLookups": [ + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 426 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 446 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 466 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 480 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 494 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 514 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 528 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 542 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 556 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 570 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 590 + ], + "MarkFilteringSet": "" + }, + { + "Type": 1, + "Flag": 0, + "SubtableCount": 1, + "Subtables": [ + 610 + ], + "MarkFilteringSet": "" + } + ], + "GPOSScriptLang": [], + "GPOSFeatures": [], + "GPOSLookups": [], + "MarkGlyphSets": [], + "rtlPUAstr": "", + "haskernGPOS": false, + "hassmallcapsGSUB": false + }, + "cache": { + "GDEFdata.json": { + "GlyphClassBases": " 00061| 00068| 00069| 0006D| 00070| 00074| 003B1| 003B2| 00710| 00712| 01000| 01001| 010A0| 010A1| 04E00| 04E09| 04E8C| 09AA8", + "GlyphClassMarks": "", + "GlyphClassLigatures": "", + "GlyphClassComponents": "", + "MarkGlyphSets": [], + "MarkAttachmentType": [] + }, + "GSUB.dat": "622 bytes, sha256 57e022f98e985373333d011b0ae7d7a39d73da90186ea4c1a9b190b141d8f481", + "GSUBdata.json": [ + [ + { + "4256": 0 + } + ], + [ + { + "945": 0 + } + ], + [ + { + "39592": 0 + } + ], + [ + { + "39592": 0 + } + ], + [ + { + "39592": 0 + } + ], + [ + { + "97": 0 + } + ], + [ + { + "97": 0 + } + ], + [ + { + "97": 0 + } + ], + [ + { + "97": 0 + } + ], + [ + { + "97": 0 + } + ], + [ + { + "4096": 0 + } + ], + [ + { + "1808": 0 + } + ] + ] + } +} diff --git a/tests/data/otldump/Noto-LanguageTags-Synthetic.txt b/tests/data/otldump/Noto-LanguageTags-Synthetic.txt new file mode 100644 index 000000000..dbf2b3998 --- /dev/null +++ b/tests/data/otldump/Noto-LanguageTags-Synthetic.txt @@ -0,0 +1,17 @@ +

GDEF table

+

Glyph classes

+

Glyph class 1

+
Base glyph (single character, spacing glyph)
+
a h i m p t α β ܐ ܒ က ခ Ⴀ Ⴁ 一 三 二 骨
+

GSUB Tables

+

GSUB Scripts & Languages

+
+
geor
KGE : locl
grek
PGR : locl
hani
ZHH : locl
ZHS : locl
ZHT : locl
latn
ATH : locl
IPPH: locl
IRT : locl
MOL : locl
PRO : locl
mym2
MONT: locl
syrc
SYRE: locl
+
+
GPOS table not defined
+ +=== detail: script syrc language DFLT === +

GSUB Tables

+
This font's GSUB script "syrc" offers no language system "DFLT". It has: SYRE
+ +
GPOS table not defined
diff --git a/tests/data/shaping/Noto-LanguageTags-Synthetic.txt b/tests/data/shaping/Noto-LanguageTags-Synthetic.txt new file mode 100644 index 000000000..c1ebd4bd4 --- /dev/null +++ b/tests/data/shaping/Noto-LanguageTags-Synthetic.txt @@ -0,0 +1,100 @@ +=== latin === +0xFF 0041 0056 0041 0054 0061 0072 => 0041 0056 0041 0054 0061 0072 group=CCCCCC +0x80 0041 0056 0041 0054 0061 0072 => 0041 0056 0041 0054 0061 0072 group=CCCCCC +0xFF +kern -liga 0041 0056 0041 0054 0061 0072 => 0041 0056 0041 0054 0061 0072 group=CCCCCC +=== cyrillic === +0xFF 0416 0430 0439 => 0416 0430 0439 group=CCC +0x80 0416 0430 0439 => 0416 0430 0439 group=CCC +0xFF +kern -liga 0416 0430 0439 => 0416 0430 0439 group=CCC +=== greek === +0xFF 03B1 03B2 03C2 => 03B1 03B2 03C2 group=CCC +0x80 03B1 03B2 03C2 => 03B1 03B2 03C2 group=CCC +0xFF +kern -liga 03B1 03B2 03C2 => 03B1 03B2 03C2 group=CCC +=== hiragana === +0xFF 3042 3043 3044 => 3042 3043 3044 group=CCC +0x80 3042 3043 3044 => 3042 3043 3044 group=CCC +0xFF +kern -liga 3042 3043 3044 => 3042 3043 3044 group=CCC +=== arabic === +0xFF 0628 0640 0645 0644 0627 => 0628 0640 0645 0644 0627 group=CCCCC +0x80 0628 0640 0645 0644 0627 => 0628 0640 0645 0644 0627 group=CCCCC +0xFF +kern -liga 0628 0640 0645 0644 0627 => 0628 0640 0645 0644 0627 group=CCCCC +=== syriac === +0xFF 0710 0712 0713 0715 => 0710 0712 0713 0715 group=CCCC +0x80 0710 0712 0713 0715 => 0710 0712 0713 0715 group=CCCC +0xFF +kern -liga 0710 0712 0713 0715 => 0710 0712 0713 0715 group=CCCC +=== nko === +0xFF 07CA 07CB 07CC => 07CA 07CB 07CC group=CCC +0x80 07CA 07CB 07CC => 07CA 07CB 07CC group=CCC +0xFF +kern -liga 07CA 07CB 07CC => 07CA 07CB 07CC group=CCC +=== devanagari === +0xFF 0915 094D 0937 093F => 0915 094D 0937 093F group=CCCC +0x80 0915 094D 0937 093F => 0915 094D 0937 093F group=CCCC +0xFF +kern -liga 0915 094D 0937 093F => 0915 094D 0937 093F group=CCCC +=== bengali === +0xFF 0995 09CD 09B7 09BF => 0995 09CD 09B7 09BF group=CCCC +0x80 0995 09CD 09B7 09BF => 0995 09CD 09B7 09BF group=CCCC +0xFF +kern -liga 0995 09CD 09B7 09BF => 0995 09CD 09B7 09BF group=CCCC +=== gurmukhi === +0xFF 0A15 0A4D 0A38 0A3F => 0A15 0A4D 0A38 0A3F group=CCCC +0x80 0A15 0A4D 0A38 0A3F => 0A15 0A4D 0A38 0A3F group=CCCC +0xFF +kern -liga 0A15 0A4D 0A38 0A3F => 0A15 0A4D 0A38 0A3F group=CCCC +=== tamil === +0xFF 0B95 0BCD 0BB7 0BBF => 0B95 0BCD 0BB7 0BBF group=CCCC +0x80 0B95 0BCD 0BB7 0BBF => 0B95 0BCD 0BB7 0BBF group=CCCC +0xFF +kern -liga 0B95 0BCD 0BB7 0BBF => 0B95 0BCD 0BB7 0BBF group=CCCC +=== malayalam === +0xFF 0D15 0D4D 0D37 0D3F => 0D15 0D4D 0D37 0D3F group=CCCC +0x80 0D15 0D4D 0D37 0D3F => 0D15 0D4D 0D37 0D3F group=CCCC +0xFF +kern -liga 0D15 0D4D 0D37 0D3F => 0D15 0D4D 0D37 0D3F group=CCCC +=== sinhala === +0xFF 0D9A 0DCA 0DBB 0DBB => 0D9A 0DCA 0DBB 0DBB group=CCCC +0x80 0D9A 0DCA 0DBB 0DBB => 0D9A 0DCA 0DBB 0DBB group=CCCC +0xFF +kern -liga 0D9A 0DCA 0DBB 0DBB => 0D9A 0DCA 0DBB 0DBB group=CCCC +=== khmer === +0xFF 1780 17D2 1781 17C1 => 1780 17D2 1781 17C1 group=CCCC +0x80 1780 17D2 1781 17C1 => 1780 17D2 1781 17C1 group=CCCC +0xFF +kern -liga 1780 17D2 1781 17C1 => 1780 17D2 1781 17C1 group=CCCC +=== thai === +0xFF 0E01 0E34 0E48 0E23 => 0E01 0E34 0E48 0E23 group=CCCC +0x80 0E01 0E34 0E48 0E23 => 0E01 0E34 0E48 0E23 group=CCCC +0xFF +kern -liga 0E01 0E34 0E48 0E23 => 0E01 0E34 0E48 0E23 group=CCCC +=== lao === +0xFF 0E81 0EB4 0E8D => 0E81 0EB4 0E8D group=CCC +0x80 0E81 0EB4 0E8D => 0E81 0EB4 0E8D group=CCC +0xFF +kern -liga 0E81 0EB4 0E8D => 0E81 0EB4 0E8D group=CCC +=== myanmar === +0xFF 1000 103A 1039 1001 => 1000 103A 1039 1001 group=CCCC +0x80 1000 103A 1039 1001 => 1000 103A 1039 1001 group=CCCC +0xFF +kern -liga 1000 103A 1039 1001 => 1000 103A 1039 1001 group=CCCC +=== new tai lue === +0xFF 1980 19B0 1981 => 1980 19B0 1981 group=CCC +0x80 1980 19B0 1981 => 1980 19B0 1981 group=CCC +0xFF +kern -liga 1980 19B0 1981 => 1980 19B0 1981 group=CCC +=== cham === +0xFF AA00 AA33 AA01 => AA00 AA33 AA01 group=CCC +0x80 AA00 AA33 AA01 => AA00 AA33 AA01 group=CCC +0xFF +kern -liga AA00 AA33 AA01 => AA00 AA33 AA01 group=CCC +=== tai tham === +0xFF 1A20 1A60 1A21 => 1A20 1A60 1A21 group=CCC +0x80 1A20 1A60 1A21 => 1A20 1A60 1A21 group=CCC +0xFF +kern -liga 1A20 1A60 1A21 => 1A20 1A60 1A21 group=CCC +=== mixed scripts === +0xFF 0041 0628 0915 0042 => 0041 0628 0915 0042 group=CCCC +0x80 0041 0628 0915 0042 => 0041 0628 0915 0042 group=CCCC +0xFF +kern -liga 0041 0628 0915 0042 => 0041 0628 0915 0042 group=CCCC +=== spaced === +0xFF 0041 0020 0628 0020 0042 => 0041 0020 0628 0020 0042 group=CSCSC +0x80 0041 0020 0628 0020 0042 => 0041 0020 0628 0020 0042 group=CSCSC +0xFF +kern -liga 0041 0020 0628 0020 0042 => 0041 0020 0628 0020 0042 group=CSCSC +=== font characters 0 === +0xFF 0020 0061 0068 0069 006D 0070 0074 03B1 => 0020 0061 0068 0069 006D 0070 0074 03B1 group=SCCCCCCC +0x80 0020 0061 0068 0069 006D 0070 0074 03B1 => 0020 0061 0068 0069 006D 0070 0074 03B1 group=SCCCCCCC +0xFF +kern -liga 0020 0061 0068 0069 006D 0070 0074 03B1 => 0020 0061 0068 0069 006D 0070 0074 03B1 group=SCCCCCCC +=== font characters 1 === +0xFF 03B2 0710 0712 1000 1001 10A0 10A1 4E00 => 03B2 0710 0712 1000 1001 10A0 10A1 4E00 group=CCCCCCCC +0x80 03B2 0710 0712 1000 1001 10A0 10A1 4E00 => 03B2 0710 0712 1000 1001 10A0 10A1 4E00 group=CCCCCCCC +0xFF +kern -liga 03B2 0710 0712 1000 1001 10A0 10A1 4E00 => 03B2 0710 0712 1000 1001 10A0 10A1 4E00 group=CCCCCCCC +=== font characters 2 === +0xFF 4E09 4E8C 9AA8 => 4E09 4E8C 9AA8 group=CCC +0x80 4E09 4E8C 9AA8 => 4E09 4E8C 9AA8 group=CCC +0xFF +kern -liga 4E09 4E8C 9AA8 => 4E09 4E8C 9AA8 group=CCC diff --git a/tests/data/subset/Noto-LanguageTags-Synthetic.json b/tests/data/subset/Noto-LanguageTags-Synthetic.json new file mode 100644 index 000000000..d5ff0a6b3 --- /dev/null +++ b/tests/data/subset/Noto-LanguageTags-Synthetic.json @@ -0,0 +1,123 @@ +{ + "_": "Generated by composer subset:update. See tests/Mpdf/Fonts/SubsetGoldenMaster.php.", + "useOTL=0x00": { + "characters": 20, + "makeSubset": { + "bytes": 3748, + "sha256": "9f18a7ddb377e51ff54f312b7c8bc2cce478ba64dc5b6d3002fdd3146ec3bd3e", + "tables": { + "OS/2": "96 bytes, checksum 0x5E34F64F, sha256 1f7c612aa4c82cb015efd76dbb01962f913e0d7e903b774dcee570de35b6f300", + "cmap": "180 bytes, checksum 0x652E569B, sha256 0288429daf7b8865e16b8bcdeddd51c1bff64412c404a5500a2fb415773add80", + "gasp": "8 bytes, checksum 0x00000010, sha256 4ca731f86ad506ac0e320283dc9461926346de3c4d91d539ea7f6620d3826940", + "glyf": "2124 bytes, checksum 0x3FFB0FD4, sha256 b1fbcae81219663740147f5ba8f45ec41827a76abb62a4cd0117f879b8fec7bc", + "head": "54 bytes, checksum 0x3102FEA8, sha256 1084ee2f2873d9d4c12eb4c6280133eba5aeb2c72800d7e73b4db0eba137135d", + "hhea": "36 bytes, checksum 0x09760125, sha256 0d1d7c23dcce8319b99caf64e47a4c211ffb1a4c4e48f1ee52732ed062504a19", + "hmtx": "80 bytes, checksum 0x3A6004F8, sha256 89d2ba3f47d9a102512952c5c94174c3499564921236a5550262d13616790d9a", + "loca": "42 bytes, checksum 0x13FC11D8, sha256 96852a75ef627da304ababd4cefec56a809aafeb8a1e1317221978dfd674c2a2", + "maxp": "32 bytes, checksum 0x001D0051, sha256 09189d869a40d53d0f84b16e4213dea19289b92a604cd9b191d68d0f59555662", + "name": "870 bytes, checksum 0x4B4D66E3, sha256 1a8da8b6114f22a951965f10e8769199a8180dcbba8a350d6d0061cfa1f2381d", + "post": "32 bytes, checksum 0xFF860032, sha256 22dd499f08932214bde96943f3882f3619324478217095c049b68a57c6c10693" + }, + "maxUni": 65535, + "defaultWidth": 1000, + "codeToGlyph": "20 entries, sha256 ec8a2f22af881109af82f32d84c6a9a60073f2010b7cb5c3fa316d1f3d62ed8f" + }, + "makeSubsetSIP": { + "bytes": 3680, + "sha256": "d8d5a303ef12a328fa3041e531fe646c3dcd5e5e649ed2fafa57a5177a64f0c9", + "tables": { + "OS/2": "96 bytes, checksum 0x5D08CEBB, sha256 40ec5f44c8d585d5468d3663ed6a140206deda3b435edc25b3e81da65d2b70b6", + "cmap": "110 bytes, checksum 0x00A40125, sha256 cd026da39ad03d826ec61164e29c1977344d4b17ae03617fe33822a7434ab827", + "gasp": "8 bytes, checksum 0x00000010, sha256 4ca731f86ad506ac0e320283dc9461926346de3c4d91d539ea7f6620d3826940", + "glyf": "2124 bytes, checksum 0x3FFB0FD4, sha256 8e296933650350db28d99a62b6898cacd5dda4ca47e947f2c7b7d69fd94efa5c", + "head": "54 bytes, checksum 0x3102FEA8, sha256 cbe94e45b3b3ddbd77d019fb0b37fe976156f63737f295ecf319897fbc490951", + "hhea": "36 bytes, checksum 0x09760125, sha256 0d1d7c23dcce8319b99caf64e47a4c211ffb1a4c4e48f1ee52732ed062504a19", + "hmtx": "80 bytes, checksum 0x3A6004F8, sha256 ef01a34d193ab3a15f92faf6b9d25a7e6087ee413a8f39f00a51a03f0aaf4740", + "loca": "42 bytes, checksum 0x15541330, sha256 e6a5d7d0c6e2a38ee931a83bfbe47a94f227ebc9cccfce0465b989a57a24a140", + "maxp": "32 bytes, checksum 0x001D0051, sha256 09189d869a40d53d0f84b16e4213dea19289b92a604cd9b191d68d0f59555662", + "name": "870 bytes, checksum 0x4B4466E3, sha256 f834e67e927dcd7c64b617ee0744d536a924258c896e6c42bbbfbdfa326c9f1a", + "post": "32 bytes, checksum 0xFF860032, sha256 22dd499f08932214bde96943f3882f3619324478217095c049b68a57c6c10693" + }, + "maxUniChar": 65535, + "defaultWidth": 1000 + }, + "repackageTTF": { + "bytes": 3704, + "sha256": "d0d334dc878b79acd528818b481b27c59f74c3a5a990dea15da6f9f9453e5868", + "tables": { + "OS/2": "96 bytes, checksum 0x5E34F64F, sha256 1f7c612aa4c82cb015efd76dbb01962f913e0d7e903b774dcee570de35b6f300", + "cmap": "156 bytes, checksum 0xB5590617, sha256 b74b2a37d121070dcec7e2c68f798b0e12498e84c4cd3fec2fc9fd550f5e8b54", + "gasp": "8 bytes, checksum 0x00000010, sha256 4ca731f86ad506ac0e320283dc9461926346de3c4d91d539ea7f6620d3826940", + "glyf": "2102 bytes, checksum 0xE5A06A37, sha256 af7e6396b3ee9523f1cc490c980e2eb02c48bd9d5cbe2c03fd671ead3adadf92", + "head": "54 bytes, checksum 0x3102FEA8, sha256 da6a0f4955cc0a3de4d86ac3c30f008d07fe37fae67e4a364217b9aefac8ba69", + "hhea": "36 bytes, checksum 0x09760125, sha256 0d1d7c23dcce8319b99caf64e47a4c211ffb1a4c4e48f1ee52732ed062504a19", + "hmtx": "80 bytes, checksum 0x3A6004F8, sha256 89d2ba3f47d9a102512952c5c94174c3499564921236a5550262d13616790d9a", + "loca": "42 bytes, checksum 0x13C211A4, sha256 e753e2efce78654bad50cb173dd4ebdd42c3bf0c74302b1b79d46adb2a4ae064", + "maxp": "32 bytes, checksum 0x001D0051, sha256 09189d869a40d53d0f84b16e4213dea19289b92a604cd9b191d68d0f59555662", + "name": "870 bytes, checksum 0x4B4D66E3, sha256 1a8da8b6114f22a951965f10e8769199a8180dcbba8a350d6d0061cfa1f2381d", + "post": "32 bytes, checksum 0xFF860032, sha256 22dd499f08932214bde96943f3882f3619324478217095c049b68a57c6c10693" + }, + "maxUni": 0 + } + }, + "useOTL=0xFF": { + "characters": 20, + "makeSubset": { + "bytes": 3748, + "sha256": "9f18a7ddb377e51ff54f312b7c8bc2cce478ba64dc5b6d3002fdd3146ec3bd3e", + "tables": { + "OS/2": "96 bytes, checksum 0x5E34F64F, sha256 1f7c612aa4c82cb015efd76dbb01962f913e0d7e903b774dcee570de35b6f300", + "cmap": "180 bytes, checksum 0x652E569B, sha256 0288429daf7b8865e16b8bcdeddd51c1bff64412c404a5500a2fb415773add80", + "gasp": "8 bytes, checksum 0x00000010, sha256 4ca731f86ad506ac0e320283dc9461926346de3c4d91d539ea7f6620d3826940", + "glyf": "2124 bytes, checksum 0x3FFB0FD4, sha256 b1fbcae81219663740147f5ba8f45ec41827a76abb62a4cd0117f879b8fec7bc", + "head": "54 bytes, checksum 0x3102FEA8, sha256 1084ee2f2873d9d4c12eb4c6280133eba5aeb2c72800d7e73b4db0eba137135d", + "hhea": "36 bytes, checksum 0x09760125, sha256 0d1d7c23dcce8319b99caf64e47a4c211ffb1a4c4e48f1ee52732ed062504a19", + "hmtx": "80 bytes, checksum 0x3A6004F8, sha256 89d2ba3f47d9a102512952c5c94174c3499564921236a5550262d13616790d9a", + "loca": "42 bytes, checksum 0x13FC11D8, sha256 96852a75ef627da304ababd4cefec56a809aafeb8a1e1317221978dfd674c2a2", + "maxp": "32 bytes, checksum 0x001D0051, sha256 09189d869a40d53d0f84b16e4213dea19289b92a604cd9b191d68d0f59555662", + "name": "870 bytes, checksum 0x4B4D66E3, sha256 1a8da8b6114f22a951965f10e8769199a8180dcbba8a350d6d0061cfa1f2381d", + "post": "32 bytes, checksum 0xFF860032, sha256 22dd499f08932214bde96943f3882f3619324478217095c049b68a57c6c10693" + }, + "maxUni": 65535, + "defaultWidth": 1000, + "codeToGlyph": "20 entries, sha256 ec8a2f22af881109af82f32d84c6a9a60073f2010b7cb5c3fa316d1f3d62ed8f" + }, + "makeSubsetSIP": { + "bytes": 3680, + "sha256": "d8d5a303ef12a328fa3041e531fe646c3dcd5e5e649ed2fafa57a5177a64f0c9", + "tables": { + "OS/2": "96 bytes, checksum 0x5D08CEBB, sha256 40ec5f44c8d585d5468d3663ed6a140206deda3b435edc25b3e81da65d2b70b6", + "cmap": "110 bytes, checksum 0x00A40125, sha256 cd026da39ad03d826ec61164e29c1977344d4b17ae03617fe33822a7434ab827", + "gasp": "8 bytes, checksum 0x00000010, sha256 4ca731f86ad506ac0e320283dc9461926346de3c4d91d539ea7f6620d3826940", + "glyf": "2124 bytes, checksum 0x3FFB0FD4, sha256 8e296933650350db28d99a62b6898cacd5dda4ca47e947f2c7b7d69fd94efa5c", + "head": "54 bytes, checksum 0x3102FEA8, sha256 cbe94e45b3b3ddbd77d019fb0b37fe976156f63737f295ecf319897fbc490951", + "hhea": "36 bytes, checksum 0x09760125, sha256 0d1d7c23dcce8319b99caf64e47a4c211ffb1a4c4e48f1ee52732ed062504a19", + "hmtx": "80 bytes, checksum 0x3A6004F8, sha256 ef01a34d193ab3a15f92faf6b9d25a7e6087ee413a8f39f00a51a03f0aaf4740", + "loca": "42 bytes, checksum 0x15541330, sha256 e6a5d7d0c6e2a38ee931a83bfbe47a94f227ebc9cccfce0465b989a57a24a140", + "maxp": "32 bytes, checksum 0x001D0051, sha256 09189d869a40d53d0f84b16e4213dea19289b92a604cd9b191d68d0f59555662", + "name": "870 bytes, checksum 0x4B4466E3, sha256 f834e67e927dcd7c64b617ee0744d536a924258c896e6c42bbbfbdfa326c9f1a", + "post": "32 bytes, checksum 0xFF860032, sha256 22dd499f08932214bde96943f3882f3619324478217095c049b68a57c6c10693" + }, + "maxUniChar": 65535, + "defaultWidth": 1000 + }, + "repackageTTF": { + "bytes": 3720, + "sha256": "ec95efaf70c3cb221767f5e50c95d29f3154576c6157edc7373841c29fd07eeb", + "tables": { + "OS/2": "96 bytes, checksum 0x5E34F64F, sha256 1f7c612aa4c82cb015efd76dbb01962f913e0d7e903b774dcee570de35b6f300", + "cmap": "172 bytes, checksum 0xB5590664, sha256 2c2ba8e4962d05f96ef55d98e31266d825e8fa7370c84c0c1daa56c2306014f1", + "gasp": "8 bytes, checksum 0x00000010, sha256 4ca731f86ad506ac0e320283dc9461926346de3c4d91d539ea7f6620d3826940", + "glyf": "2102 bytes, checksum 0xE5A06A37, sha256 af7e6396b3ee9523f1cc490c980e2eb02c48bd9d5cbe2c03fd671ead3adadf92", + "head": "54 bytes, checksum 0x3102FEA8, sha256 0cae13c407f33113b2e207f861975027b0aeae26206dbf1378e250f5a6b1ffb9", + "hhea": "36 bytes, checksum 0x09760125, sha256 0d1d7c23dcce8319b99caf64e47a4c211ffb1a4c4e48f1ee52732ed062504a19", + "hmtx": "80 bytes, checksum 0x3A6004F8, sha256 89d2ba3f47d9a102512952c5c94174c3499564921236a5550262d13616790d9a", + "loca": "42 bytes, checksum 0x13C211A4, sha256 e753e2efce78654bad50cb173dd4ebdd42c3bf0c74302b1b79d46adb2a4ae064", + "maxp": "32 bytes, checksum 0x001D0051, sha256 09189d869a40d53d0f84b16e4213dea19289b92a604cd9b191d68d0f59555662", + "name": "870 bytes, checksum 0x4B4D66E3, sha256 1a8da8b6114f22a951965f10e8769199a8180dcbba8a350d6d0061cfa1f2381d", + "post": "32 bytes, checksum 0xFF860032, sha256 22dd499f08932214bde96943f3882f3619324478217095c049b68a57c6c10693" + }, + "maxUni": 0 + } + } +} diff --git a/tests/data/ttf/Noto-LanguageTags-Synthetic.ttf b/tests/data/ttf/Noto-LanguageTags-Synthetic.ttf new file mode 100644 index 0000000000000000000000000000000000000000..2db14d2a7a4b979e6286c7dd4e4363253331066f GIT binary patch literal 4384 zcma)A32dCj6@D}S9$p`7uaDiB^|ANb&U)>=ec$W#ZO7|4ICh+De1`Z)Y~$D=je?R! z2!$4@X(8c8g+{%ANR>iWA(~2Ug{lTsm0H>&6)1<0s-glBsPS&U`FCRnTB`o*c{B6o z-ShtU&zKkyWziXuXeiPV?xbAGC(=F&tUeOm+g9IVievsF5nmdNR(U7=HwTH>-B>>~ zwHQ10lQES6YiXE2J{Ma(hCT=M52H_;TiBd^_vw2=SbrVU;QUN%+QZ{lu=W?U;5-nT zb!LY==p@_x;@U>tA;o8yU&DN8VRNM(qX(2_|C68mSu z55@t;=m;&+FMDJbXZ9g4(TiDwdzB_2(D zC-JRY`#$HNvyXEAv2->4Du3zB%#5rPLjLEon|9MYJxI^fpIIKOW9Qi)*xy)!=kbU5 z%lt#BTY6Y}Lt%srB~2$Dkj`O8CG4mM-eeQ))0uUQnHke1!L($GA7f2h=h!6RG(Q!; z9)FfKZAc&7+Qp06rFc_(PV(M*j$LBcZ~c~CLT1z_K4$OmO5|S|3__wy3i=O>&GEIwQ{U z1$RxBE$kUPm}hNv+6$U<%=((Cj`-D9le@=eeI(r;uvIw`9qc~WN=4$mT-Ess)WIN> zL6^=iE7wg-C>*0TqeJ7KU`1P*RP?7~j=|YQzN5fE1l% z5e^7J`KpNtx6y1i8Z8!Hy7d0D)`thgy)i8yVY?@Eg1Qg`^&$bqwA zbIHML%&1EG+wz*k_^RM zO2~zjNVpQPiWCWkLT(Akqrw_fG)@NE7Mm@tfo#dJKf_RvRcg@Wr)6bWc2(Can(nr> zx+G0zsm+<^>~ux4n(rH%Jlk2+?7u&-zB*$!7Aq8;N==}4slT${J?E-5WqJ;E9h`?< zV~HBxDV0+Zp$hNQrHn#M)deNUP%Rgs)$0wZ*rt*hQ5GZErq_>o4)+BP)YbZe0~OYW z{&L?;1((?O%37R#{n{d}wk~W6yILZQ@v`w>^S}45w^!_KZJ0DE(`v?QLwoB>Y*pcN zVU}i(cv|;Kib!TKVry=mjUE2^l*fX7aIpi@tLz0)r=#WqvrD>oYk|EmjCpI~bLn~h zK1OQFNUj<{A)(#JKiSG`;j>$pPf9bl9^>!#{i`?rQJ={4TFRGR;xEfwvP41u5WlhI z8Id+_J-}by6Q>bW?ZLzg(j}=Fva`uT4hm2+brZgAu)-QZ%QGV-&?|j9gbS~=!{PMm zy?6xz3?iLAk(p{Y$IBluH1Uu@%hjP!2{!vFhv#0*CAMRBQKK<<`o?m}0#n+f8AlFP36J-%a5P2n6NS|B? zh2UTy;6wRhA4*(Vyxh_~7qr`t>~iQ$ZN9dV^8G*kr^;633N)I<3uZLoQ+ z%PDG&A8}uV_0`F^CqEY=PsCeRf^WxmVcYqZ;N4{5w&Mdi?!D37oyW_H4Rw*q${|aB zsKe27cbRXz(bZR%*O}LMw0ZnocX|I}devxssUx?-+EHN6E!35iD~SlA<{GhTl~@{fOL8afyXEiQ)A zG`ei9vPYpw&rqgk78`4OyoX-k0&G*wyH<7qY9din;O#i4+& z&|9P4ohB)z`uVo%&bbD)GPBBJ@#%EC(qIuzAJQiK5mn+VgL))PFw1)fi$sYOMJkZA zOKiG9lrEL^qv(O`s+_VGFT3Hhs}z|buO$+(c0{xpj`}iZf4`$D;!fvEY0tFF638vg ztnwSYIRP7&l!`8{;1VxxtI4$4v&To0M-L6HplUFM2GhsUl4FMI@OWy>NlUM!#u5eT zH>t4#XF8r5E9oBQPmNV~&}1G%eF% zn!&gXC!aO^j!_*|;des*xp6AHP2+~7Wm?2qIk~`b0_U|gaG7Wr5?0YxXdQ4Gm`+eG zVbz2)K@2=XgOj4b-V3TJ%$?-M+A?UCvDOEwC7Q!(3~&auQ}nB{WPx44d}W)y@+;{p zp<`3lJP#>ruyAUd>#MaSJ=oDR3f>iHKO%b}V&_K8s^#c)VO9^Xh26SB{OFG;Z=dou%# zBGw|x_hLMa@e0ObCj@O2-C@T3gS$nd-jFF>S-fq+VW2TZL!0 Date: Thu, 17 Sep 2026 10:11:12 +1000 Subject: [PATCH 2/2] Cut a language tag at its first singleton in one step, and ask chinese() whether a language is Chinese (#206) The singleton cut becomes one preg_replace, the language comes off the front of the subtags so one array serves every rule, chinese() owns the test for a Chinese tag, and the extended language is looked up once. An empty tag returns before any of it, as it did before #206. Every expectation still matches hb_ot_tags_from_script_and_language(), the 87,094-tag comparison with HarfBuzz's own table differs only where it did, and each mechanism's mutation fails the same tests. Co-Authored-By: Claude Opus 5 (1M context) --- src/Shaper/OtlTags.php | 61 +++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/src/Shaper/OtlTags.php b/src/Shaper/OtlTags.php index 84ef52547..428d19b06 100644 --- a/src/Shaper/OtlTags.php +++ b/src/Shaper/OtlTags.php @@ -159,9 +159,8 @@ public static function language($ietf, $available) * Only the subtags before the first singleton count, so an extension or private use subtag * selects nothing. A script is the subtag after the language; a region may be anywhere. * - * An extended language Ucdn::$ot_languages has no key for keeps the language subtag's tags, so - * ar-afb stays ARA. HarfBuzz's far larger table gives most extended languages their - * macrolanguage's tag, and takes the rest as their own ISO 639-3 code. + * An extended language Ucdn::$ot_languages has no key for keeps the language subtag's tags, where + * HarfBuzz would take an unlisted one as its own ISO 639-3 code. * * @param string $ietf * @@ -170,21 +169,18 @@ public static function language($ietf, $available) private static function languageSystems($ietf) { $tag = strtolower((string) $ietf); - $subtags = explode('-', $tag); - - foreach ($subtags as $i => $subtag) { - if ($i > 0 && strlen($subtag) == 1 && isset($subtags[$i + 1])) { - $subtags = array_slice($subtags, 0, $i); - break; - } + if ($tag == '') { + return []; } - if ($subtags[0] == 'x') { + + $subtags = explode('-', preg_replace('/-[^-]-.*/s', '', $tag)); + $language = array_shift($subtags); + if ($language == 'x') { return []; } - $following = array_slice($subtags, 1); foreach (self::$variants as $variant => $langsys) { - if (in_array($variant, $following, true)) { + if (in_array($variant, $subtags, true)) { return $langsys; } } @@ -193,12 +189,11 @@ private static function languageSystems($ietf) return self::$retired[$tag]; } - $language = $subtags[0]; - $script = isset($subtags[1]) ? $subtags[1] : ''; + $script = isset($subtags[0]) ? $subtags[0] : ''; if (isset(self::$languageSubtags[$language])) { foreach (self::$languageSubtags[$language] as $subtag => $langsys) { - if (strlen($subtag) == 4 ? $script == $subtag : in_array($subtag, $following, true)) { + if (strlen($subtag) == 4 ? $script == $subtag : in_array($subtag, $subtags, true)) { return $langsys; } } @@ -206,18 +201,15 @@ private static function languageSystems($ietf) $own = self::tags($language); - if ($own && in_array($own[0], ['ZHS ', 'ZHT ', 'ZHH '], true)) { - $chinese = self::chinese($own[0], $script, $following); - if ($chinese) { - return $chinese; - } + $chinese = self::chinese($own, $script, $subtags); + if ($chinese) { + return $chinese; } - if (preg_match('/^[a-z]{3}$/', $script) && self::tags($script)) { - return self::tags($script); - } + // A three-digit region such as 419 has no key, so three characters that find one are a language + $extended = strlen($script) == 3 ? self::tags($script) : []; - return $own; + return $extended ?: $own; } /** @@ -230,28 +222,31 @@ private static function languageSystems($ietf) * Only Hans moves Cantonese (yue, ZHH) and Literary Chinese (lzh, ZHT) off their own tags. Every * other Chinese language is Simplified by default and takes zh's rules. * - * @param string $own The language's own tag - * @param string $script The subtag after the language - * @param string[] $following The subtags after the language + * @param string[] $own The language's own tags + * @param string $script The subtag after the language + * @param string[] $subtags The subtags after the language * - * @return string[] The language systems, or none where the language's own tag stands + * @return string[] The language systems, or none */ - private static function chinese($own, $script, array $following) + private static function chinese(array $own, $script, array $subtags) { + if (!$own || !in_array($own[0], ['ZHS ', 'ZHT ', 'ZHH '], true)) { + return []; + } if ($script == 'hans') { return ['ZHS ']; } - if ($own != 'ZHS ') { + if ($own[0] != 'ZHS ') { return []; } if ($script == 'hant') { - $region = isset($following[1]) ? $following[1] : ''; + $region = isset($subtags[1]) ? $subtags[1] : ''; return $region == 'hk' || $region == 'mo' ? self::$chineseRegions[$region] : ['ZHT ']; } foreach (self::$chineseRegions as $region => $langsys) { - if (in_array($region, $following, true)) { + if (in_array($region, $subtags, true)) { return $langsys; } }