Hi TN: money class edge case bug fix

Jenkinsfile

@@ -29,7 +29,7 @@ pipeline {
     MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
     JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
     KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6'
-    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-0'
+    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-27-26-0'
     DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {

nemo_text_processing/text_normalization/hi/data/money/quantities.tsv

@@ -0,0 +1,10 @@
+हजार
+हज़ार
+लाख
+करोड़
+करोड
+अरब
+खरब
+मिलियन
+बिलियन
+ट्रिलियन

nemo_text_processing/text_normalization/hi/taggers/money.py

@@ -92,6 +92,40 @@ def __init__(self, cardinal: GraphFst):
         )
         guarded_decimal_digits = has_3plus_sig_digits @ decimal_digits
 
+        # Load scale words dynamically from TSV
+        quantity_graph = (
+            pynutil.insert('quantity: "')
+            + pynini.string_file(get_abs_path("data/money/quantities.tsv")).optimize()
+            + pynutil.insert('"')
+        )
+
+        # Allow an optional space between the number and the quantity word
+        delete_space_opt = pynini.closure(pynutil.delete(" "), 0, 1)
+
+        # Path for scaled money with decimals (e.g. ₹२.५ करोड़)
+        graph_scaled_decimal = (
+            optional_graph_negative
+            + currency_major
+            + insert_space
+            + integer
+            + pynini.cross(".", " ")
+            + decimal_digits
+            + delete_space_opt
+            + insert_space
+            + quantity_graph
+        ).optimize()
+
+        # Path for scaled money without decimals (e.g. ₹५ करोड़)
+        graph_scaled_major = (
+            optional_graph_negative
+            + currency_major
+            + insert_space
+            + integer
+            + delete_space_opt
+            + insert_space
+            + quantity_graph
+        ).optimize()
+
         graph_decimal_path = (
             optional_graph_negative
             + currency_major
@@ -150,6 +184,8 @@ def _build_major_and_minor(sym_maj_labels, int_graph):
         graph_currencies = (
             pynutil.add_weight(graph_major_only_singular | graph_major_and_minor_singular, -0.001)
             | pynutil.add_weight(graph_decimal_path, -0.0005)
+            | pynutil.add_weight(graph_scaled_decimal, -0.0002)
+            | pynutil.add_weight(graph_scaled_major, -0.0002)
             | graph_major_only
             | graph_major_and_minor
         )

nemo_text_processing/text_normalization/hi/verbalizers/money.py

@@ -46,6 +46,12 @@ def __init__(self):
 
         currency_minor = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')
 
+        quantity = pynutil.delete('quantity: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')
+
+        drop_preserve_order = pynini.closure(
+            pynutil.delete("preserve_order:") + sp + pynutil.delete("true") + sp, 0, 1
+        )
+
         graph_major_only = integer_part + sp + currency_major
 
         all_major_names = [maj for maj, _ in load_labels(get_abs_path("data/money/major_minor_currencies.tsv"))]
@@ -97,6 +103,47 @@ def __init__(self):
             )
         graph_decimal_money = pynini.union(*decimal_graphs)
 
-        graph = graph_major_only | graph_major_minor | pynutil.add_weight(graph_minor_only, -0.1) | graph_decimal_money
+        scaled_decimal_graphs = []
+        scaled_major_graphs = []
+
+        for major in all_major_names:
+            graph_major_slot = pynutil.delete('currency_maj: "') + pynutil.delete(major) + pynutil.delete('"')
+
+            scaled_decimal_graphs.append(
+                drop_preserve_order
+                + graph_major_slot
+                + sp
+                + integer_part
+                + sp
+                + pynutil.insert(" दशमलव ")
+                + fractional_part
+                + sp
+                + quantity
+                + pynutil.insert(NEMO_SPACE)
+                + pynutil.insert(major)
+            )
+
+            scaled_major_graphs.append(
+                drop_preserve_order
+                + graph_major_slot
+                + sp
+                + integer_part
+                + sp
+                + quantity
+                + pynutil.insert(NEMO_SPACE)
+                + pynutil.insert(major)
+            )
+
+        graph_scaled_decimal = pynini.union(*scaled_decimal_graphs)
+        graph_scaled_major = pynini.union(*scaled_major_graphs)
+
+        graph = (
+            graph_major_only
+            | graph_major_minor
+            | pynutil.add_weight(graph_minor_only, -0.1)
+            | graph_decimal_money
+            | graph_scaled_decimal
+            | graph_scaled_major
+        )
 
         self.fst = self.delete_tokens(graph).optimize()

tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt

@@ -141,4 +141,9 @@ $1.2000~एक डॉलर बीस सेंट
 ₹५,५६,३२०~पाँच लाख छप्पन हज़ार तीन सौ बीस रुपए
 ₹१,००,०००~एक लाख रुपए
 ₹२,१४८~दो हज़ार एक सौ अड़तालीस रुपए
-₹९९,९९९~निन्यानबे हज़ार नौ सौ निन्यानबे रुपए
+₹९९,९९९~निन्यानबे हज़ार नौ सौ निन्यानबे रुपए
+₹२.५ करोड़~दो दशमलव पाँच करोड़ रुपए
+₹५ करोड़~पाँच करोड़ रुपए
+$१.५ मिलियन~एक दशमलव पाँच मिलियन डॉलर
+£१२ लाख~बारह लाख पाउंड
+₹१.५ हजार~एक दशमलव पाँच हजार रुपए