Ko tn staging v1

Jenkinsfile

@@ -28,6 +28,7 @@ pipeline {
     MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
     JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-17-24-1'
     HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-22-25-0'
+    KO_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-03-25-0'
     DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {
@@ -318,6 +319,22 @@ pipeline {
         }
       }
     }
+    stage('L0: Create KO TN Grammars') {
+      when {
+        anyOf {
+          branch 'main'
+          changeRequest target: 'main'
+        }
+      }   
+      failFast true
+      parallel {
+        stage('L0: KO TN grammars') {
+          steps {
+            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=ko --text="100" --cache_dir ${KO_TN_CACHE}'
+          }
+        }
+      }
+    }
 
 
 // L1 Tests starts here
@@ -406,6 +423,11 @@ pipeline {
             sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hy/ -m "not pleasefixme" --cpu --tn_cache_dir ${HY_TN_CACHE}'
           }
         }
+        stage('L1: Run all KO TN/ITN tests (restore grammars from cache)') {
+          steps {
+            sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ko/ -m "not pleasefixme" --cpu --tn_cache_dir ${KO_TN_CACHE}'
+          }
+        }
       }
     }
 

nemo_text_processing/text_normalization/ko/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nemo_text_processing.text_normalization.en.taggers.tokenize_and_classify import ClassifyFst
+from nemo_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst
+from nemo_text_processing.text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst

nemo_text_processing/text_normalization/ko/data/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/ko/data/date/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/ko/data/date/exceptions.tsv

@@ -0,0 +1,2 @@
+6	유
+10	시

nemo_text_processing/text_normalization/ko/data/date/week.tsv

@@ -0,0 +1,8 @@
+월	월요일
+화	화요일
+수	수요일
+목	목요일
+금	금요일
+토	토요일
+일	일요일
+공	공휴일

nemo_text_processing/text_normalization/ko/data/electronic/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/ko/data/electronic/cc_cues.tsv

@@ -0,0 +1,11 @@
+카드 끝자리	카드 끝자리
+카드 마지막 네자리	카드 마지막 네자리
+카드 마지막 4자리	카드 마지막 네자리
+신용카드 번호	신용카드 번호
+신용카드	신용카드
+체크카드 번호	체크카드 번호
+체크카드	체크카드
+카드번호	카드번호
+결제 카드	결제 카드
+결제카드	결제카드
+카드	카드

nemo_text_processing/text_normalization/ko/data/electronic/domain.tsv

@@ -0,0 +1,28 @@
+.com	닷컴
+.org	닷 오알지
+.gov	닷 거브
+.edu	닷 에듀
+.net	닷 넷
+.ai	닷 에이아이
+.io	닷 아이오
+.dev	닷 데브
+.app	닷 앱
+.cloud	닷 클라우드
+.shop	닷 샵
+.store	닷 스토어
+.co	닷 씨오
+.me	닷 미
+.kr	닷 케이알
+.co.kr	닷 씨오 닷 케이알
+.ac.kr	닷 에이씨 닷 케이알
+.or.kr	닷 오알 닷 케이알
+.go.kr	닷 지오 닷 케이알
+.re.kr	닷 알이 닷 케이알
+.cn	닷 씨엔
+.fr	닷 에프알
+.de	닷 디이
+.it	닷 아이티
+.uk	닷 유케이
+.br	닷 비알
+.in	닷 아이엔
+.ru	닷 알유

nemo_text_processing/text_normalization/ko/data/electronic/extensions.tsv

@@ -0,0 +1,6 @@
+.jpg	닷 제이피지
+.png	닷 피엔지
+.pdf	닷 피디에프
+.JPG	닷 제이피지
+.PNG	닷 피엔지
+.PDF	닷 피디에프

nemo_text_processing/text_normalization/ko/data/electronic/symbol.tsv

@@ -0,0 +1,12 @@
+@	골뱅이
+.	점
+/	슬래시
+-	대시
+_	언더바
+:	콜론
+?	물음표
+=	이퀄
+&	앰퍼샌드
+%	퍼센트
++	플러스
+#	샵

nemo_text_processing/text_normalization/ko/data/measure/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/ko/data/measure/unit.tsv

@@ -0,0 +1,18 @@
+kg	킬로그램
+g	그램
+km	킬로미터
+m	미터
+cm	센티미터
+mm	밀리미터
+L	리터
+l	리터
+mL	밀리리터
+ml	밀리리터
+h	시간
+s	초
+N	뉴턴
+W	와트
+Hz	헤르츠
+°	도
+%	퍼센트
+rpm	분당회전수