Proof of concept for using langid

README.md

@@ -1 +1,26 @@
-# language_annotation
+## Proof of Concept
+
+This repository is a proof of concept to detect 
+sparse instances of a particular language within
+a largely monolingual text. 
+
+## Steps to Run
+```
+git clone "<REPO_URL>" 
+python -m venv .venv && source .venv/bin/activate # Optional
+pip install .
+python use_lang.py
+```
+
+## Purpose
+
+One of the projects that we are working on, is 
+OCR transcription of Handwritten German Shorthand text into Latin
+german script, such that historical texts that were
+written in German shorthand become accessible.
+
+One of the issues with this task is that many shorthand writers don't exclusively write in German
+Shorthand, but will often use English for proper nouns, and other loan words. 
+
+This leads to the task being multilingual character recognition. Therefore one way to improve performance of our OCR model would be to annotate the places where
+Latin Script (for english) or Arabic Script (for numerals) is being used (this is a hypothesis). 

pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+authors = [
+    {name = "Ojas Mishra", email = "ojm15@pitt.edu"}
+]
+readme = "README.md"
+name = "detect_sparse_lang"
+version = "0.0.1"
+dependencies = [
+    "lxml",
+    "langid",
+]

use_lang.py

@@ -0,0 +1,25 @@
+import langid
+import lxml.etree
+import lxml.html
+from itertools import islice
+from collections import deque
+def sliding_window(iterable, n):
+    "Collect data into overlapping fixed-length chunks or blocks."
+    # sliding_window('ABCDEFG', 3) → ABC BCD CDE DEF EFG
+    iterator = iter(iterable)
+    window = deque(islice(iterator, n - 1), maxlen=n)
+    for x in iterator:
+        window.append(x)
+        yield tuple(window)
+text = "Hallo! Mein name ist Ojas. Heute habe ich der Buch. \"The Three Musketeers\" liest"
+
+window_size = 5
+skip_count = 3
+#iterable = text.split()
+tree = lxml.html.parse("wiki.html").getroot()
+text = "".join(tree.itertext()).split()
+
+for window in list(sliding_window(text, window_size))[::skip_count]:
+    result = langid.classify(" ".join(window))
+    if result[0] != 'de':
+        print(window, langid.classify(" ".join(window)))