Lazy *full.kbb keys found from any pass; << writes to absolute paths (4.2.39)

.github/workflows/build-linux.yml

@@ -318,6 +318,46 @@ jobs:
           fi
           echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr"
 
+      - name: Lazy kbb lookups from any pass, and << to a path (Non-20.04)
+        if: matrix.id != 'docker-ubuntu-20.04'
+        shell: bash
+        run: |
+          set -eo pipefail
+          # kbb-lazy-lookup: a key of a lazy *full.kbb used to be found only if
+          # the tokenizer or dictfindword() had loaded it, and down() listed
+          # only those. findconcept/getconcept now search the file on a miss,
+          # down() loads it whole, and the tokenizer searches only dictionary
+          # files (top concept "dictionary", or a .dict twin) -- not domain
+          # KBs, which made spelling candidates 100x slower.
+          FIX=.github/workflows/tests/kbb-lazy-lookup
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
+          ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> /tmp/lazy.err
+          got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt")
+          want=$(printf 'domain not a word: 1\ntwin linked: 1\nkilo: 1\nnested: 1\nquoted key: 1\nmiss: 1\nmiss again: 1\ngetconcept: 1\nchildren: 17')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(kbb-lazy-lookup): out.txt was:"; echo "$got"; cat /tmp/lazy.err; exit 1
+          fi
+          # output-path: "path" << x silently wrote nothing for an absolute
+          # path (it was appended to the output folder). $apppath follows -ANA,
+          # so give it an absolute one.
+          FIX=.github/workflows/tests/output-path
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
+          ABS="$(pwd -W 2>/dev/null || pwd)/$FIX"
+          ./bin/nlp -ANA "$ABS" -WORK ./ "$ABS/input/text.txt" 2> /tmp/opath.err
+          check() {
+            got=$(tr -d '\r' < "$1" 2>/dev/null || echo "(missing)")
+            [ "$got" = "$2" ] || { echo "FAIL(output-path): $1 was '$got'"; cat /tmp/opath.err; exit 1; }
+          }
+          check "$FIX/output/abs.txt" "$(printf 'one\ntwo')"
+          check "$FIX/output/expr.txt" expr
+          check "$FIX/input/text.txt_log/rel.txt" rel
+          check "$FIX/input/text.txt_log/out.txt" done
+          n=$(grep -c "can't open output file" /tmp/opath.err || true)
+          if [ "$n" != 1 ]; then
+            echo "FAIL(output-path): bad path reported $n times on stderr, want 1"; cat /tmp/opath.err; exit 1
+          fi
+          echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path"
+
       - name: Run-mode globals ($dev / $silent, Non-20.04)
         if: matrix.id != 'docker-ubuntu-20.04'
         shell: bash

.github/workflows/build-macos.yml

@@ -167,6 +167,45 @@ jobs:
           fi
           echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr"
 
+      - name: Lazy kbb lookups from any pass, and << to a path
+        shell: bash
+        run: |
+          set -eo pipefail
+          # kbb-lazy-lookup: a key of a lazy *full.kbb used to be found only if
+          # the tokenizer or dictfindword() had loaded it, and down() listed
+          # only those. findconcept/getconcept now search the file on a miss,
+          # down() loads it whole, and the tokenizer searches only dictionary
+          # files (top concept "dictionary", or a .dict twin) -- not domain
+          # KBs, which made spelling candidates 100x slower.
+          FIX=.github/workflows/tests/kbb-lazy-lookup
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
+          ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> /tmp/lazy.err
+          got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt")
+          want=$(printf 'domain not a word: 1\ntwin linked: 1\nkilo: 1\nnested: 1\nquoted key: 1\nmiss: 1\nmiss again: 1\ngetconcept: 1\nchildren: 17')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(kbb-lazy-lookup): out.txt was:"; echo "$got"; cat /tmp/lazy.err; exit 1
+          fi
+          # output-path: "path" << x silently wrote nothing for an absolute
+          # path (it was appended to the output folder). $apppath follows -ANA,
+          # so give it an absolute one.
+          FIX=.github/workflows/tests/output-path
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
+          ABS="$(pwd -W 2>/dev/null || pwd)/$FIX"
+          ./bin/nlp -ANA "$ABS" -WORK ./ "$ABS/input/text.txt" 2> /tmp/opath.err
+          check() {
+            got=$(tr -d '\r' < "$1" 2>/dev/null || echo "(missing)")
+            [ "$got" = "$2" ] || { echo "FAIL(output-path): $1 was '$got'"; cat /tmp/opath.err; exit 1; }
+          }
+          check "$FIX/output/abs.txt" "$(printf 'one\ntwo')"
+          check "$FIX/output/expr.txt" expr
+          check "$FIX/input/text.txt_log/rel.txt" rel
+          check "$FIX/input/text.txt_log/out.txt" done
+          n=$(grep -c "can't open output file" /tmp/opath.err || true)
+          if [ "$n" != 1 ]; then
+            echo "FAIL(output-path): bad path reported $n times on stderr, want 1"; cat /tmp/opath.err; exit 1
+          fi
+          echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path"
+
       # INTEL MACS (#231). The steps above built and tested arm64 (Apple
       # Silicon). Build the same tree for x86_64 too and merge the two with
       # lipo, so nlpm.exe and the compile libraries are universal binaries:

.github/workflows/build-windows.yml

@@ -255,6 +255,45 @@ jobs:
           fi
           echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr"
 
+      - name: Lazy kbb lookups from any pass, and << to a path
+        shell: bash
+        run: |
+          set -eo pipefail
+          # kbb-lazy-lookup: a key of a lazy *full.kbb used to be found only if
+          # the tokenizer or dictfindword() had loaded it, and down() listed
+          # only those. findconcept/getconcept now search the file on a miss,
+          # down() loads it whole, and the tokenizer searches only dictionary
+          # files (top concept "dictionary", or a .dict twin) -- not domain
+          # KBs, which made spelling candidates 100x slower.
+          FIX=.github/workflows/tests/kbb-lazy-lookup
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
+          ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> /tmp/lazy.err
+          got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt")
+          want=$(printf 'domain not a word: 1\ntwin linked: 1\nkilo: 1\nnested: 1\nquoted key: 1\nmiss: 1\nmiss again: 1\ngetconcept: 1\nchildren: 17')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(kbb-lazy-lookup): out.txt was:"; echo "$got"; cat /tmp/lazy.err; exit 1
+          fi
+          # output-path: "path" << x silently wrote nothing for an absolute
+          # path (it was appended to the output folder). $apppath follows -ANA,
+          # so give it an absolute one.
+          FIX=.github/workflows/tests/output-path
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
+          ABS="$(pwd -W 2>/dev/null || pwd)/$FIX"
+          ./bin/Release/nlp.exe -ANA "$ABS" -WORK ./ "$ABS/input/text.txt" 2> /tmp/opath.err
+          check() {
+            got=$(tr -d '\r' < "$1" 2>/dev/null || echo "(missing)")
+            [ "$got" = "$2" ] || { echo "FAIL(output-path): $1 was '$got'"; cat /tmp/opath.err; exit 1; }
+          }
+          check "$FIX/output/abs.txt" "$(printf 'one\ntwo')"
+          check "$FIX/output/expr.txt" expr
+          check "$FIX/input/text.txt_log/rel.txt" rel
+          check "$FIX/input/text.txt_log/out.txt" done
+          n=$(grep -c "can't open output file" /tmp/opath.err || true)
+          if [ "$n" != 1 ]; then
+            echo "FAIL(output-path): bad path reported $n times on stderr, want 1"; cat /tmp/opath.err; exit 1
+          fi
+          echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path"
+
       - name: Run-mode globals ($dev / $silent)
         shell: bash
         run: |

.github/workflows/tests/kbb-lazy-lookup/input/text.txt

@@ -0,0 +1 @@
+The golf hotel.

.github/workflows/tests/kbb-lazy-lookup/kb/user/codes-full.kbb

@@ -0,0 +1,21 @@
+# A domain KB: lazily loaded, and searched only when a pass asks for a key.
+codes
+  "heart attack": code=I21
+  alpha: rank=1
+    A001: desc=first
+    A002: desc=second
+  bravo: rank=2
+  charlie: rank=3
+  delta: rank=4
+  echo: rank=5
+  foxtrot: rank=6
+  golf: rank=7
+  hotel: rank=8
+  india: rank=9
+  juliet: rank=10
+  kilo: rank=11
+  lima: rank=12
+  mike: rank=13
+  november: rank=14
+  oscar: rank=15
+  papa: rank=16

.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.dict

@@ -0,0 +1,2 @@
+hotel pos=noun
+zebra pos=noun

.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.kbb

@@ -0,0 +1,3 @@
+terms
+  hotel: sense=lodging
+  zebra: sense=animal

.github/workflows/tests/kbb-lazy-lookup/spec/analyzer.seq

@@ -0,0 +1,2 @@
+dicttokz	nil	# tokenize against the dictionary
+nlp	lookup	# a later pass looks keys up

.github/workflows/tests/kbb-lazy-lookup/spec/lookup.nlp

@@ -0,0 +1,33 @@
+###############################################
+# FILE:     lookup.nlp
+# SUBJ:     Look keys up in the lazy codes-full.kbb the ordinary way, after
+#           tokenizing. Each line is 1 when the engine behaves.
+###############################################
+
+@CODE
+L("codes") = findconcept(findroot(),"codes");
+# The tokenizer and dictfindword do not search a domain KB: "golf" is in the
+# input and in codes-full.kbb, but it is not a dictionary word.
+"out.txt" << "domain not a word: " << str(dictfindword("golf") == 0) << "\n";
+# A twin kbb is still searched by the tokenizer, linked to its dict word.
+"out.txt" << "twin linked: " << str(conval(dictfindword("hotel"),"meaning") != 0) << "\n";
+# findconcept loads a key on demand, with its whole block.
+L("kilo") = findconcept(L("codes"),"kilo");
+"out.txt" << "kilo: " << str(numval(L("kilo"),"rank") == 11) << "\n";
+L("a1") = findconcept(findconcept(L("codes"),"alpha"),"A001");
+"out.txt" << "nested: " << str(strval(L("a1"),"desc") == "first") << "\n";
+"out.txt" << "quoted key: " << str(strval(findconcept(L("codes"),"heart attack"),"code") == "I21") << "\n";
+"out.txt" << "miss: " << str(findconcept(L("codes"),"zulu") == 0) << "\n";
+"out.txt" << "miss again: " << str(findconcept(L("codes"),"zulu") == 0) << "\n";
+# getconcept finds the file's key instead of making an empty one.
+"out.txt" << "getconcept: " << str(numval(getconcept(L("codes"),"lima"),"rank") == 12) << "\n";
+# down() lists every key once: 17, with kilo, alpha, lima, heart attack
+# loaded above and not built twice.
+L("n") = 0;
+L("c") = down(L("codes"));
+while (L("c")) {
+	L("n")++;
+	L("c") = next(L("c"));
+}
+"out.txt" << "children: " << str(L("n")) << "\n";
+@@CODE

.github/workflows/tests/output-path/input/text.txt

@@ -0,0 +1 @@
+x

.github/workflows/tests/output-path/spec/analyzer.seq

@@ -0,0 +1,2 @@
+tokenize	nil	# tokenize
+nlp	write	# write to paths

.github/workflows/tests/output-path/spec/write.nlp

@@ -0,0 +1,18 @@
+###############################################
+# FILE:     write.nlp
+# SUBJ:     << to a path. An absolute path is used as given; a relative name
+#           goes in the input's _log folder; a path that can't be opened is
+#           reported once, to err.log and stderr.
+###############################################
+
+@CODE
+L("abs") = G("$apppath") + "/output/abs.txt";
+L("abs") << "one\n";
+L("abs") << "two\n";
+G("$apppath") + "/output/expr.txt" << "expr\n";
+"rel.txt" << "rel\n";
+L("bad") = G("$apppath") + "/no/such/dir/x.txt";
+L("bad") << "a\n";
+L("bad") << "b\n";
+"out.txt" << "done\n";
+@@CODE