From 2b40297e61699d25f2340b1e41883dd23e7d6604 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 20 Oct 2026 11:12:38 -0400 Subject: [PATCH 1/2] "path" << x writes to an absolute path The output operator appended every file name to the input's _log folder, so an absolute path, like G("$apppath") + "/out.txt", became <_log>/C:/.../out.txt. The open failed, nothing checked it, and every write was dropped with no message. jsonwrite's help already says an absolute path is used as given, and openfile() works that way. Var::filevar now uses a path that starts with "/", "\" or a drive letter as given. A relative name still goes in the _log folder. If the file can't be opened, it says so once, in err.log and on stderr, and later writes to that name do nothing. The path was also formatted into a MAXSTR buffer with no bound; it is a std::string now. Fixture output-path covers both kinds of path and a bad one. Co-Authored-By: Claude Opus 5.5 --- .../tests/output-path/input/text.txt | 1 + .../tests/output-path/spec/analyzer.seq | 2 ++ .../tests/output-path/spec/write.nlp | 18 ++++++++++++ lite/var.cpp | 29 ++++++++++++++----- 4 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/tests/output-path/input/text.txt create mode 100644 .github/workflows/tests/output-path/spec/analyzer.seq create mode 100644 .github/workflows/tests/output-path/spec/write.nlp diff --git a/.github/workflows/tests/output-path/input/text.txt b/.github/workflows/tests/output-path/input/text.txt new file mode 100644 index 00000000..587be6b4 --- /dev/null +++ b/.github/workflows/tests/output-path/input/text.txt @@ -0,0 +1 @@ +x diff --git a/.github/workflows/tests/output-path/spec/analyzer.seq b/.github/workflows/tests/output-path/spec/analyzer.seq new file mode 100644 index 00000000..35c9bf6f --- /dev/null +++ b/.github/workflows/tests/output-path/spec/analyzer.seq @@ -0,0 +1,2 @@ +tokenize nil # tokenize +nlp write # write to paths diff --git a/.github/workflows/tests/output-path/spec/write.nlp b/.github/workflows/tests/output-path/spec/write.nlp new file mode 100644 index 00000000..f2aff10a --- /dev/null +++ b/.github/workflows/tests/output-path/spec/write.nlp @@ -0,0 +1,18 @@ +############################################### +# FILE: write.nlp +# SUBJ: << to a path. An absolute path is used as given; a relative name +# goes in the input's _log folder; a path that can't be opened is +# reported once, to err.log and stderr. +############################################### + +@CODE +L("abs") = G("$apppath") + "/output/abs.txt"; +L("abs") << "one\n"; +L("abs") << "two\n"; +G("$apppath") + "/output/expr.txt" << "expr\n"; +"rel.txt" << "rel\n"; +L("bad") = G("$apppath") + "/no/such/dir/x.txt"; +L("bad") << "a\n"; +L("bad") << "b\n"; +"out.txt" << "done\n"; +@@CODE diff --git a/lite/var.cpp b/lite/var.cpp index e4c3f1b6..854b39ac 100644 --- a/lite/var.cpp +++ b/lite/var.cpp @@ -14,6 +14,7 @@ All rights reserved. *******************************************************************************/ #include "StdAfx.h" +#include #include "machine.h" // 03/09/00 AM. #include "u_out.h" // 01/19/06 AM. #include "lite/lite.h" // 07/07/03 AM. @@ -2453,17 +2454,31 @@ else parse->setVars(dlist); } -// Set up proper path for file. -// Should check if it's relative or absolute. -_TCHAR buf[MAXSTR]; +// A relative name goes in the output folder. An absolute path ("/x", +// "\x", "C:...") is used as given, as openfile() does: it used to be +// appended to the output folder too, so the open failed and every write +// was silently dropped. (The old MAXSTR buffer also overflowed on a long +// path.) +std::string path; +if (fname[0] == '/' || fname[0] == '\\' || (fname[0] && fname[1] == ':')) + path = fname; +else + path = std::string(parse->getOutdir()) + DIR_CH + fname; _TCHAR *str = 0; -//sprintf(buf, "%s\\output\\%s", parse->getAppdir(), fname); // 03/10/99 AM. -//sprintf_s(buf, "%s\\output\\%s", parse->getAppdir(), fname); // 03/10/99 AM. -_stprintf(buf, _T("%s%c%s"), parse->getOutdir(),DIR_CH, fname); // 03/10/99 AM. -parse->internStr(buf, /*UP*/ str); +parse->internStr(path.data(), /*UP*/ str); // Open the output file for APPENDING. ostr = new std::_t_ofstream(TCHAR2CA(str), std::ios::app); +if (!*ostr) + { + // Report it once. The var keeps the failed stream, so later writes to + // this name do nothing instead of repeating the error. + std::_t_strstream gerrStr; + gerrStr << _T("[Output op: can't open file ") << str << _T("]") << std::ends; + parse->errOut(&gerrStr,false); + std::_t_cerr << _T("[Error: can't open output file ") << str + << _T(". Nothing written to it.]") << std::endl; + } #ifdef UNICODE *ostr << BOM_UTF8; #endif From 2928213aeb6b3561ff72b2b3b381ac5e0e69e6bc Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 20 Oct 2026 11:12:41 -0400 Subject: [PATCH 2/2] Lazy *full.kbb keys are found from any pass (4.2.39) A *full.kbb stays on disk and is binary-searched one key at a time, but only the tokenizer and dictfindword() reached that search. A later pass calling findconcept() on the file's top concept found only keys the tokenizer had happened to load, and down()/next() listed only those. An ICD analyzer had to call dictfindword() before every lookup, and a scan of its head words silently missed codes. - findconcept() and getconcept() search the file on a miss under its top concept, build the key's whole block, and return it. Misses are remembered per file. A key with a space, which SaveKB writes quoted ("heart attack":), is found too. - down(), and findconcept() by position, load the whole file the first time, so every key is listed once. Keys already loaded are not built twice. - The tokenizer and dictfindword() search only dictionary files: a *full.kbb whose top concept is "dictionary", or that has a same-stem .dict twin (en-full.kbb with en-full.dict, linked as before). A domain KB such as an ICD index is no longer searched for every token and spelling candidate, which made a spell pass go from 0.1 s to 12 s, and its keys are no longer registered as dictionary words. Fixture kbb-lazy-lookup checks each of these. With the shipped en-full.kbb, en-full.dict and en-synonyms-full.kbb, the dicttokz parse tree of 15x doj.txt is byte-identical to 4.3.15's. The new CI step runs kbb-lazy-lookup and output-path. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-linux.yml | 40 +++++ .github/workflows/build-macos.yml | 39 +++++ .github/workflows/build-windows.yml | 39 +++++ .../tests/kbb-lazy-lookup/input/text.txt | 1 + .../kbb-lazy-lookup/kb/user/codes-full.kbb | 21 +++ .../kbb-lazy-lookup/kb/user/terms-full.dict | 2 + .../kbb-lazy-lookup/kb/user/terms-full.kbb | 3 + .../tests/kbb-lazy-lookup/spec/analyzer.seq | 2 + .../tests/kbb-lazy-lookup/spec/lookup.nlp | 33 ++++ cs/libconsh/cg.cpp | 152 ++++++++++++++++-- include/Api/consh/cg.h | 14 ++ lite/fn.cpp | 4 + lite/fnrun.cpp | 2 + nlp/main.cpp | 2 +- 14 files changed, 340 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/tests/kbb-lazy-lookup/input/text.txt create mode 100644 .github/workflows/tests/kbb-lazy-lookup/kb/user/codes-full.kbb create mode 100644 .github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.dict create mode 100644 .github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.kbb create mode 100644 .github/workflows/tests/kbb-lazy-lookup/spec/analyzer.seq create mode 100644 .github/workflows/tests/kbb-lazy-lookup/spec/lookup.nlp diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 2eb6af74..edcd71ea 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -318,6 +318,46 @@ jobs: fi echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr" + - name: Lazy kbb lookups from any pass, and << to a path (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # kbb-lazy-lookup: a key of a lazy *full.kbb used to be found only if + # the tokenizer or dictfindword() had loaded it, and down() listed + # only those. findconcept/getconcept now search the file on a miss, + # down() loads it whole, and the tokenizer searches only dictionary + # files (top concept "dictionary", or a .dict twin) -- not domain + # KBs, which made spelling candidates 100x slower. + FIX=.github/workflows/tests/kbb-lazy-lookup + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> /tmp/lazy.err + got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt") + want=$(printf 'domain not a word: 1\ntwin linked: 1\nkilo: 1\nnested: 1\nquoted key: 1\nmiss: 1\nmiss again: 1\ngetconcept: 1\nchildren: 17') + if [ "$got" != "$want" ]; then + echo "FAIL(kbb-lazy-lookup): out.txt was:"; echo "$got"; cat /tmp/lazy.err; exit 1 + fi + # output-path: "path" << x silently wrote nothing for an absolute + # path (it was appended to the output folder). $apppath follows -ANA, + # so give it an absolute one. + FIX=.github/workflows/tests/output-path + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ABS="$(pwd -W 2>/dev/null || pwd)/$FIX" + ./bin/nlp -ANA "$ABS" -WORK ./ "$ABS/input/text.txt" 2> /tmp/opath.err + check() { + got=$(tr -d '\r' < "$1" 2>/dev/null || echo "(missing)") + [ "$got" = "$2" ] || { echo "FAIL(output-path): $1 was '$got'"; cat /tmp/opath.err; exit 1; } + } + check "$FIX/output/abs.txt" "$(printf 'one\ntwo')" + check "$FIX/output/expr.txt" expr + check "$FIX/input/text.txt_log/rel.txt" rel + check "$FIX/input/text.txt_log/out.txt" done + n=$(grep -c "can't open output file" /tmp/opath.err || true) + if [ "$n" != 1 ]; then + echo "FAIL(output-path): bad path reported $n times on stderr, want 1"; cat /tmp/opath.err; exit 1 + fi + echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path" + - name: Run-mode globals ($dev / $silent, Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 2bcc8f1b..93f13f67 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -167,6 +167,45 @@ jobs: fi echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr" + - name: Lazy kbb lookups from any pass, and << to a path + shell: bash + run: | + set -eo pipefail + # kbb-lazy-lookup: a key of a lazy *full.kbb used to be found only if + # the tokenizer or dictfindword() had loaded it, and down() listed + # only those. findconcept/getconcept now search the file on a miss, + # down() loads it whole, and the tokenizer searches only dictionary + # files (top concept "dictionary", or a .dict twin) -- not domain + # KBs, which made spelling candidates 100x slower. + FIX=.github/workflows/tests/kbb-lazy-lookup + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> /tmp/lazy.err + got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt") + want=$(printf 'domain not a word: 1\ntwin linked: 1\nkilo: 1\nnested: 1\nquoted key: 1\nmiss: 1\nmiss again: 1\ngetconcept: 1\nchildren: 17') + if [ "$got" != "$want" ]; then + echo "FAIL(kbb-lazy-lookup): out.txt was:"; echo "$got"; cat /tmp/lazy.err; exit 1 + fi + # output-path: "path" << x silently wrote nothing for an absolute + # path (it was appended to the output folder). $apppath follows -ANA, + # so give it an absolute one. + FIX=.github/workflows/tests/output-path + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ABS="$(pwd -W 2>/dev/null || pwd)/$FIX" + ./bin/nlp -ANA "$ABS" -WORK ./ "$ABS/input/text.txt" 2> /tmp/opath.err + check() { + got=$(tr -d '\r' < "$1" 2>/dev/null || echo "(missing)") + [ "$got" = "$2" ] || { echo "FAIL(output-path): $1 was '$got'"; cat /tmp/opath.err; exit 1; } + } + check "$FIX/output/abs.txt" "$(printf 'one\ntwo')" + check "$FIX/output/expr.txt" expr + check "$FIX/input/text.txt_log/rel.txt" rel + check "$FIX/input/text.txt_log/out.txt" done + n=$(grep -c "can't open output file" /tmp/opath.err || true) + if [ "$n" != 1 ]; then + echo "FAIL(output-path): bad path reported $n times on stderr, want 1"; cat /tmp/opath.err; exit 1 + fi + echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path" + # INTEL MACS (#231). The steps above built and tested arm64 (Apple # Silicon). Build the same tree for x86_64 too and merge the two with # lipo, so nlpm.exe and the compile libraries are universal binaries: diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 37d4512d..3e880fd8 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -255,6 +255,45 @@ jobs: fi echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr" + - name: Lazy kbb lookups from any pass, and << to a path + shell: bash + run: | + set -eo pipefail + # kbb-lazy-lookup: a key of a lazy *full.kbb used to be found only if + # the tokenizer or dictfindword() had loaded it, and down() listed + # only those. findconcept/getconcept now search the file on a miss, + # down() loads it whole, and the tokenizer searches only dictionary + # files (top concept "dictionary", or a .dict twin) -- not domain + # KBs, which made spelling candidates 100x slower. + FIX=.github/workflows/tests/kbb-lazy-lookup + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> /tmp/lazy.err + got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt") + want=$(printf 'domain not a word: 1\ntwin linked: 1\nkilo: 1\nnested: 1\nquoted key: 1\nmiss: 1\nmiss again: 1\ngetconcept: 1\nchildren: 17') + if [ "$got" != "$want" ]; then + echo "FAIL(kbb-lazy-lookup): out.txt was:"; echo "$got"; cat /tmp/lazy.err; exit 1 + fi + # output-path: "path" << x silently wrote nothing for an absolute + # path (it was appended to the output folder). $apppath follows -ANA, + # so give it an absolute one. + FIX=.github/workflows/tests/output-path + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ABS="$(pwd -W 2>/dev/null || pwd)/$FIX" + ./bin/Release/nlp.exe -ANA "$ABS" -WORK ./ "$ABS/input/text.txt" 2> /tmp/opath.err + check() { + got=$(tr -d '\r' < "$1" 2>/dev/null || echo "(missing)") + [ "$got" = "$2" ] || { echo "FAIL(output-path): $1 was '$got'"; cat /tmp/opath.err; exit 1; } + } + check "$FIX/output/abs.txt" "$(printf 'one\ntwo')" + check "$FIX/output/expr.txt" expr + check "$FIX/input/text.txt_log/rel.txt" rel + check "$FIX/input/text.txt_log/out.txt" done + n=$(grep -c "can't open output file" /tmp/opath.err || true) + if [ "$n" != 1 ]; then + echo "FAIL(output-path): bad path reported $n times on stderr, want 1"; cat /tmp/opath.err; exit 1 + fi + echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path" + - name: Run-mode globals ($dev / $silent) shell: bash run: | diff --git a/.github/workflows/tests/kbb-lazy-lookup/input/text.txt b/.github/workflows/tests/kbb-lazy-lookup/input/text.txt new file mode 100644 index 00000000..480a654d --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/input/text.txt @@ -0,0 +1 @@ +The golf hotel. diff --git a/.github/workflows/tests/kbb-lazy-lookup/kb/user/codes-full.kbb b/.github/workflows/tests/kbb-lazy-lookup/kb/user/codes-full.kbb new file mode 100644 index 00000000..878a56a3 --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/kb/user/codes-full.kbb @@ -0,0 +1,21 @@ +# A domain KB: lazily loaded, and searched only when a pass asks for a key. +codes + "heart attack": code=I21 + alpha: rank=1 + A001: desc=first + A002: desc=second + bravo: rank=2 + charlie: rank=3 + delta: rank=4 + echo: rank=5 + foxtrot: rank=6 + golf: rank=7 + hotel: rank=8 + india: rank=9 + juliet: rank=10 + kilo: rank=11 + lima: rank=12 + mike: rank=13 + november: rank=14 + oscar: rank=15 + papa: rank=16 diff --git a/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.dict b/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.dict new file mode 100644 index 00000000..17e43973 --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.dict @@ -0,0 +1,2 @@ +hotel pos=noun +zebra pos=noun diff --git a/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.kbb b/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.kbb new file mode 100644 index 00000000..c6011a43 --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.kbb @@ -0,0 +1,3 @@ +terms + hotel: sense=lodging + zebra: sense=animal diff --git a/.github/workflows/tests/kbb-lazy-lookup/spec/analyzer.seq b/.github/workflows/tests/kbb-lazy-lookup/spec/analyzer.seq new file mode 100644 index 00000000..f912b863 --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/spec/analyzer.seq @@ -0,0 +1,2 @@ +dicttokz nil # tokenize against the dictionary +nlp lookup # a later pass looks keys up diff --git a/.github/workflows/tests/kbb-lazy-lookup/spec/lookup.nlp b/.github/workflows/tests/kbb-lazy-lookup/spec/lookup.nlp new file mode 100644 index 00000000..d16bcea7 --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/spec/lookup.nlp @@ -0,0 +1,33 @@ +############################################### +# FILE: lookup.nlp +# SUBJ: Look keys up in the lazy codes-full.kbb the ordinary way, after +# tokenizing. Each line is 1 when the engine behaves. +############################################### + +@CODE +L("codes") = findconcept(findroot(),"codes"); +# The tokenizer and dictfindword do not search a domain KB: "golf" is in the +# input and in codes-full.kbb, but it is not a dictionary word. +"out.txt" << "domain not a word: " << str(dictfindword("golf") == 0) << "\n"; +# A twin kbb is still searched by the tokenizer, linked to its dict word. +"out.txt" << "twin linked: " << str(conval(dictfindword("hotel"),"meaning") != 0) << "\n"; +# findconcept loads a key on demand, with its whole block. +L("kilo") = findconcept(L("codes"),"kilo"); +"out.txt" << "kilo: " << str(numval(L("kilo"),"rank") == 11) << "\n"; +L("a1") = findconcept(findconcept(L("codes"),"alpha"),"A001"); +"out.txt" << "nested: " << str(strval(L("a1"),"desc") == "first") << "\n"; +"out.txt" << "quoted key: " << str(strval(findconcept(L("codes"),"heart attack"),"code") == "I21") << "\n"; +"out.txt" << "miss: " << str(findconcept(L("codes"),"zulu") == 0) << "\n"; +"out.txt" << "miss again: " << str(findconcept(L("codes"),"zulu") == 0) << "\n"; +# getconcept finds the file's key instead of making an empty one. +"out.txt" << "getconcept: " << str(numval(getconcept(L("codes"),"lima"),"rank") == 12) << "\n"; +# down() lists every key once: 17, with kilo, alpha, lima, heart attack +# loaded above and not built twice. +L("n") = 0; +L("c") = down(L("codes")); +while (L("c")) { + L("n")++; + L("c") = next(L("c")); +} +"out.txt" << "children: " << str(L("n")) << "\n"; +@@CODE diff --git a/cs/libconsh/cg.cpp b/cs/libconsh/cg.cpp index f91b89ed..16965ed1 100644 --- a/cs/libconsh/cg.cpp +++ b/cs/libconsh/cg.cpp @@ -904,7 +904,10 @@ CONCEPT *CG::findConcept( _TCHAR *name ) { -return (CONCEPT *) kbm_->acon_->con_get_child((CON *)parent, name); +CONCEPT *con = (CONCEPT *) kbm_->acon_->con_get_child((CON *)parent, name); +if (con || fullKBBs_.empty() || lazyBusy_ || !parent || !name) + return con; +return findLazyKey(parent, name); // A key of a lazy *full.kbb. } // Find the nth concept under parent. @@ -1962,6 +1965,9 @@ CONCEPT *CG::getConcept(CONCEPT *parent, _TCHAR *name) CONCEPT *con = (CONCEPT *) kbm_->acon_->con_get_child((CON *) parent, name); if (con) // FIX. // 08/07/00 AM. return con; // FIX. // 08/07/00 AM. +if (!fullKBBs_.empty() && !lazyBusy_ && parent && name + && (con = findLazyKey(parent, name))) + return con; // A key of a lazy *full.kbb. dirty_ = true; // Todo: check if made concept. // 05/12/00 AM. return (CONCEPT *) kbm_->acon_->con_add_basic(name, (CON *) parent); // 08/07/00 AM. } @@ -3714,7 +3720,7 @@ for (FullFile &f : fullDicts_) } for (FullFile &f : fullKBBs_) { - if (!f.merge || !f.merged.insert(str).second) + if (!f.merge || !f.words || !f.merged.insert(str).second) continue; CONCEPT *found = searchKBBFile(f, str); CONCEPT *meaning = 0; @@ -4795,7 +4801,17 @@ bool CG::openFullKBB(const std::string &file, bool merge) rootName = _T("dictionary"); // no top concept found; fall back f.root = getConcept(findRoot(), rootName.data()); - std::_t_cerr << _T("[Lazy-loading words from ") << file + // Searched for every word only if it is a dictionary: its top concept + // is "dictionary", or a same-stem .dict twin sits beside it (en-full.kbb + // with en-full.dict). A domain KB (an ICD index, synonyms) is searched + // only when findconcept/getconcept/down ask for its keys: searching it + // for every token and spelling candidate made a spell pass 100x slower. + std::filesystem::path twin(file); + twin.replace_extension(".dict"); + f.words = rootName == _T("dictionary") || std::filesystem::exists(twin); + + std::_t_cerr << _T("[Lazy-loading ") << (f.words ? _T("words") : _T("keys")) + << _T(" from ") << file << _T(" under \"") << rootName << _T("\"]") << std::endl; // A word missed before this file was open may be in it. Only matters when @@ -4874,6 +4890,8 @@ CONCEPT *CG::findFullKBBWord(_TCHAR *str) for (FullFile &f : fullKBBs_) { if (f.merge) continue; // Searched by mergeFullWord. + if (!f.words) + continue; // A domain KB: findconcept only. CONCEPT *word = searchKBBFile(f, str); if (word && !first) first = word; @@ -4884,11 +4902,23 @@ CONCEPT *CG::findFullKBBWord(_TCHAR *str) // Binary-search one open *full.kbb for str. If found, build that word's block // (the word line plus its indented senses) into the KB, cache it, and return it. CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) +{ + std::streamoff keyStart; + if (!locateKBBKey(f, str, keyStart)) + return 0; + CONCEPT *word = buildKBBBlock(f, keyStart); + if (f.words) + cacheWordCon(str, word); // A domain key (eg an ICD code) is no word. + return word; +} + +// Binary-search one open *full.kbb for the key line " key:". Sets keyStart to +// the byte offset of that line. +bool CG::locateKBBKey(FullFile &f, const std::string &str, std::streamoff &keyStart) { std::string buf; - _TCHAR *line; std::streamoff lo = 0, hi = f.size; - std::streamoff keyStart, afterKey; + std::streamoff afterKey; std::string key; // Lower-bound: first word line whose word >= str. @@ -4902,7 +4932,7 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) hi = mid; continue; } - if (strcmp(key.c_str(), str) < 0) { + if (key < str) { if (afterKey <= lo) // Defensive: never move lo backwards/sideways break; // (would loop forever). str is past EOF here. lo = afterKey; @@ -4917,12 +4947,17 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) f.stream.seekg(lo); do { if (!nextKBBKey(f, keyStart, key, afterKey)) - return 0; - } while (strcmp(key.c_str(), str) < 0); - if (strcmp(key.c_str(), str) != 0) - return 0; + return false; + } while (key < str); + return key == str; +} - // Build the word's block under the "dictionary" root. +// Build the block of the key line at keyStart (the key plus its indented +// lines) under the file's top concept, and return the key's concept. +CONCEPT *CG::buildKBBBlock(FullFile &f, std::streamoff keyStart) +{ + std::string buf; + _TCHAR *line; f.stream.clear(); f.stream.seekg(keyStart); @@ -4930,6 +4965,8 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) cons.push_back(f.root); // cons[0] = parent for the word. int32_t index = 0; + bool busy = lazyBusy_; // Making the key must not + lazyBusy_ = true; // search the file again. bool inBlock = false; // Single-line "/* */" only. readFullLine(f.stream, buf); // The word line. line = buf.data(); @@ -4956,11 +4993,100 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) break; parseKBBLine(line, cons, index, f.root); } - - cacheWordCon(str, word); + lazyBusy_ = busy; return word; } +/******************************************** +* FN: FINDLAZYKEY +* SUBJ: Find a key of a lazy *full.kbb on demand. +* RET: The key's concept, or 0. +* NOTE: findconcept() and getconcept() call this on a miss. A key under +* the top concept of an open *full.kbb is binary-searched in that +* file only and its block built, so a later pass finds it the +* ordinary way: it used to be found only if the tokenizer or a +* dictfindword() call had loaded it. A name the file lacks is +* remembered, so it is searched once. SaveKB quotes a name with a +* space ("heart attack":), so that form is tried too. +********************************************/ + +CONCEPT *CG::findLazyKey(CONCEPT *parent, _TCHAR *name) +{ + for (FullFile &f : fullKBBs_) { + if (f.root != parent) + continue; + if (f.all || f.misses.count(name)) + return 0; + std::streamoff keyStart; + std::string key = name; + bool found = locateKBBKey(f, key, keyStart); + if (!found) { + key = "\"" + key + "\""; + found = locateKBBKey(f, key, keyStart); + } + if (!found) { + f.misses.insert(name); + return 0; + } + if (f.merge) + f.merged.insert(name); // mergeFullWord must not build it twice. + CONCEPT *con = buildKBBBlock(f, keyStart); + if (f.words) + cacheWordCon(name, con); + return con; + } + return 0; +} + +/******************************************** +* FN: LOADLAZY +* SUBJ: Load every key of a lazy *full.kbb, for down() and next(). +* NOTE: Only for the file whose top concept is con. Keys already in memory +* are kept and not built twice. Costs what loading the file eagerly +* would, once, and only in an analyzer that lists the keys. +********************************************/ + +void CG::loadLazy(CONCEPT *con) +{ + if (!con || fullKBBs_.empty()) + return; + for (FullFile &f : fullKBBs_) { + if (f.root != con || f.all) + continue; + f.all = true; + + // Names already loaded, read once: searching the children for each + // key would be quadratic on a large file. + std::set have; + std::vector<_TCHAR> name(MAXMSG * 4); + for (CONCEPT *c = Down(con); c; c = Next(c)) { + conceptName(c, name.data(), (long) name.size()); + have.insert(name.data()); + } + + std::streamoff keyStart, afterKey; + std::string key; + f.stream.clear(); + f.stream.seekg(0); + while (nextKBBKey(f, keyStart, key, afterKey)) { + std::string bare = key; + if (bare.size() > 1 && bare.front() == '"' && bare.back() == '"') + bare = bare.substr(1, bare.size() - 2); + if (have.count(bare)) + continue; + CONCEPT *c = buildKBBBlock(f, keyStart); + if (f.words && c) + cacheWordCon(bare.data(), c); + if (f.merge) + f.merged.insert(bare); + f.stream.clear(); + f.stream.seekg(afterKey); // buildKBBBlock read ahead. + } + f.stream.clear(); + return; + } +} + // Look up str across every open *full.dict, returning the first match. CONCEPT *CG::findFullDictWord(_TCHAR *str) { diff --git a/include/Api/consh/cg.h b/include/Api/consh/cg.h index a3239265..53072342 100644 --- a/include/Api/consh/cg.h +++ b/include/Api/consh/cg.h @@ -270,6 +270,13 @@ class LIBCONSH_API CG // already in memory, once each, and merged into them. // 09/14/26 DD. bool merge = false; std::set merged; // merge only: words already searched. + // Searched for every word by the tokenizer and dictfindword: a kbb + // whose top concept is "dictionary", or that has a same-stem .dict + // twin. Any other kbb (a domain KB) is searched only by findconcept, + // getconcept and down on its top concept. + bool words = true; + bool all = false; // down() loaded the whole file. + std::set misses; // Keys findconcept found absent. }; bool stemEndsWithFull(const std::string &stem); bool openFullKBB(const std::string &file, bool merge = false); @@ -288,6 +295,12 @@ class LIBCONSH_API CG CONCEPT *findFullDictWord(_TCHAR *str); CONCEPT *mergeFullWord(_TCHAR *str, CONCEPT *word); // 09/14/26 DD. CONCEPT *searchKBBFile(FullFile &f, _TCHAR *str); + bool locateKBBKey(FullFile &f, const std::string &key, std::streamoff &keyStart); + CONCEPT *buildKBBBlock(FullFile &f, std::streamoff keyStart); + CONCEPT *findLazyKey(CONCEPT *parent, _TCHAR *name); + // Load every key of the lazy file whose top concept is con, so down() + // and next() see them all. No-op for any other concept. + void loadLazy(CONCEPT *con); CONCEPT *searchDictFile(FullFile &f, _TCHAR *str); void cacheWordCon(_TCHAR *name, CONCEPT *con); void logMissingWord(_TCHAR *str); @@ -701,6 +714,7 @@ class LIBCONSH_API CG // Negative cache + log of words not found in any "*full" file, so each // unknown word is searched once and recorded once for later KB curation. std::set fullMissCache_; // Run-wide: words absent, skip re-search. + bool lazyBusy_ = false; // Building a lazy block: no nested lazy lookups. std::set loggedThisFile_; // Per-file: words already in this file's log. std::_t_ofstream fullMissLog_; bool fullMissLogOpen_ = false; diff --git a/lite/fn.cpp b/lite/fn.cpp index 56c05d75..0e3a0bb5 100644 --- a/lite/fn.cpp +++ b/lite/fn.cpp @@ -1145,7 +1145,10 @@ CONCEPT *child = 0; if (name1 && *name1) // 2nd arg was name. child = cg->findConcept(conc1, name1); else // 2nd arg was num. + { + cg->loadLazy(conc1); // A lazy *full.kbb: every key counts. child = cg->findConcept(conc1, num1); + } if (child) // FIX. // 11/15/00 AM. sem = new RFASem(child, RS_KBCONCEPT, cg); @@ -2480,6 +2483,7 @@ CONCEPT *hier1 = hier_sem->getKBconcept(); // Need to get the current KB. CG *cg = parse->getAna()->getCG(); +cg->loadLazy(hier1); // A lazy *full.kbb: list every key, not only loaded ones. CONCEPT *conc = cg->Down(hier1); // return false; // 04/28/00 AM. diff --git a/lite/fnrun.cpp b/lite/fnrun.cpp index b6887909..2010a731 100644 --- a/lite/fnrun.cpp +++ b/lite/fnrun.cpp @@ -244,6 +244,7 @@ if (sem->getType() != RS_KBCONCEPT) return 0; } CONCEPT *conc1 = sem->getKBconcept(); +cg->loadLazy(conc1); // A lazy *full.kbb: every key counts. CONCEPT *child = cg->findConcept(conc1, num); delete sem; // MEM LEAK. // 06/27/00 AM. @@ -1850,6 +1851,7 @@ CONCEPT *hier1 = hier_sem->getKBconcept(); Parse *parse = nlppp->getParse(); CG *cg = parse->getAna()->getCG(); +cg->loadLazy(hier1); // A lazy *full.kbb: list every key, not only loaded ones. CONCEPT *conc = cg->Down(hier1); // return false; // 06/02/00 AM. diff --git a/nlp/main.cpp b/nlp/main.cpp index bad1b1ac..abd988fa 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,7 +15,7 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.38" +#define NLP_ENGINE_VERSION "4.2.39" bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &, bool &, _TCHAR *&); void cmdHelpargs(_TCHAR *);