diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 2eb6af74..edcd71ea 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -318,6 +318,46 @@ jobs: fi echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr" + - name: Lazy kbb lookups from any pass, and << to a path (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # kbb-lazy-lookup: a key of a lazy *full.kbb used to be found only if + # the tokenizer or dictfindword() had loaded it, and down() listed + # only those. findconcept/getconcept now search the file on a miss, + # down() loads it whole, and the tokenizer searches only dictionary + # files (top concept "dictionary", or a .dict twin) -- not domain + # KBs, which made spelling candidates 100x slower. + FIX=.github/workflows/tests/kbb-lazy-lookup + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> /tmp/lazy.err + got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt") + want=$(printf 'domain not a word: 1\ntwin linked: 1\nkilo: 1\nnested: 1\nquoted key: 1\nmiss: 1\nmiss again: 1\ngetconcept: 1\nchildren: 17') + if [ "$got" != "$want" ]; then + echo "FAIL(kbb-lazy-lookup): out.txt was:"; echo "$got"; cat /tmp/lazy.err; exit 1 + fi + # output-path: "path" << x silently wrote nothing for an absolute + # path (it was appended to the output folder). $apppath follows -ANA, + # so give it an absolute one. + FIX=.github/workflows/tests/output-path + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ABS="$(pwd -W 2>/dev/null || pwd)/$FIX" + ./bin/nlp -ANA "$ABS" -WORK ./ "$ABS/input/text.txt" 2> /tmp/opath.err + check() { + got=$(tr -d '\r' < "$1" 2>/dev/null || echo "(missing)") + [ "$got" = "$2" ] || { echo "FAIL(output-path): $1 was '$got'"; cat /tmp/opath.err; exit 1; } + } + check "$FIX/output/abs.txt" "$(printf 'one\ntwo')" + check "$FIX/output/expr.txt" expr + check "$FIX/input/text.txt_log/rel.txt" rel + check "$FIX/input/text.txt_log/out.txt" done + n=$(grep -c "can't open output file" /tmp/opath.err || true) + if [ "$n" != 1 ]; then + echo "FAIL(output-path): bad path reported $n times on stderr, want 1"; cat /tmp/opath.err; exit 1 + fi + echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path" + - name: Run-mode globals ($dev / $silent, Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 2bcc8f1b..93f13f67 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -167,6 +167,45 @@ jobs: fi echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr" + - name: Lazy kbb lookups from any pass, and << to a path + shell: bash + run: | + set -eo pipefail + # kbb-lazy-lookup: a key of a lazy *full.kbb used to be found only if + # the tokenizer or dictfindword() had loaded it, and down() listed + # only those. findconcept/getconcept now search the file on a miss, + # down() loads it whole, and the tokenizer searches only dictionary + # files (top concept "dictionary", or a .dict twin) -- not domain + # KBs, which made spelling candidates 100x slower. + FIX=.github/workflows/tests/kbb-lazy-lookup + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> /tmp/lazy.err + got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt") + want=$(printf 'domain not a word: 1\ntwin linked: 1\nkilo: 1\nnested: 1\nquoted key: 1\nmiss: 1\nmiss again: 1\ngetconcept: 1\nchildren: 17') + if [ "$got" != "$want" ]; then + echo "FAIL(kbb-lazy-lookup): out.txt was:"; echo "$got"; cat /tmp/lazy.err; exit 1 + fi + # output-path: "path" << x silently wrote nothing for an absolute + # path (it was appended to the output folder). $apppath follows -ANA, + # so give it an absolute one. + FIX=.github/workflows/tests/output-path + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ABS="$(pwd -W 2>/dev/null || pwd)/$FIX" + ./bin/nlp -ANA "$ABS" -WORK ./ "$ABS/input/text.txt" 2> /tmp/opath.err + check() { + got=$(tr -d '\r' < "$1" 2>/dev/null || echo "(missing)") + [ "$got" = "$2" ] || { echo "FAIL(output-path): $1 was '$got'"; cat /tmp/opath.err; exit 1; } + } + check "$FIX/output/abs.txt" "$(printf 'one\ntwo')" + check "$FIX/output/expr.txt" expr + check "$FIX/input/text.txt_log/rel.txt" rel + check "$FIX/input/text.txt_log/out.txt" done + n=$(grep -c "can't open output file" /tmp/opath.err || true) + if [ "$n" != 1 ]; then + echo "FAIL(output-path): bad path reported $n times on stderr, want 1"; cat /tmp/opath.err; exit 1 + fi + echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path" + # INTEL MACS (#231). The steps above built and tested arm64 (Apple # Silicon). Build the same tree for x86_64 too and merge the two with # lipo, so nlpm.exe and the compile libraries are universal binaries: diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 37d4512d..3e880fd8 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -255,6 +255,45 @@ jobs: fi echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr" + - name: Lazy kbb lookups from any pass, and << to a path + shell: bash + run: | + set -eo pipefail + # kbb-lazy-lookup: a key of a lazy *full.kbb used to be found only if + # the tokenizer or dictfindword() had loaded it, and down() listed + # only those. findconcept/getconcept now search the file on a miss, + # down() loads it whole, and the tokenizer searches only dictionary + # files (top concept "dictionary", or a .dict twin) -- not domain + # KBs, which made spelling candidates 100x slower. + FIX=.github/workflows/tests/kbb-lazy-lookup + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> /tmp/lazy.err + got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt") + want=$(printf 'domain not a word: 1\ntwin linked: 1\nkilo: 1\nnested: 1\nquoted key: 1\nmiss: 1\nmiss again: 1\ngetconcept: 1\nchildren: 17') + if [ "$got" != "$want" ]; then + echo "FAIL(kbb-lazy-lookup): out.txt was:"; echo "$got"; cat /tmp/lazy.err; exit 1 + fi + # output-path: "path" << x silently wrote nothing for an absolute + # path (it was appended to the output folder). $apppath follows -ANA, + # so give it an absolute one. + FIX=.github/workflows/tests/output-path + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ABS="$(pwd -W 2>/dev/null || pwd)/$FIX" + ./bin/Release/nlp.exe -ANA "$ABS" -WORK ./ "$ABS/input/text.txt" 2> /tmp/opath.err + check() { + got=$(tr -d '\r' < "$1" 2>/dev/null || echo "(missing)") + [ "$got" = "$2" ] || { echo "FAIL(output-path): $1 was '$got'"; cat /tmp/opath.err; exit 1; } + } + check "$FIX/output/abs.txt" "$(printf 'one\ntwo')" + check "$FIX/output/expr.txt" expr + check "$FIX/input/text.txt_log/rel.txt" rel + check "$FIX/input/text.txt_log/out.txt" done + n=$(grep -c "can't open output file" /tmp/opath.err || true) + if [ "$n" != 1 ]; then + echo "FAIL(output-path): bad path reported $n times on stderr, want 1"; cat /tmp/opath.err; exit 1 + fi + echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path" + - name: Run-mode globals ($dev / $silent) shell: bash run: | diff --git a/.github/workflows/tests/kbb-lazy-lookup/input/text.txt b/.github/workflows/tests/kbb-lazy-lookup/input/text.txt new file mode 100644 index 00000000..480a654d --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/input/text.txt @@ -0,0 +1 @@ +The golf hotel. diff --git a/.github/workflows/tests/kbb-lazy-lookup/kb/user/codes-full.kbb b/.github/workflows/tests/kbb-lazy-lookup/kb/user/codes-full.kbb new file mode 100644 index 00000000..878a56a3 --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/kb/user/codes-full.kbb @@ -0,0 +1,21 @@ +# A domain KB: lazily loaded, and searched only when a pass asks for a key. +codes + "heart attack": code=I21 + alpha: rank=1 + A001: desc=first + A002: desc=second + bravo: rank=2 + charlie: rank=3 + delta: rank=4 + echo: rank=5 + foxtrot: rank=6 + golf: rank=7 + hotel: rank=8 + india: rank=9 + juliet: rank=10 + kilo: rank=11 + lima: rank=12 + mike: rank=13 + november: rank=14 + oscar: rank=15 + papa: rank=16 diff --git a/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.dict b/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.dict new file mode 100644 index 00000000..17e43973 --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.dict @@ -0,0 +1,2 @@ +hotel pos=noun +zebra pos=noun diff --git a/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.kbb b/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.kbb new file mode 100644 index 00000000..c6011a43 --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/kb/user/terms-full.kbb @@ -0,0 +1,3 @@ +terms + hotel: sense=lodging + zebra: sense=animal diff --git a/.github/workflows/tests/kbb-lazy-lookup/spec/analyzer.seq b/.github/workflows/tests/kbb-lazy-lookup/spec/analyzer.seq new file mode 100644 index 00000000..f912b863 --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/spec/analyzer.seq @@ -0,0 +1,2 @@ +dicttokz nil # tokenize against the dictionary +nlp lookup # a later pass looks keys up diff --git a/.github/workflows/tests/kbb-lazy-lookup/spec/lookup.nlp b/.github/workflows/tests/kbb-lazy-lookup/spec/lookup.nlp new file mode 100644 index 00000000..d16bcea7 --- /dev/null +++ b/.github/workflows/tests/kbb-lazy-lookup/spec/lookup.nlp @@ -0,0 +1,33 @@ +############################################### +# FILE: lookup.nlp +# SUBJ: Look keys up in the lazy codes-full.kbb the ordinary way, after +# tokenizing. Each line is 1 when the engine behaves. +############################################### + +@CODE +L("codes") = findconcept(findroot(),"codes"); +# The tokenizer and dictfindword do not search a domain KB: "golf" is in the +# input and in codes-full.kbb, but it is not a dictionary word. +"out.txt" << "domain not a word: " << str(dictfindword("golf") == 0) << "\n"; +# A twin kbb is still searched by the tokenizer, linked to its dict word. +"out.txt" << "twin linked: " << str(conval(dictfindword("hotel"),"meaning") != 0) << "\n"; +# findconcept loads a key on demand, with its whole block. +L("kilo") = findconcept(L("codes"),"kilo"); +"out.txt" << "kilo: " << str(numval(L("kilo"),"rank") == 11) << "\n"; +L("a1") = findconcept(findconcept(L("codes"),"alpha"),"A001"); +"out.txt" << "nested: " << str(strval(L("a1"),"desc") == "first") << "\n"; +"out.txt" << "quoted key: " << str(strval(findconcept(L("codes"),"heart attack"),"code") == "I21") << "\n"; +"out.txt" << "miss: " << str(findconcept(L("codes"),"zulu") == 0) << "\n"; +"out.txt" << "miss again: " << str(findconcept(L("codes"),"zulu") == 0) << "\n"; +# getconcept finds the file's key instead of making an empty one. +"out.txt" << "getconcept: " << str(numval(getconcept(L("codes"),"lima"),"rank") == 12) << "\n"; +# down() lists every key once: 17, with kilo, alpha, lima, heart attack +# loaded above and not built twice. +L("n") = 0; +L("c") = down(L("codes")); +while (L("c")) { + L("n")++; + L("c") = next(L("c")); +} +"out.txt" << "children: " << str(L("n")) << "\n"; +@@CODE diff --git a/.github/workflows/tests/output-path/input/text.txt b/.github/workflows/tests/output-path/input/text.txt new file mode 100644 index 00000000..587be6b4 --- /dev/null +++ b/.github/workflows/tests/output-path/input/text.txt @@ -0,0 +1 @@ +x diff --git a/.github/workflows/tests/output-path/spec/analyzer.seq b/.github/workflows/tests/output-path/spec/analyzer.seq new file mode 100644 index 00000000..35c9bf6f --- /dev/null +++ b/.github/workflows/tests/output-path/spec/analyzer.seq @@ -0,0 +1,2 @@ +tokenize nil # tokenize +nlp write # write to paths diff --git a/.github/workflows/tests/output-path/spec/write.nlp b/.github/workflows/tests/output-path/spec/write.nlp new file mode 100644 index 00000000..f2aff10a --- /dev/null +++ b/.github/workflows/tests/output-path/spec/write.nlp @@ -0,0 +1,18 @@ +############################################### +# FILE: write.nlp +# SUBJ: << to a path. An absolute path is used as given; a relative name +# goes in the input's _log folder; a path that can't be opened is +# reported once, to err.log and stderr. +############################################### + +@CODE +L("abs") = G("$apppath") + "/output/abs.txt"; +L("abs") << "one\n"; +L("abs") << "two\n"; +G("$apppath") + "/output/expr.txt" << "expr\n"; +"rel.txt" << "rel\n"; +L("bad") = G("$apppath") + "/no/such/dir/x.txt"; +L("bad") << "a\n"; +L("bad") << "b\n"; +"out.txt" << "done\n"; +@@CODE diff --git a/cs/libconsh/cg.cpp b/cs/libconsh/cg.cpp index f91b89ed..16965ed1 100644 --- a/cs/libconsh/cg.cpp +++ b/cs/libconsh/cg.cpp @@ -904,7 +904,10 @@ CONCEPT *CG::findConcept( _TCHAR *name ) { -return (CONCEPT *) kbm_->acon_->con_get_child((CON *)parent, name); +CONCEPT *con = (CONCEPT *) kbm_->acon_->con_get_child((CON *)parent, name); +if (con || fullKBBs_.empty() || lazyBusy_ || !parent || !name) + return con; +return findLazyKey(parent, name); // A key of a lazy *full.kbb. } // Find the nth concept under parent. @@ -1962,6 +1965,9 @@ CONCEPT *CG::getConcept(CONCEPT *parent, _TCHAR *name) CONCEPT *con = (CONCEPT *) kbm_->acon_->con_get_child((CON *) parent, name); if (con) // FIX. // 08/07/00 AM. return con; // FIX. // 08/07/00 AM. +if (!fullKBBs_.empty() && !lazyBusy_ && parent && name + && (con = findLazyKey(parent, name))) + return con; // A key of a lazy *full.kbb. dirty_ = true; // Todo: check if made concept. // 05/12/00 AM. return (CONCEPT *) kbm_->acon_->con_add_basic(name, (CON *) parent); // 08/07/00 AM. } @@ -3714,7 +3720,7 @@ for (FullFile &f : fullDicts_) } for (FullFile &f : fullKBBs_) { - if (!f.merge || !f.merged.insert(str).second) + if (!f.merge || !f.words || !f.merged.insert(str).second) continue; CONCEPT *found = searchKBBFile(f, str); CONCEPT *meaning = 0; @@ -4795,7 +4801,17 @@ bool CG::openFullKBB(const std::string &file, bool merge) rootName = _T("dictionary"); // no top concept found; fall back f.root = getConcept(findRoot(), rootName.data()); - std::_t_cerr << _T("[Lazy-loading words from ") << file + // Searched for every word only if it is a dictionary: its top concept + // is "dictionary", or a same-stem .dict twin sits beside it (en-full.kbb + // with en-full.dict). A domain KB (an ICD index, synonyms) is searched + // only when findconcept/getconcept/down ask for its keys: searching it + // for every token and spelling candidate made a spell pass 100x slower. + std::filesystem::path twin(file); + twin.replace_extension(".dict"); + f.words = rootName == _T("dictionary") || std::filesystem::exists(twin); + + std::_t_cerr << _T("[Lazy-loading ") << (f.words ? _T("words") : _T("keys")) + << _T(" from ") << file << _T(" under \"") << rootName << _T("\"]") << std::endl; // A word missed before this file was open may be in it. Only matters when @@ -4874,6 +4890,8 @@ CONCEPT *CG::findFullKBBWord(_TCHAR *str) for (FullFile &f : fullKBBs_) { if (f.merge) continue; // Searched by mergeFullWord. + if (!f.words) + continue; // A domain KB: findconcept only. CONCEPT *word = searchKBBFile(f, str); if (word && !first) first = word; @@ -4884,11 +4902,23 @@ CONCEPT *CG::findFullKBBWord(_TCHAR *str) // Binary-search one open *full.kbb for str. If found, build that word's block // (the word line plus its indented senses) into the KB, cache it, and return it. CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) +{ + std::streamoff keyStart; + if (!locateKBBKey(f, str, keyStart)) + return 0; + CONCEPT *word = buildKBBBlock(f, keyStart); + if (f.words) + cacheWordCon(str, word); // A domain key (eg an ICD code) is no word. + return word; +} + +// Binary-search one open *full.kbb for the key line " key:". Sets keyStart to +// the byte offset of that line. +bool CG::locateKBBKey(FullFile &f, const std::string &str, std::streamoff &keyStart) { std::string buf; - _TCHAR *line; std::streamoff lo = 0, hi = f.size; - std::streamoff keyStart, afterKey; + std::streamoff afterKey; std::string key; // Lower-bound: first word line whose word >= str. @@ -4902,7 +4932,7 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) hi = mid; continue; } - if (strcmp(key.c_str(), str) < 0) { + if (key < str) { if (afterKey <= lo) // Defensive: never move lo backwards/sideways break; // (would loop forever). str is past EOF here. lo = afterKey; @@ -4917,12 +4947,17 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) f.stream.seekg(lo); do { if (!nextKBBKey(f, keyStart, key, afterKey)) - return 0; - } while (strcmp(key.c_str(), str) < 0); - if (strcmp(key.c_str(), str) != 0) - return 0; + return false; + } while (key < str); + return key == str; +} - // Build the word's block under the "dictionary" root. +// Build the block of the key line at keyStart (the key plus its indented +// lines) under the file's top concept, and return the key's concept. +CONCEPT *CG::buildKBBBlock(FullFile &f, std::streamoff keyStart) +{ + std::string buf; + _TCHAR *line; f.stream.clear(); f.stream.seekg(keyStart); @@ -4930,6 +4965,8 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) cons.push_back(f.root); // cons[0] = parent for the word. int32_t index = 0; + bool busy = lazyBusy_; // Making the key must not + lazyBusy_ = true; // search the file again. bool inBlock = false; // Single-line "/* */" only. readFullLine(f.stream, buf); // The word line. line = buf.data(); @@ -4956,11 +4993,100 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) break; parseKBBLine(line, cons, index, f.root); } - - cacheWordCon(str, word); + lazyBusy_ = busy; return word; } +/******************************************** +* FN: FINDLAZYKEY +* SUBJ: Find a key of a lazy *full.kbb on demand. +* RET: The key's concept, or 0. +* NOTE: findconcept() and getconcept() call this on a miss. A key under +* the top concept of an open *full.kbb is binary-searched in that +* file only and its block built, so a later pass finds it the +* ordinary way: it used to be found only if the tokenizer or a +* dictfindword() call had loaded it. A name the file lacks is +* remembered, so it is searched once. SaveKB quotes a name with a +* space ("heart attack":), so that form is tried too. +********************************************/ + +CONCEPT *CG::findLazyKey(CONCEPT *parent, _TCHAR *name) +{ + for (FullFile &f : fullKBBs_) { + if (f.root != parent) + continue; + if (f.all || f.misses.count(name)) + return 0; + std::streamoff keyStart; + std::string key = name; + bool found = locateKBBKey(f, key, keyStart); + if (!found) { + key = "\"" + key + "\""; + found = locateKBBKey(f, key, keyStart); + } + if (!found) { + f.misses.insert(name); + return 0; + } + if (f.merge) + f.merged.insert(name); // mergeFullWord must not build it twice. + CONCEPT *con = buildKBBBlock(f, keyStart); + if (f.words) + cacheWordCon(name, con); + return con; + } + return 0; +} + +/******************************************** +* FN: LOADLAZY +* SUBJ: Load every key of a lazy *full.kbb, for down() and next(). +* NOTE: Only for the file whose top concept is con. Keys already in memory +* are kept and not built twice. Costs what loading the file eagerly +* would, once, and only in an analyzer that lists the keys. +********************************************/ + +void CG::loadLazy(CONCEPT *con) +{ + if (!con || fullKBBs_.empty()) + return; + for (FullFile &f : fullKBBs_) { + if (f.root != con || f.all) + continue; + f.all = true; + + // Names already loaded, read once: searching the children for each + // key would be quadratic on a large file. + std::set have; + std::vector<_TCHAR> name(MAXMSG * 4); + for (CONCEPT *c = Down(con); c; c = Next(c)) { + conceptName(c, name.data(), (long) name.size()); + have.insert(name.data()); + } + + std::streamoff keyStart, afterKey; + std::string key; + f.stream.clear(); + f.stream.seekg(0); + while (nextKBBKey(f, keyStart, key, afterKey)) { + std::string bare = key; + if (bare.size() > 1 && bare.front() == '"' && bare.back() == '"') + bare = bare.substr(1, bare.size() - 2); + if (have.count(bare)) + continue; + CONCEPT *c = buildKBBBlock(f, keyStart); + if (f.words && c) + cacheWordCon(bare.data(), c); + if (f.merge) + f.merged.insert(bare); + f.stream.clear(); + f.stream.seekg(afterKey); // buildKBBBlock read ahead. + } + f.stream.clear(); + return; + } +} + // Look up str across every open *full.dict, returning the first match. CONCEPT *CG::findFullDictWord(_TCHAR *str) { diff --git a/include/Api/consh/cg.h b/include/Api/consh/cg.h index a3239265..53072342 100644 --- a/include/Api/consh/cg.h +++ b/include/Api/consh/cg.h @@ -270,6 +270,13 @@ class LIBCONSH_API CG // already in memory, once each, and merged into them. // 09/14/26 DD. bool merge = false; std::set merged; // merge only: words already searched. + // Searched for every word by the tokenizer and dictfindword: a kbb + // whose top concept is "dictionary", or that has a same-stem .dict + // twin. Any other kbb (a domain KB) is searched only by findconcept, + // getconcept and down on its top concept. + bool words = true; + bool all = false; // down() loaded the whole file. + std::set misses; // Keys findconcept found absent. }; bool stemEndsWithFull(const std::string &stem); bool openFullKBB(const std::string &file, bool merge = false); @@ -288,6 +295,12 @@ class LIBCONSH_API CG CONCEPT *findFullDictWord(_TCHAR *str); CONCEPT *mergeFullWord(_TCHAR *str, CONCEPT *word); // 09/14/26 DD. CONCEPT *searchKBBFile(FullFile &f, _TCHAR *str); + bool locateKBBKey(FullFile &f, const std::string &key, std::streamoff &keyStart); + CONCEPT *buildKBBBlock(FullFile &f, std::streamoff keyStart); + CONCEPT *findLazyKey(CONCEPT *parent, _TCHAR *name); + // Load every key of the lazy file whose top concept is con, so down() + // and next() see them all. No-op for any other concept. + void loadLazy(CONCEPT *con); CONCEPT *searchDictFile(FullFile &f, _TCHAR *str); void cacheWordCon(_TCHAR *name, CONCEPT *con); void logMissingWord(_TCHAR *str); @@ -701,6 +714,7 @@ class LIBCONSH_API CG // Negative cache + log of words not found in any "*full" file, so each // unknown word is searched once and recorded once for later KB curation. std::set fullMissCache_; // Run-wide: words absent, skip re-search. + bool lazyBusy_ = false; // Building a lazy block: no nested lazy lookups. std::set loggedThisFile_; // Per-file: words already in this file's log. std::_t_ofstream fullMissLog_; bool fullMissLogOpen_ = false; diff --git a/lite/fn.cpp b/lite/fn.cpp index 56c05d75..0e3a0bb5 100644 --- a/lite/fn.cpp +++ b/lite/fn.cpp @@ -1145,7 +1145,10 @@ CONCEPT *child = 0; if (name1 && *name1) // 2nd arg was name. child = cg->findConcept(conc1, name1); else // 2nd arg was num. + { + cg->loadLazy(conc1); // A lazy *full.kbb: every key counts. child = cg->findConcept(conc1, num1); + } if (child) // FIX. // 11/15/00 AM. sem = new RFASem(child, RS_KBCONCEPT, cg); @@ -2480,6 +2483,7 @@ CONCEPT *hier1 = hier_sem->getKBconcept(); // Need to get the current KB. CG *cg = parse->getAna()->getCG(); +cg->loadLazy(hier1); // A lazy *full.kbb: list every key, not only loaded ones. CONCEPT *conc = cg->Down(hier1); // return false; // 04/28/00 AM. diff --git a/lite/fnrun.cpp b/lite/fnrun.cpp index b6887909..2010a731 100644 --- a/lite/fnrun.cpp +++ b/lite/fnrun.cpp @@ -244,6 +244,7 @@ if (sem->getType() != RS_KBCONCEPT) return 0; } CONCEPT *conc1 = sem->getKBconcept(); +cg->loadLazy(conc1); // A lazy *full.kbb: every key counts. CONCEPT *child = cg->findConcept(conc1, num); delete sem; // MEM LEAK. // 06/27/00 AM. @@ -1850,6 +1851,7 @@ CONCEPT *hier1 = hier_sem->getKBconcept(); Parse *parse = nlppp->getParse(); CG *cg = parse->getAna()->getCG(); +cg->loadLazy(hier1); // A lazy *full.kbb: list every key, not only loaded ones. CONCEPT *conc = cg->Down(hier1); // return false; // 06/02/00 AM. diff --git a/lite/var.cpp b/lite/var.cpp index e4c3f1b6..854b39ac 100644 --- a/lite/var.cpp +++ b/lite/var.cpp @@ -14,6 +14,7 @@ All rights reserved. *******************************************************************************/ #include "StdAfx.h" +#include #include "machine.h" // 03/09/00 AM. #include "u_out.h" // 01/19/06 AM. #include "lite/lite.h" // 07/07/03 AM. @@ -2453,17 +2454,31 @@ else parse->setVars(dlist); } -// Set up proper path for file. -// Should check if it's relative or absolute. -_TCHAR buf[MAXSTR]; +// A relative name goes in the output folder. An absolute path ("/x", +// "\x", "C:...") is used as given, as openfile() does: it used to be +// appended to the output folder too, so the open failed and every write +// was silently dropped. (The old MAXSTR buffer also overflowed on a long +// path.) +std::string path; +if (fname[0] == '/' || fname[0] == '\\' || (fname[0] && fname[1] == ':')) + path = fname; +else + path = std::string(parse->getOutdir()) + DIR_CH + fname; _TCHAR *str = 0; -//sprintf(buf, "%s\\output\\%s", parse->getAppdir(), fname); // 03/10/99 AM. -//sprintf_s(buf, "%s\\output\\%s", parse->getAppdir(), fname); // 03/10/99 AM. -_stprintf(buf, _T("%s%c%s"), parse->getOutdir(),DIR_CH, fname); // 03/10/99 AM. -parse->internStr(buf, /*UP*/ str); +parse->internStr(path.data(), /*UP*/ str); // Open the output file for APPENDING. ostr = new std::_t_ofstream(TCHAR2CA(str), std::ios::app); +if (!*ostr) + { + // Report it once. The var keeps the failed stream, so later writes to + // this name do nothing instead of repeating the error. + std::_t_strstream gerrStr; + gerrStr << _T("[Output op: can't open file ") << str << _T("]") << std::ends; + parse->errOut(&gerrStr,false); + std::_t_cerr << _T("[Error: can't open output file ") << str + << _T(". Nothing written to it.]") << std::endl; + } #ifdef UNICODE *ostr << BOM_UTF8; #endif diff --git a/nlp/main.cpp b/nlp/main.cpp index bad1b1ac..abd988fa 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,7 +15,7 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.38" +#define NLP_ENGINE_VERSION "4.2.39" bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &, bool &, _TCHAR *&); void cmdHelpargs(_TCHAR *);