diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 6af604cf..2eb6af74 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -280,6 +280,44 @@ jobs: grep -qF '("note" "lone * and / here")' "$TREE" || { echo "FAIL(data block comments): dict string lost its * or /"; exit 1; } echo "[block comments] /* */ and .seq // are skipped in .seq, .kb, .dict and .kbb" + - name: Long kbb/dict lines and NLP++ errors (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # Three failures that exited 0 with nothing on the console. + # kbb-long-line: readKBB read lines into a 2,048-byte buffer and + # silently stopped loading at the first longer line. Its long line + # is a quoted array, which also loaded a stray 0 after each value. + # kbb-long-value: parseKBBLine and parseDictLine copied values + # unbounded into 512-byte buffers, overflowing the stack at startup. + # code-region-error: an NLP++ error still abandons the rest of its + # region, but is now echoed to stderr, not only written to err.log. + check() { + FIX=.github/workflows/tests/$1 + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> "/tmp/$1.err" + if [ ! -f "$FIX/input/text.txt_log/out.txt" ]; then + echo "FAIL($1): no out.txt"; cat "/tmp/$1.err"; exit 1 + fi + got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt") + if [ "$got" != "$2" ]; then + echo "FAIL($1): out.txt was '$got'"; cat "/tmp/$1.err"; exit 1 + fi + } + check kbb-long-line '30|item30|140' + check kbb-long-value '30|item30|600|3000|600|3000|noun' + check code-region-error "$(printf 'item 1\nitem 2\nitem 3\nnext pass ran')" + ERR=/tmp/code-region-error.err + grep -qF '[Error in NLP++ @CODE: pass 2 (loop) line 13, rest of the region skipped.' "$ERR" \ + || { echo "FAIL(code-region-error): @CODE error not on stderr"; cat "$ERR"; exit 1; } + # The @POST rule matches all four words: one line, not four. + n=$(grep -cF '[Error in NLP++ @POST: pass 4 (post) line 6,' "$ERR" || true) + if [ "$n" != 1 ]; then + echo "FAIL(code-region-error): @POST error echoed $n times, want 1"; cat "$ERR"; exit 1 + fi + echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr" + - name: Run-mode globals ($dev / $silent, Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 04913ee4..2bcc8f1b 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -130,6 +130,43 @@ jobs: fi echo "[rule debugger] pause points fire and leave the analysis unchanged" + - name: Long kbb/dict lines and NLP++ errors + shell: bash + run: | + set -eo pipefail + # Three failures that exited 0 with nothing on the console. + # kbb-long-line: readKBB read lines into a 2,048-byte buffer and + # silently stopped loading at the first longer line. Its long line + # is a quoted array, which also loaded a stray 0 after each value. + # kbb-long-value: parseKBBLine and parseDictLine copied values + # unbounded into 512-byte buffers, overflowing the stack at startup. + # code-region-error: an NLP++ error still abandons the rest of its + # region, but is now echoed to stderr, not only written to err.log. + check() { + FIX=.github/workflows/tests/$1 + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> "/tmp/$1.err" + if [ ! -f "$FIX/input/text.txt_log/out.txt" ]; then + echo "FAIL($1): no out.txt"; cat "/tmp/$1.err"; exit 1 + fi + got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt") + if [ "$got" != "$2" ]; then + echo "FAIL($1): out.txt was '$got'"; cat "/tmp/$1.err"; exit 1 + fi + } + check kbb-long-line '30|item30|140' + check kbb-long-value '30|item30|600|3000|600|3000|noun' + check code-region-error "$(printf 'item 1\nitem 2\nitem 3\nnext pass ran')" + ERR=/tmp/code-region-error.err + grep -qF '[Error in NLP++ @CODE: pass 2 (loop) line 13, rest of the region skipped.' "$ERR" \ + || { echo "FAIL(code-region-error): @CODE error not on stderr"; cat "$ERR"; exit 1; } + # The @POST rule matches all four words: one line, not four. + n=$(grep -cF '[Error in NLP++ @POST: pass 4 (post) line 6,' "$ERR" || true) + if [ "$n" != 1 ]; then + echo "FAIL(code-region-error): @POST error echoed $n times, want 1"; cat "$ERR"; exit 1 + fi + echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr" + # INTEL MACS (#231). The steps above built and tested arm64 (Apple # Silicon). Build the same tree for x86_64 too and merge the two with # lipo, so nlpm.exe and the compile libraries are universal binaries: diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index df2243ee..37d4512d 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -218,6 +218,43 @@ jobs: grep -qF '("note" "lone * and / here")' "$TREE" || { echo "FAIL(data block comments): dict string lost its * or /"; exit 1; } echo "[block comments] /* */ and .seq // are skipped in .seq, .kb, .dict and .kbb" + - name: Long kbb/dict lines and NLP++ errors + shell: bash + run: | + set -eo pipefail + # Three failures that exited 0 with nothing on the console. + # kbb-long-line: readKBB read lines into a 2,048-byte buffer and + # silently stopped loading at the first longer line. Its long line + # is a quoted array, which also loaded a stray 0 after each value. + # kbb-long-value: parseKBBLine and parseDictLine copied values + # unbounded into 512-byte buffers, overflowing the stack at startup. + # code-region-error: an NLP++ error still abandons the rest of its + # region, but is now echoed to stderr, not only written to err.log. + check() { + FIX=.github/workflows/tests/$1 + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> "/tmp/$1.err" + if [ ! -f "$FIX/input/text.txt_log/out.txt" ]; then + echo "FAIL($1): no out.txt"; cat "/tmp/$1.err"; exit 1 + fi + got=$(tr -d '\r' < "$FIX/input/text.txt_log/out.txt") + if [ "$got" != "$2" ]; then + echo "FAIL($1): out.txt was '$got'"; cat "/tmp/$1.err"; exit 1 + fi + } + check kbb-long-line '30|item30|140' + check kbb-long-value '30|item30|600|3000|600|3000|noun' + check code-region-error "$(printf 'item 1\nitem 2\nitem 3\nnext pass ran')" + ERR=/tmp/code-region-error.err + grep -qF '[Error in NLP++ @CODE: pass 2 (loop) line 13, rest of the region skipped.' "$ERR" \ + || { echo "FAIL(code-region-error): @CODE error not on stderr"; cat "$ERR"; exit 1; } + # The @POST rule matches all four words: one line, not four. + n=$(grep -cF '[Error in NLP++ @POST: pass 4 (post) line 6,' "$ERR" || true) + if [ "$n" != 1 ]; then + echo "FAIL(code-region-error): @POST error echoed $n times, want 1"; cat "$ERR"; exit 1 + fi + echo "[long lines] kbb/dict lines and values of any length load; NLP++ errors reach stderr" + - name: Run-mode globals ($dev / $silent) shell: bash run: | diff --git a/.github/workflows/tests/code-region-error/input/text.txt b/.github/workflows/tests/code-region-error/input/text.txt new file mode 100644 index 00000000..4d0e08d8 --- /dev/null +++ b/.github/workflows/tests/code-region-error/input/text.txt @@ -0,0 +1 @@ +one two three four diff --git a/.github/workflows/tests/code-region-error/spec/after.nlp b/.github/workflows/tests/code-region-error/spec/after.nlp new file mode 100644 index 00000000..94a617d7 --- /dev/null +++ b/.github/workflows/tests/code-region-error/spec/after.nlp @@ -0,0 +1,3 @@ +@CODE +"out.txt" << "next pass ran\n"; +@@CODE diff --git a/.github/workflows/tests/code-region-error/spec/analyzer.seq b/.github/workflows/tests/code-region-error/spec/analyzer.seq new file mode 100644 index 00000000..2446d376 --- /dev/null +++ b/.github/workflows/tests/code-region-error/spec/analyzer.seq @@ -0,0 +1,4 @@ +tokenize nil # tokenize +nlp loop # @CODE error mid-loop +nlp after # the next pass still runs +nlp post # @POST error on every match diff --git a/.github/workflows/tests/code-region-error/spec/loop.nlp b/.github/workflows/tests/code-region-error/spec/loop.nlp new file mode 100644 index 00000000..86020e74 --- /dev/null +++ b/.github/workflows/tests/code-region-error/spec/loop.nlp @@ -0,0 +1,17 @@ +# findconcept() on a null concept at item 3 is an NLP++ error. It abandons +# the rest of the @CODE region (items 3-5 and "loop finished" never write), +# and nlp exits 0. That is unchanged. What the CI step asserts is that the +# error is now on stderr, naming the pass, file and line, not only in the +# input's err.log. +@CODE +L("i") = 1; +while (L("i") <= 5) { + "out.txt" << "item " << str(L("i")) << "\n"; + L("parent") = makeconcept(findroot(),"reprotest"); + if (L("i") == 3) + L("parent") = 0; + L("c") = findconcept(L("parent"),"anything"); # line 13 + L("i")++; +} +"out.txt" << "loop finished\n"; +@@CODE diff --git a/.github/workflows/tests/code-region-error/spec/post.nlp b/.github/workflows/tests/code-region-error/spec/post.nlp new file mode 100644 index 00000000..09d6ce81 --- /dev/null +++ b/.github/workflows/tests/code-region-error/spec/post.nlp @@ -0,0 +1,8 @@ +# The same error in @POST, on a rule that matches every word. It is echoed +# to stderr once, not once per match. +@NODES _ROOT + +@POST + findconcept(0,"anything"); +@RULES +_xNIL <- _xALPHA @@ diff --git a/.github/workflows/tests/kbb-long-line/input/text.txt b/.github/workflows/tests/kbb-long-line/input/text.txt new file mode 100644 index 00000000..587be6b4 --- /dev/null +++ b/.github/workflows/tests/kbb-long-line/input/text.txt @@ -0,0 +1 @@ +x diff --git a/.github/workflows/tests/kbb-long-line/kb/user/longline.kbb b/.github/workflows/tests/kbb-long-line/kb/user/longline.kbb new file mode 100644 index 00000000..0c481697 --- /dev/null +++ b/.github/workflows/tests/kbb-long-line/kb/user/longline.kbb @@ -0,0 +1,34 @@ +# 30 concepts. item10's line is ~2,700 chars of short values. readKBB +# read lines into a 2,048-byte buffer: the load silently stopped at +# item10 (loaded 10 of 30), with nothing in any log. +longline + item1: n=1 + item2: n=2 + item3: n=3 + item4: n=4 + item5: n=5 + item6: n=6 + item7: n=7 + item8: n=8 + item9: n=9 + item10: term=["term number 000","term number 001","term number 002","term number 003","term number 004","term number 005","term number 006","term number 007","term number 008","term number 009","term number 010","term number 011","term number 012","term number 013","term number 014","term number 015","term number 016","term number 017","term number 018","term number 019","term number 020","term number 021","term number 022","term number 023","term number 024","term number 025","term number 026","term number 027","term number 028","term number 029","term number 030","term number 031","term number 032","term number 033","term number 034","term number 035","term number 036","term number 037","term number 038","term number 039","term number 040","term number 041","term number 042","term number 043","term number 044","term number 045","term number 046","term number 047","term number 048","term number 049","term number 050","term number 051","term number 052","term number 053","term number 054","term number 055","term number 056","term number 057","term number 058","term number 059","term number 060","term number 061","term number 062","term number 063","term number 064","term number 065","term number 066","term number 067","term number 068","term number 069","term number 070","term number 071","term number 072","term number 073","term number 074","term number 075","term number 076","term number 077","term number 078","term number 079","term number 080","term number 081","term number 082","term number 083","term number 084","term number 085","term number 086","term number 087","term number 088","term number 089","term number 090","term number 091","term number 092","term number 093","term number 094","term number 095","term number 096","term number 097","term number 098","term number 099","term number 100","term number 101","term number 102","term number 103","term number 104","term number 105","term number 106","term number 107","term number 108","term number 109","term number 110","term number 111","term number 112","term number 113","term number 114","term number 115","term number 116","term number 117","term number 118","term number 119","term number 120","term number 121","term number 122","term number 123","term number 124","term number 125","term number 126","term number 127","term number 128","term number 129","term number 130","term number 131","term number 132","term number 133","term number 134","term number 135","term number 136","term number 137","term number 138","term number 139"] + item11: n=11 + item12: n=12 + item13: n=13 + item14: n=14 + item15: n=15 + item16: n=16 + item17: n=17 + item18: n=18 + item19: n=19 + item20: n=20 + item21: n=21 + item22: n=22 + item23: n=23 + item24: n=24 + item25: n=25 + item26: n=26 + item27: n=27 + item28: n=28 + item29: n=29 + item30: n=30 diff --git a/.github/workflows/tests/kbb-long-line/spec/analyzer.seq b/.github/workflows/tests/kbb-long-line/spec/analyzer.seq new file mode 100644 index 00000000..cfb2a3ce --- /dev/null +++ b/.github/workflows/tests/kbb-long-line/spec/analyzer.seq @@ -0,0 +1,2 @@ +tokenize nil # tokenize +nlp probe # count what loaded diff --git a/.github/workflows/tests/kbb-long-line/spec/probe.nlp b/.github/workflows/tests/kbb-long-line/spec/probe.nlp new file mode 100644 index 00000000..7e66776e --- /dev/null +++ b/.github/workflows/tests/kbb-long-line/spec/probe.nlp @@ -0,0 +1,23 @@ +############################################### +# FILE: probe.nlp +# SUBJ: Count what longline.kbb loaded, and the values on its long line. +# NOTE: Expected out.txt: 30|item30|140 +############################################### + +@CODE +L("root") = findconcept(findroot(),"longline"); +L("n") = 0; +L("c") = down(L("root")); +while (L("c")) { + L("n")++; + L("last") = conceptname(L("c")); + L("c") = next(L("c")); +} +L("vals") = 0; +L("v") = attrvals(findattr(findconcept(L("root"),"item10"),"term")); +while (L("v")) { + L("vals")++; + L("v") = nextval(L("v")); +} +"out.txt" << str(L("n")) << "|" << L("last") << "|" << str(L("vals")) << "\n"; +@@CODE diff --git a/.github/workflows/tests/kbb-long-value/input/text.txt b/.github/workflows/tests/kbb-long-value/input/text.txt new file mode 100644 index 00000000..587be6b4 --- /dev/null +++ b/.github/workflows/tests/kbb-long-value/input/text.txt @@ -0,0 +1 @@ +x diff --git a/.github/workflows/tests/kbb-long-value/kb/user/longline.kbb b/.github/workflows/tests/kbb-long-value/kb/user/longline.kbb new file mode 100644 index 00000000..d1c91921 --- /dev/null +++ b/.github/workflows/tests/kbb-long-value/kb/user/longline.kbb @@ -0,0 +1,34 @@ +# 30 concepts. item10 has a 600-char value, item20 a 3,000-char one. +# parseKBBLine copied values unbounded into 512-char buffers: a +# stack buffer overflow that aborted the engine at startup. +longline + item1: n=1 + item2: n=2 + item3: n=3 + item4: n=4 + item5: n=5 + item6: n=6 + item7: n=7 + item8: n=8 + item9: n=9 + item10: note="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + item11: n=11 + item12: n=12 + item13: n=13 + item14: n=14 + item15: n=15 + item16: n=16 + item17: n=17 + item18: n=18 + item19: n=19 + item20: note="yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" + item21: n=21 + item22: n=22 + item23: n=23 + item24: n=24 + item25: n=25 + item26: n=26 + item27: n=27 + item28: n=28 + item29: n=29 + item30: n=30 diff --git a/.github/workflows/tests/kbb-long-value/kb/user/longvalue.dict b/.github/workflows/tests/kbb-long-value/kb/user/longvalue.dict new file mode 100644 index 00000000..7d9e1d86 --- /dev/null +++ b/.github/workflows/tests/kbb-long-value/kb/user/longvalue.dict @@ -0,0 +1,5 @@ +# Same for .dict: parseDictLine's 512-char buffers overflowed on a long +# quoted value, and readDict skipped any line of 2,048+ chars. +longword note="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +bigword note="yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" +afterword pos=noun diff --git a/.github/workflows/tests/kbb-long-value/spec/analyzer.seq b/.github/workflows/tests/kbb-long-value/spec/analyzer.seq new file mode 100644 index 00000000..ace79164 --- /dev/null +++ b/.github/workflows/tests/kbb-long-value/spec/analyzer.seq @@ -0,0 +1,2 @@ +tokenize nil # tokenize +nlp probe # measure what loaded diff --git a/.github/workflows/tests/kbb-long-value/spec/probe.nlp b/.github/workflows/tests/kbb-long-value/spec/probe.nlp new file mode 100644 index 00000000..aa53133c --- /dev/null +++ b/.github/workflows/tests/kbb-long-value/spec/probe.nlp @@ -0,0 +1,24 @@ +############################################### +# FILE: probe.nlp +# SUBJ: Measure the long values longline.kbb and longvalue.dict loaded. +# NOTE: Expected out.txt: 30|item30|600|3000|600|3000|noun +############################################### + +@CODE +L("root") = findconcept(findroot(),"longline"); +L("n") = 0; +L("c") = down(L("root")); +while (L("c")) { + L("n")++; + L("last") = conceptname(L("c")); + L("c") = next(L("c")); +} +L("k10") = strlength(strval(findconcept(L("root"),"item10"),"note")); +L("k20") = strlength(strval(findconcept(L("root"),"item20"),"note")); +L("d1") = strlength(strval(dictfindword("longword"),"note")); +L("d2") = strlength(strval(dictfindword("bigword"),"note")); +L("pos") = strval(dictfindword("afterword"),"pos"); +"out.txt" << str(L("n")) << "|" << L("last") << "|" << str(L("k10")) << "|" + << str(L("k20")) << "|" << str(L("d1")) << "|" << str(L("d2")) << "|" + << L("pos") << "\n"; +@@CODE diff --git a/cs/libconsh/cg.cpp b/cs/libconsh/cg.cpp index 79209a87..f91b89ed 100644 --- a/cs/libconsh/cg.cpp +++ b/cs/libconsh/cg.cpp @@ -3072,28 +3072,30 @@ f_phr << std::endl << _T("quit") << std::endl << std::endl; f_hier.close(); f_word.close(); f_phr.close(); -_TCHAR buf[MAXMSG]; +// Any length of line: a MAXMSG buffer cut the dump off at the first +// line of 2048+ chars, eg a long attribute value. +std::string buf; std::_t_ofstream *f_dump = new std::_t_ofstream(TCHAR2A(file)); // Final output file. std::_t_ifstream *f_in = new std::_t_ifstream(TCHAR2A(o_hier)); -while (f_in->getline(buf, MAXMSG)) // Copy first file. +while (std::getline(*f_in, buf)) // Copy first file. *f_dump << buf << std::endl; delete f_in; *f_dump << std::endl; // Separate commands. f_in = new std::_t_ifstream(TCHAR2A(o_word)); -while (f_in->getline(buf, MAXMSG)) // Copy first file. +while (std::getline(*f_in, buf)) // Copy first file. *f_dump << buf << std::endl; delete f_in; *f_dump << std::endl; // Separate commands. f_in = new std::_t_ifstream(TCHAR2A(o_attr)); -while (f_in->getline(buf, MAXMSG)) // Copy first file. +while (std::getline(*f_in, buf)) // Copy first file. *f_dump << buf << std::endl; delete f_in; *f_dump << std::endl; // Separate commands. f_in = new std::_t_ifstream(TCHAR2A(o_phr)); -while (f_in->getline(buf, MAXMSG)) // Copy first file. +while (std::getline(*f_in, buf)) // Copy first file. *f_dump << buf << std::endl; delete f_in; @@ -3869,7 +3871,9 @@ void CG::setDictSource(bool on) { dictSource = on; } bool CG::getDictSource() { return dictSource; } bool CG::readDict(std::string file, std::vector kbfiles) { - _TCHAR buf[MAXMSG]; + // Any length of line. A MAXMSG buffer used to skip (with a cgerr + // message) any line of 2048+ chars; parseDictLine is no longer limited. + std::string line; int lineCount = 0; bool inBlock = false; // Open "/*" carried between lines. @@ -3877,25 +3881,9 @@ bool CG::readDict(std::string file, std::vector kbfiles) allDictStream_.open(file, std::ios::in); - for (;;) { - allDictStream_.getline(buf, MAXMSG); - if (allDictStream_.bad()) - break; // unrecoverable I/O error - if (allDictStream_.eof() && allDictStream_.gcount() == 0) - break; // clean end of file + while (std::getline(allDictStream_, line)) { + _TCHAR *buf = line.data(); lineCount++; - // A line longer than the buffer leaves failbit set (but not eof). - // Recover -- clear the error and discard the rest of the line -- so the - // remaining entries are still read, instead of getline() aborting the - // whole file and silently dropping every entry after an over-long line. - if (allDictStream_.fail() && !allDictStream_.eof()) { - allDictStream_.clear(); - allDictStream_.ignore((std::numeric_limits::max)(), '\n'); - sprintf(errout_, "%d 1 [dict line too long (limit %d chars), skipped - %s]", - lineCount, MAXMSG - 1, file.c_str()); - *cgerr << errout_ << std::endl; - continue; - } // Blank out /* ... */ before the line lexer sees it. A comment-only // line becomes all white and is dropped by the check below. // 07/31/26 DD. strip_block_comments(buf, inBlock, '#'); @@ -3927,9 +3915,11 @@ bool CG::readDict(std::string file, std::vector kbfiles) bool CG::parseDictLine(_TCHAR *buf, CONCEPT *ambigKB, const std::string &file, int lineCount) { bool dirty; CONCEPT *wordCon, *parentCon; - _TCHAR token[MAXSTR]; - _TCHAR attr[MAXSTR]; - _TCHAR val[MAXSTR]; + // Sized to the data: these were _TCHAR[MAXSTR], and a quoted value or word + // of 512+ chars was copied in unbounded, overflowing the stack. + std::string token; + std::string attr; + std::string val; DICT_CALL caller; caller.line = lineCount; @@ -4104,29 +4094,27 @@ bool CG::parseDictLine(_TCHAR *buf, CONCEPT *ambigKB, const std::string &file, i addValue = false; int len = lens[i]; int beg = begins[i]; - _tcsnccpy(token, &line[beg],len); - token[len] = '\0'; + token.assign(&line[beg],len); cc = token[0]; if (cc == '\\') { beg++; len--; - _tcsnccpy(token, &line[beg],len); - token[len] = '\0'; + token.assign(&line[beg],len); cc = token[0]; } // Tokens before the first token before the equals sign if (i < eqSign - 1) { - wordCon = kbm_->dict_get_word(token,dirty,caller); + wordCon = kbm_->dict_get_word(token.data(),dirty,caller); if (parentCon) { - parentCon = getConcept(parentCon,token); + parentCon = getConcept(parentCon,token.data()); isPhrase = true; } else { parentCon = wordCon; } // add ambigous concept to word concept if (ambigKB) { - CONCEPT *kbcon = findConcept(ambigKB,token); + CONCEPT *kbcon = findConcept(ambigKB,token.data()); if (kbcon) { CONCEPT *ambigCon = NULL; if (!findVal(wordCon,"meaning",ambigCon)) { @@ -4140,8 +4128,7 @@ bool CG::parseDictLine(_TCHAR *buf, CONCEPT *ambigKB, const std::string &file, i } else if (cc == '"') { int len = lens[i] - 2; - _tcsnccpy(val, &token[1], len); - val[len] = '\0'; + val.assign(token, 1, len > 0 ? len : 0); addValue = true; wasQuoted = true; @@ -4157,22 +4144,17 @@ bool CG::parseDictLine(_TCHAR *buf, CONCEPT *ambigKB, const std::string &file, i i++; vend = begins[i] + lens[i]; } - int vlen = vend - vbeg; - if (vlen >= MAXSTR) - vlen = MAXSTR - 1; - _tcsnccpy(val, &line[vbeg], vlen); - val[vlen] = '\0'; + val.assign(&line[vbeg], vend - vbeg); addValue = true; attrFlag = false; } else { - _tcsnccpy(attr, token, lens[i]); - attr[lens[i]] = '\0'; + attr.assign(token, 0, lens[i]); attrFlag = true; - if (!_tcsicmp(attr, _T("l"))) { + if (!_tcsicmp(attr.c_str(), _T("l"))) { suggestedLit = true; } - else if (!_tcsicmp(attr, _T("s"))) { + else if (!_tcsicmp(attr.c_str(), _T("s"))) { suggestedAttr = true; } } @@ -4180,12 +4162,12 @@ bool CG::parseDictLine(_TCHAR *buf, CONCEPT *ambigKB, const std::string &file, i if (addValue) { // A double-quoted value is always a string, even if it looks // numeric (at="123"): preserve the author's intent. // #385. - if (!wasQuoted && unicu::isNumeric(val)) { + if (!wasQuoted && unicu::isNumeric(val.data())) { long long vnum = 0; - unicu::strToLong(val,vnum); - addVal(parentCon,attr,vnum); + unicu::strToLong(val.data(),vnum); + addVal(parentCon,attr.data(),vnum); } else - addSval(parentCon,attr,val); + addSval(parentCon,attr.data(),val.data()); attrFlag = false; addValue = false; wasQuoted = false; @@ -4199,7 +4181,7 @@ bool CG::parseDictLine(_TCHAR *buf, CONCEPT *ambigKB, const std::string &file, i } if (isPhrase && suggestedLit) { - addSval(parentCon,_T("l"),val); + addSval(parentCon,_T("l"),val.data()); } else if (isPhrase && !suggestedAttr) { addSval(parentCon,_T("s"),_T("phrase")); } @@ -4261,7 +4243,10 @@ bool CG::readKBBs(std::vector files) { } bool CG::readKBB(std::string file) { - _TCHAR buf[MAXMSG]; + // Any length of line. This was getline() into a MAXMSG buffer: a line of + // 2048+ chars set failbit, peek() then returned EOF, and every line after + // it was silently dropped. + std::string line; std::ifstream streamer; streamer.open(file, std::ios::in); @@ -4270,8 +4255,8 @@ bool CG::readKBB(std::string file) { int32_t index = 0; // Indentation state, carried between lines. bool inBlock = false; // Open "/*" carried between lines. - while (streamer.peek() != EOF) { - streamer.getline(buf,MAXMSG); + while (std::getline(streamer, line)) { + _TCHAR *buf = line.data(); // Blank out /* ... */ before the line lexer sees it. // 07/31/26 DD. // Only a line the stripper actually touched is dropped when it comes // back all white: parseKBBLine reads leading spaces as indentation and @@ -4283,7 +4268,7 @@ bool CG::readKBB(std::string file) { parseKBBLine(buf, cons, index, findRoot()); } - allDictStream_.close(); + streamer.close(); if (inBlock) { // Every entry after the "/*" was blanked out -- report rather than @@ -4302,15 +4287,22 @@ void CG::parseKBBLine(_TCHAR *buf, std::vector &cons, int32_t &index, bool found; CONCEPT *con = 0, *parent; std::vector> conIndices; - _TCHAR word[MAXSTR]; - _TCHAR attr[MAXSTR]; - _TCHAR val[MAXSTR]; + // Sized to the data: these were _TCHAR[MAXSTR], and a concept name, + // attribute or value of 512+ chars was copied in unbounded. + std::string word; + std::string attr; + std::string val; int32_t indent = 0; int32_t lastIndent = 0; bool attrFlag = false; bool openSquare = false; bool openDouble = false; bool collectingConcept = false; + // A quoted value was just stored. The empty text between its closing + // quote and the next ',', ']' or space is not another value: it used + // to be stored as the number 0 after every quoted value but the last + // on a line (["a","b"] loaded as a, 0, b, 0). + bool quoted = false; icu::StringPiece sp(buf); const char *line = sp.data(); @@ -4325,6 +4317,11 @@ void CG::parseKBBLine(_TCHAR *buf, std::vector &cons, int32_t &index, found = false; U8_NEXT(line, end, length, c); + // Copy n bytes of the line from start (none if n < 1). + auto take = [&](std::string &s, int32_t n) { + s.assign(&line[start], n > 0 ? n : 0); + }; + // Get indent int spaces = 0; while (c) { @@ -4366,14 +4363,16 @@ void CG::parseKBBLine(_TCHAR *buf, std::vector &cons, int32_t &index, // last one (which EOL terminates) was silently dropped. else if (conceptDone && attrFlag && !openDouble && !openSquare && unicu::isWhiteSpace(c)) { - _tcsnccpy(val, &line[start], end-start-1); - val[end-start-1] = '\0'; - if (unicu::isNumeric(val)) { + take(val, end-start-1); + if (quoted) + ; + else if (unicu::isNumeric(val.data())) { long long vnum = 0; - unicu::strToLong(val, vnum); - addVal(con, attr, vnum); + unicu::strToLong(val.data(), vnum); + addVal(con, attr.data(), vnum); } else - addSval(con, attr, val); + addSval(con, attr.data(), val.data()); + quoted = false; start = end; attrFlag = false; openDouble = false; @@ -4389,7 +4388,7 @@ void CG::parseKBBLine(_TCHAR *buf, std::vector &cons, int32_t &index, } else if (conceptDone && openSquare && collectingConcept && (c == ']' || c ==',')) { CONCEPT *conPath = addConceptByPath(line,conIndices); - addVal(con,attr,conPath); + addVal(con,attr.data(),conPath); start = end; if (c != ',') { openDouble = false; @@ -4400,14 +4399,14 @@ void CG::parseKBBLine(_TCHAR *buf, std::vector &cons, int32_t &index, conIndices.clear(); } else if (conceptDone && openSquare && openDouble && c == '"') { - _tcsnccpy(val, &line[start],end-start-1); - val[end-start-1] = '\0'; - if (!_tcscmp(val, _T("concept")) || collectingConcept) { + take(val, end-start-1); + if (val == _T("concept") || collectingConcept) { collectingConcept = true; conIndices.push_back(std::make_pair(start,end-1)); } else { start = end; - addSval(con,attr,val); + addSval(con,attr.data(),val.data()); + quoted = true; } start = end; // Add to concept path @@ -4435,19 +4434,21 @@ void CG::parseKBBLine(_TCHAR *buf, std::vector &cons, int32_t &index, (!openSquare && !openDouble && c == ',') || (openDouble && c == '"') || end == length)) { - _tcsnccpy(val, &line[start],end-start); int adjust = length && c != ']' && c != '"' && c != ',' ? 0 : 1; - val[end-start-adjust] = '\0'; + take(val, end-start-adjust); // An unquoted numeric value is stored as a number; a quoted value // (openDouble) is always a string. This is what lets a KBB round- // trip num/float vs string, instead of everything being a string. // -- #439. - if (!openDouble && unicu::isNumeric(val)) { + if (quoted && !openDouble) + ; + else if (!openDouble && unicu::isNumeric(val.data())) { long long vnum = 0; - unicu::strToLong(val,vnum); - addVal(con,attr,vnum); + unicu::strToLong(val.data(),vnum); + addVal(con,attr.data(),vnum); } else - addSval(con,attr,val); + addSval(con,attr.data(),val.data()); + quoted = openDouble; start = end; if (end >= length) { conceptDone = false; @@ -4464,23 +4465,22 @@ void CG::parseKBBLine(_TCHAR *buf, std::vector &cons, int32_t &index, openDouble = false; } else if (conceptDone && c == '=') { - _tcsnccpy(attr, &line[start],end-start-1); int adjust = openDouble ? 2 : 1; // This is in case the attribute was double quoted - attr[end-start-adjust] = '\0'; + take(attr, end-start-adjust); openDouble = false; + quoted = false; start = end; attrFlag = true; } else if (!conceptDone && (c == ':' || end == length || (openDouble && !attrFlag && c == '"'))) { - _tcsnccpy(word, &line[start],end-start); int adjust = (c == ':' || c == '"') ? 1 : 0; openDouble = false; - word[end-start-adjust] = '\0'; + take(word, end-start-adjust); if (index == 0) parent = root; else parent = cons[index-1]; - con = makeConcept(parent,word); + con = makeConcept(parent,word.data()); // Backup if indent less than last for (int j=lastIndent; j && j>=index; j--) cons.pop_back(); @@ -4499,12 +4499,13 @@ void CG::parseKBBLine(_TCHAR *buf, std::vector &cons, int32_t &index, CONCEPT *CG::addConceptByPath(const char *line, std::vector> conIndices) { CONCEPT *con = findRoot(); int i = 0; - _TCHAR word[MAXMSG]; + std::string word; for (auto indice: conIndices) { - _tcsnccpy(word, &line[indice.first],indice.second); - word[indice.second-indice.first] = '\0'; + // indice.second is an end offset, not a length. Copying that many + // bytes into a MAXMSG buffer overflowed it on a long line. + word.assign(&line[indice.first], indice.second - indice.first); if (i++) - con = makeConcept(con,word); + con = makeConcept(con,word.data()); } return con; } @@ -4518,14 +4519,16 @@ CONCEPT *CG::addConceptByPath(const char *line, std::vector> * if it is not, we warn and fall back to loading it the normal way. ********************************************/ -// Trim a trailing CR and/or LF left by getline on a CRLF file opened in binary -// mode. Returns the new length. -static int32_t chompCRLF(_TCHAR *buf) +// Read one whole line of any length into str, CR/LF trimmed. The lazy loaders +// used getline() into a MAXMSG buffer, which failed on a 2048+ char line. +// false at end of file, like getline(). +static bool readFullLine(std::istream &in, std::string &str) { - int32_t n = (int32_t) _tcslen(buf); - while (n > 0 && (buf[n-1] == '\r' || buf[n-1] == '\n')) - buf[--n] = '\0'; - return n; + if (!std::getline(in, str)) + return false; + while (!str.empty() && (str.back() == '\r' || str.back() == '\n')) + str.pop_back(); + return true; } // True if the file stem ends in the word "full" (eg "en-full", or just "full"). @@ -4604,15 +4607,14 @@ void CG::setMissingLogDir(const _TCHAR *dir) // byte offset just after it. false at end of file. bool CG::nextKBBKey(FullFile &f, std::streamoff &keyStart, std::string &key, std::streamoff &afterKey, bool *multiLine) { - _TCHAR line[MAXMSG]; + std::string str; while (true) { std::streamoff pos = f.stream.tellg(); if (pos < 0) return false; - f.stream.getline(line, MAXMSG); - if (f.stream.fail()) + if (!readFullLine(f.stream, str)) return false; - chompCRLF(line); + _TCHAR *line = str.data(); // Single-line "/* ... */" only: this file is binary-searched, so a // comment that runs past its own line cannot be recognized from an // arbitrary seek. Report it and let the caller reject the file. @@ -4649,15 +4651,14 @@ bool CG::nextKBBKey(FullFile &f, std::streamoff &keyStart, std::string &key, std // open *full.dict and return its first token (the word). bool CG::nextDictKey(FullFile &f, std::streamoff &keyStart, std::string &key, std::streamoff &afterKey, bool *multiLine) { - _TCHAR line[MAXMSG]; + std::string str; while (true) { std::streamoff pos = f.stream.tellg(); if (pos < 0) return false; - f.stream.getline(line, MAXMSG); - if (f.stream.fail()) + if (!readFullLine(f.stream, str)) return false; - chompCRLF(line); + _TCHAR *line = str.data(); // Single-line "/* ... */" only -- see nextKBBKey. bool inBlock = false; if (strip_block_comments(line, inBlock, '#') && inBlock && multiLine) @@ -4774,30 +4775,25 @@ bool CG::openFullKBB(const std::string &file, bool merge) // "en-roots-full.kbb" has its own top concept ("roots"), and its words must // hang off that so the analyzer can find it (getconcept(findroot(),"roots")). // This also creates that concept up front, before any word is looked up. - _TCHAR rootName[MAXMSG]; - rootName[0] = '\0'; + std::string rootName; f.stream.clear(); f.stream.seekg(0); { - _TCHAR line[MAXMSG]; - while (f.stream.getline(line, MAXMSG)) { - chompCRLF(line); + std::string str; + while (readFullLine(f.stream, str)) { + _TCHAR *line = str.data(); bool inBlock = false; // Single-line "/* */" only. strip_block_comments(line, inBlock, '#'); if (line[0] == '#' || line[0] == '\0' || line[0] == ' ') continue; // comment, blank, or indented line - int i = 0; - while (line[i] && line[i] != ':') { - rootName[i] = line[i]; - i++; - } - rootName[i] = '\0'; + for (int i = 0; line[i] && line[i] != ':'; i++) + rootName += line[i]; break; } } - if (!rootName[0]) - strcpy(rootName, _T("dictionary")); // no top concept found; fall back - f.root = getConcept(findRoot(), rootName); + if (rootName.empty()) + rootName = _T("dictionary"); // no top concept found; fall back + f.root = getConcept(findRoot(), rootName.data()); std::_t_cerr << _T("[Lazy-loading words from ") << file << _T(" under \"") << rootName << _T("\"]") << std::endl; @@ -4889,7 +4885,8 @@ CONCEPT *CG::findFullKBBWord(_TCHAR *str) // (the word line plus its indented senses) into the KB, cache it, and return it. CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) { - _TCHAR line[MAXMSG]; + std::string buf; + _TCHAR *line; std::streamoff lo = 0, hi = f.size; std::streamoff keyStart, afterKey; std::string key; @@ -4900,7 +4897,7 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) f.stream.clear(); f.stream.seekg(mid); if (mid != 0) - f.stream.getline(line, MAXMSG); // Align to a line start. + std::getline(f.stream, buf); // Align to a line start. if (!nextKBBKey(f, keyStart, key, afterKey)) { hi = mid; continue; @@ -4934,8 +4931,8 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) int32_t index = 0; bool inBlock = false; // Single-line "/* */" only. - f.stream.getline(line, MAXMSG); // The word line. - chompCRLF(line); + readFullLine(f.stream, buf); // The word line. + line = buf.data(); strip_block_comments(line, inBlock, '#'); parseKBBLine(line, cons, index, f.root); CONCEPT *word = cons.size() > 1 ? cons[1] : 0; @@ -4943,10 +4940,9 @@ CONCEPT *CG::searchKBBFile(FullFile &f, _TCHAR *str) while (true) { if (f.stream.peek() == EOF) break; - f.stream.getline(line, MAXMSG); - if (f.stream.fail()) + if (!readFullLine(f.stream, buf)) break; - chompCRLF(line); + line = buf.data(); // A comment line inside the word's block must not look like the blank // line that ends it, so skip it rather than falling into the test // below. Lines the stripper left alone keep their old meaning. @@ -4983,7 +4979,7 @@ CONCEPT *CG::findFullDictWord(_TCHAR *str) // words are routed through dict_get_word, so they are findable/cached for free. CONCEPT *CG::searchDictFile(FullFile &f, _TCHAR *str) { - _TCHAR line[MAXMSG]; + std::string buf; std::streamoff lo = 0, hi = f.size; std::streamoff keyStart, afterKey; std::string key; @@ -4993,7 +4989,7 @@ CONCEPT *CG::searchDictFile(FullFile &f, _TCHAR *str) f.stream.clear(); f.stream.seekg(mid); if (mid != 0) - f.stream.getline(line, MAXMSG); + std::getline(f.stream, buf); if (!nextDictKey(f, keyStart, key, afterKey)) { hi = mid; continue; @@ -5024,10 +5020,9 @@ CONCEPT *CG::searchDictFile(FullFile &f, _TCHAR *str) while (true) { if (f.stream.peek() == EOF) break; - f.stream.getline(line, MAXMSG); - if (f.stream.fail()) + if (!readFullLine(f.stream, buf)) break; - chompCRLF(line); + _TCHAR *line = buf.data(); bool inBlock = false; // Single-line "/* */" only. strip_block_comments(line, inBlock, '#'); int i = 0; diff --git a/include/Api/lite/code.h b/include/Api/lite/code.h index 0911eea0..8211da1f 100644 --- a/include/Api/lite/code.h +++ b/include/Api/lite/code.h @@ -57,7 +57,7 @@ class LITE_API Code bool &exitpass); // 02/06/01 AM. static void execPostcode(Parse *parse, Seqn *seqn); // @POSTCODE (#392). static void execActions(Parse *parse, Dlist *codes, - bool &exitpass); + bool &exitpass, const _TCHAR *region); static bool codeAction(Iaction *action, Parse *parse, /*UP*/ bool &badname); // 01/14/00 AM. diff --git a/include/Api/lite/parse.h b/include/Api/lite/parse.h index 23ac4911..6748d41b 100644 --- a/include/Api/lite/parse.h +++ b/include/Api/lite/parse.h @@ -20,6 +20,7 @@ All rights reserved. #include // 09/27/19 AM. #endif #include +#include typedef void TREE; // 07/07/03 AM. typedef void LTREE; // 07/07/03 AM. @@ -227,6 +228,8 @@ class LITE_API Parse void prettySeq(); // Print analyzer sequence. bool errOut(bool,bool=false); // 08/24/02 AM. bool errOut(std::_t_strstream*,bool,bool=false); // 02/25/05 AM. + // Echo an NLP++ error to stderr: region ("@CODE"), what it skipped. + void nlpppError(const _TCHAR *region, const _TCHAR *effect); inline void prettyPassnum(int, /*UP*/ _TCHAR *); inline void prettyParsedone(); @@ -649,6 +652,9 @@ class LITE_API Parse // Line number of input file being processed. // 08/24/02 AM. long inputline_; // 08/24/02 AM. + // (pass, line) of NLP++ errors already echoed to stderr for this input. + std::set> nlpppErrs_; + // Pass info to analyzer, to G("$datum") variable. // 03/13/03 AM. _TCHAR *datum_; // 03/13/03 AM. diff --git a/lite/check.cpp b/lite/check.cpp index 57e826fb..e36ead83 100644 --- a/lite/check.cpp +++ b/lite/check.cpp @@ -154,6 +154,7 @@ for (dcheck = checks->getFirst(); dcheck; dcheck = dcheck->Right()) std::_t_strstream gerrStr; gerrStr << _T("[Check: Error in NLP++.]") << std::ends; nlppp->parse_->errOut(&gerrStr,false); + nlppp->parse_->nlpppError(_T("@CHECK"), _T("rest of the check skipped")); } // 11/22/99 AM. if (val) diff --git a/lite/code.cpp b/lite/code.cpp index 8de757e3..683359dd 100644 --- a/lite/code.cpp +++ b/lite/code.cpp @@ -156,7 +156,7 @@ if (!(codes = rulesfile->getCodes())) return; } -execActions(parse, codes, exitpass); +execActions(parse, codes, exitpass, _T("@CODE")); parse->setCodeexit(exitpass); // If @CODE exits the pass, skip @POSTCODE. } @@ -176,7 +176,7 @@ Dlist *codes = rulesfile ? rulesfile->getPostcodes() : 0; if (!codes) return; bool exitpass = false; -execActions(parse, codes, exitpass); +execActions(parse, codes, exitpass, _T("@POSTCODE")); } @@ -185,7 +185,8 @@ execActions(parse, codes, exitpass); * SUBJ: Execute the actions of a @CODE or @POSTCODE region. ********************************************/ -void Code::execActions(Parse *parse, Dlist *codes, bool &exitpass) +void Code::execActions(Parse *parse, Dlist *codes, bool &exitpass, + const _TCHAR *region) { bool ok = true; // 01/14/00 AM. exitpass = false; @@ -220,6 +221,7 @@ for (daction = codes->getFirst(); daction; daction = daction->Right()) std::_t_strstream gerrStr; gerrStr << _T("[Code actions: Error in NLP++.]") << std::ends; parse->errOut(&gerrStr,false); + parse->nlpppError(region, _T("rest of the region skipped")); } if (val) diff --git a/lite/parse.cpp b/lite/parse.cpp index 8eaac864..630f0d96 100644 --- a/lite/parse.cpp +++ b/lite/parse.cpp @@ -1640,6 +1640,41 @@ else return ::errOut(st,retval,inputpass_,inputline_); } + +/******************************************** +* FN: NLPPPERROR +* SUBJ: Echo an NLP++ runtime error to stderr. +* NOTE: The error trail goes to err.log only, so a region that stopped +* on an error (eg findconcept(0,...)) looked like a clean run: exit 0 +* and nothing on the console. Once per input for each pass and line, +* so an error in a rule that fires on every match prints one line. +********************************************/ + +void Parse::nlpppError(const _TCHAR *region, const _TCHAR *effect) +{ +if (!nlpppErrs_.insert(std::make_pair(currpass_, line_)).second) + return; + +// currpass_ is the pass the failing code is in, which for a function call +// is the function's pass rather than the one running (seq_). +_TCHAR *file = 0; +Dlist *seqs = ana ? ana->getSeqlist() : 0; +for (Delt *d = seqs ? seqs->getFirst() : 0; d; d = d->Right()) + if (d->getData()->getPassnum() == currpass_) + { + file = d->getData()->getRulesfilename(); + break; + } + +std::_t_cerr << _T("[Error in NLP++ ") << region << _T(": pass ") << currpass_; +if (file && *file) + std::_t_cerr << _T(" (") << file << _T(")"); +std::_t_cerr << _T(" line ") << line_ << _T(", ") << effect; +if (input && *input) + std::_t_cerr << _T(". Input: ") << input; +std::_t_cerr << _T(". Details in err.log.]") << std::endl; +} + /******************************************** * FN: NEWBLOB * CR: 02/27/03 AM. diff --git a/lite/post.cpp b/lite/post.cpp index acf24f01..2b63c414 100644 --- a/lite/post.cpp +++ b/lite/post.cpp @@ -199,6 +199,7 @@ for (dpost = posts->getFirst(); dpost; dpost = dpost->Right()) std::_t_strstream gerrStr; gerrStr << _T("[Post actions: Error in NLP++.]") << std::ends; errOut(&gerrStr,false, parse->currpass_,parse->line_); // 03/05/03 AM. + parse->nlpppError(_T("@POST"), _T("rest of the pass skipped")); // errOut(false); ok = false; // FIX. // 12/18/03 AM. nlppp->exitpass_ = true; // FIX. // 12/18/03 AM. diff --git a/nlp/main.cpp b/nlp/main.cpp index cd3322d7..bad1b1ac 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,7 +15,7 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.37" +#define NLP_ENGINE_VERSION "4.2.38" bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &, bool &, _TCHAR *&); void cmdHelpargs(_TCHAR *);