From 9f98b8f265ed768611b3b3537d3320c42aee01d8 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Sun, 18 Oct 2026 13:13:23 +0300 Subject: [PATCH 1/2] ci(bench): the 200-table script is checked, a failed scenario run fails the job and says why (#1546) --- .github/workflows/benchmark.yml | 10 +++++ benchmark/app_perf_bench.dart | 40 ++++++++++++++++-- .../app_bench_report.cpython-314.pyc | Bin 0 -> 5216 bytes scripts/ci/app_bench_report.py | 15 ++++++- scripts/ci/test_app_bench_report.py | 10 +++++ 5 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 scripts/ci/__pycache__/app_bench_report.cpython-314.pyc diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3c71f017..a14dc034 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -137,3 +137,13 @@ jobs: python3 scripts/ci/app_bench_report.py "$RUNNER_TEMP/app.log" > "$RUNNER_TEMP/app_report.txt" cat "$RUNNER_TEMP/app_report.txt" tail -n +2 "$RUNNER_TEMP/app_report.txt" >> "$GITHUB_STEP_SUMMARY" + + # `flutter run` exits 0 whatever the app does; an aborted scenario run + # prints `BENCH FAILED`, so the job turns red instead of reporting only the + # scenarios that ran. The job stays informational (continue-on-error). + - name: Fail when the run aborted + run: | + if grep -q 'BENCH FAILED' "$RUNNER_TEMP/app.log"; then + grep -A3 'BENCH FAILED' "$RUNNER_TEMP/app.log" | head -n 8 + exit 1 + fi diff --git a/benchmark/app_perf_bench.dart b/benchmark/app_perf_bench.dart index 840de99f..8ed6fab9 100644 --- a/benchmark/app_perf_bench.dart +++ b/benchmark/app_perf_bench.dart @@ -81,13 +81,17 @@ Future main() async { runApp(const QueryaApp()); WidgetsBinding.instance.addPostFrameCallback((_) async { + var failed = false; try { await _runAll(); } catch (e, st) { - stdout.writeln('BENCH error: $e\n$st'); + // `flutter run` reports 0 whatever the app exits with, so the CI step + // looks for this line as well (scripts/ci/app_bench_report.py). + stdout.writeln('BENCH FAILED: $e\n$st'); + failed = true; } await dataDir.delete(recursive: true).catchError((_) => dataDir); - exit(0); + exit(failed ? 1 : 0); }); } @@ -257,10 +261,22 @@ Future _runAll() async { SqlEditorCommandBridge.instance.invokeNew(); await _wait(300); await _runInActiveTab(ddl.toString()); + // The script must have run: a failed or half-run script leaves the diagram + // of the demo's own tables, which looks like a slow ERD. + final ddlSession = ws.activeSession; + stdout.writeln('BENCH erd_ddl status=${ddlSession.statusLine} ' + 'error=${ddlSession.error}'); + if (ddlSession.error != null) { + throw StateError('the 200-table script failed: ${ddlSession.error}'); + } await _scenario('erd_open_200', () async { await _tap(_byKey('open_diagram_tab')); - await _waitFor(() => _find((e) => e.widget.key == const ValueKey('erd_table_erd_t199')) != null, - 'diagram of 200 tables'); + await _waitFor(() => ws.activeSession.isDiagram as bool, 'the Diagram tab', + onTimeout: () => _dumpErdState(ws)); + await _waitFor( + () => _find((e) => e.widget.key == const ValueKey('erd_table_erd_t199')) != null, + 'diagram of 200 tables', + onTimeout: () => _dumpErdState(ws)); await _wait(800); }); await _scenario('erd_hover', () => _hoverOver(_find((e) => e.widget is ErdView)!, 3000)); @@ -271,6 +287,22 @@ Future _runAll() async { }); } +/// What the screen shows when the diagram did not come up: the active tab and +/// how many table cards exist, so a failed run says which of the two it was. +void _dumpErdState(dynamic ws) { + var cards = 0; + void visit(Element e) { + final k = e.widget.key; + if (k is ValueKey && k.value.startsWith('erd_table_')) cards++; + e.visitChildren(visit); + } + + WidgetsBinding.instance.rootElement!.visitChildren(visit); + final session = ws.activeSession; + stdout.writeln('BENCH erd_state activeTab="${session.title}" ' + 'isDiagram=${session.isDiagram} cards=$cards'); +} + /// The SQL workspace of the open connection. dynamic _workspace() => _findState((s) => s.runtimeType.toString() == 'GenericSqlWorkspaceState')!; diff --git a/scripts/ci/__pycache__/app_bench_report.cpython-314.pyc b/scripts/ci/__pycache__/app_bench_report.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bbf27aa4839b3a6d209032ca02012309fdc552f4 GIT binary patch literal 5216 zcma)AUr-yz8Q(jd{vwG#!WaxTiy=1RkH8o^nAq63;Dj0oIf&vk0&z$uP>gi)-N_i0 zq-v*=5zcr}>ZDQQc*W;&greM;Jg_N_|ow93^p$)tJ7Q;F?1Gwn;i-ID~e zai*)ez3+Cv{dV`;@6T`5?sV7?lt0|K^K!Ejp?{E%LV<5u?mHQTl88qtnn64>!l(>= zvntE8yn*N7F~Vcw&Af%T@;0E$cypIoHOw$7H_c4YISOL+cN606y&#tq)$~{%N0G<; zSRNY0+F?4noTJd5y5>^$D%@$XLd9k04rUU)DL{DrP4^B+@&km1&=NZqUG;g!9x z7FZAi7ZofDJ620JGCDFgGBw#>kaVKExC2`*WJ0n zkWc<@LRG!hZiuoM**;anG&gk?~&F3n>b*UAD5f~+e3W8K}|^G8Yfb@v&~IWJ4` z7_43I&>ST?)$F1i^h+@@>hFP3x^a%2(X6C-foNg=ou83{&uA9X7LS+aqknfu6S0 zib@r%oTBq34(rP)*=ebzOLBA|DXg#35qCx*bu2&;vl;?m95oP3 z&Q)hxy=`eo4t`eOkUElj?pp6!|7!p1C$rq4^%kh^11l=gG?7AddjBUhHFTjE^Z#nQ z0-Z}B6k~bhD{9rt&agZ6T{sz@@tMioqyUd$S01*6huf9MP{L!}m4_?gG40A@ELpR4 zH=Zuj3st4oa|l*{egz>bI)Ho@x*4b@$_mu>dxuRLUUU_0vYOcBj+r5??l$vL7HrW( zYPVT4>29rAsWbb*Ig8|pA_;jxv+3?jIV=U&9Cw>EqknW{d|0!D!qH$D0-|Ps$f+4a zi;|#fCjZ3X`0%7mHsR)E2fQ?EF(S(ZI%q~3?iCK`Ll|SZJl`lQ5J!RTghzP~swB!) zR3%65I;z%YR%bFpe|0pan9uFiZw4|=nZcVa*TXsc-goz8F0EJIRBzOO+ehuYPpmWR zP49Cz+H>}rcMX}rjBs=4hIPwfuD7ME>A|;JUN8UJfSd;(R3V3JZESTc({kJ1m>kZP zRsYiXrjj`Zlk{DGC0o{)<@z>y^|xFJ%PjGL8fc&%unC?Dv8EV@w$E@0dgQ7Z6f<}b z-|#FG<9x)cY5BH@0v@u75FPFHG807)D2xSPFcv%{X7-vxPtce}tb&|FzQhb(Zo*pl z+J5t_*Jww)!E53<2jY!AMOlsL`~b){^ChyjJykuff(~Kd zztg?b%ZkUSE?Hj@vOaG2Jj1)DSddh`%?n?9ybsS<12Zzvv$x0WVM}z#`HJVYZ8b5g z*IHaN#97Ej9L!bgwStVg9>c>jXz#v9S`B>t1aY{BikBaE!r(a)O0c*!!4V-c zaUOXusUY2=xCyE+s2GivE&|_5;y6ujr7&OzV8r1l#g2e1L*dIJ2GFVWYht*s5W zQdEsK#vLcVgY%*pPrR-;5e1wC9Om;PMrBHZL%wyzuGzcTETn^3QrLYRvUka&9f=oaG>7j_U1Q&MTD8-a>ixPq0 z0*(NJ6FnMZb90HOnwy*PARtYe$|G>o=*PqSPzTn5BSqaswAg_ueCh-};K}Dkg$ zW~7%FjU!N-e83e-V?gkn5QMY`I7bXB#C=K9(TkpXOgcoDk3-){c$D`jc)7c`Av1OT z`yX@F$&p-{JvnrbH9Abm;Rkk9d*JPrH&%Z7>S|ev%h~r7vqg#l#;YfC_KNi2wUN|N z&R)4Tx;mQa$pkarZ1akIeR9Qt((7NU-;BsopY9_%dSN;@sFJ?DMQX;zeYq_ z9BcMfd(K&ru1=fMB-=f6vputaJ#ee(*7;v}HeFfwsm;k;c}2SZ+N+sB)_HLK;QI5o zj;xQ|n%ZdEIR6{Zhpz0A;SVWG!?hn|l&rIP{pk9oTl?OBWnJ2s{NvvN--?uFW}JcWMrQ>^!`4Vd;Zgh^ed3)#J?J^?i3b zPG#E%HibV=eq_yhFJw=@n0@hkTLxyYj{&x-Y35&uu}(87N3Y} zS2ktKnzCHeqq&!OHO;-X^Z@HivTrFig2)MfQ9Mbwp*+R-nK&Ca_)!796YRg+?V#At z!rvS4>&;7OM80B9y}%Y?Db3Zrz9ORIO;ZLt>M8casAAt#F#^Ni30yP6sGS0Gvc7r1 z2Ms99HlqEgk^V;eo-*u5VFn`~!BMpMNW7%wC~HDN_-X)N-WX%|qtS^l(nq+>UlB}_ zFvitTXP;&U<6jg4BHS@7@Xv%4gm9#pL|K+(Wn7OJMtao-F`Noqk|JRka!J zdczODn;g0e7BmcFYEqs%TpjspdF}i+4q$`f|NRo6pE+BGAAng^<;p9sa$u{*s{bei za{g%l;8FDR%0X=0umJTDE*tE&e$>tpy4yhNqpq`7!|yE&)SAWbC(HKx-4$?c^9QAX z->OSQxDc4% frames=.. build p50/p90/max=.. raster ..` lines and checks each scenario of LIMITS: build p90 against its limit (ms), and for multi_tab_run the number of query panes rebuilt by one run. The first output -line is `OVER=`, the number of scenarios over a limit or missing. -Informational: always exits 0. +line is `OVER=`, the number of scenarios over a limit or missing. A run that +aborted prints `BENCH FAILED: ` (`flutter run` exits 0 whatever the +app does), which the report names. Informational: always exits 0. """ import re import sys @@ -32,6 +33,13 @@ r"build p50/p90/max=([\d.]+)/([\d.]+)/([\d.]+)\s+" r"raster p50/p90/max=([\d.]+)/([\d.]+)/([\d.]+).*?stutters=(\d+)") _PANES = re.compile(r"BENCH multi_tab_run panes rebuilt=(\d+)") +_FAILED = re.compile(r"BENCH FAILED: (.*)$", re.M) + + +def failure(text): + """The reason the bench aborted, or None.""" + m = _FAILED.search(text) + return m[1].strip() if m else None def parse(text): @@ -75,6 +83,9 @@ def report(text): over += bad lines += ["", f"Query panes rebuilt by one run in one of five tabs: " f"**{panes}** (limit {MAX_PANES_REBUILT}){' ⚠️' if bad else ''}."] + failed = failure(text) + if failed: + lines += ["", f"⚠️ The run aborted: `{failed}`. Scenarios after it did not run."] lines += ["", "Times in ms, profile build under xvfb with software " "rendering. Informational, never blocks a merge."] return over, "\n".join(lines) diff --git a/scripts/ci/test_app_bench_report.py b/scripts/ci/test_app_bench_report.py index 5485b84a..93fb4b3a 100644 --- a/scripts/ci/test_app_bench_report.py +++ b/scripts/ci/test_app_bench_report.py @@ -64,6 +64,16 @@ def test_rebuilding_every_pane_is_flagged(self): self.assertEqual(over, 1) self.assertIn("**5**", body) + def test_an_aborted_run_is_named(self): + text = line("charts_5000") + "\nBENCH FAILED: Bad state: timeout waiting for the Diagram tab\nstack\n" + self.assertEqual(abr.failure(text), "Bad state: timeout waiting for the Diagram tab") + _, body = abr.report(text) + self.assertIn("The run aborted: `Bad state: timeout waiting for the Diagram tab`", body) + + def test_a_clean_run_has_no_failure(self): + self.assertIsNone(abr.failure(all_ok())) + self.assertNotIn("aborted", abr.report(all_ok())[1]) + def test_main_prints_the_count_first(self): with tempfile.TemporaryDirectory() as d: p = os.path.join(d, "app.log") From 688527ba6466b1cdfb89f386adcbc5cc1cd510a0 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Sun, 18 Oct 2026 13:26:59 +0300 Subject: [PATCH 2/2] ci(bench): open the diagram through the workspace; the toolbar button is outside the window with seven tabs (#1546) --- benchmark/app_perf_bench.dart | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmark/app_perf_bench.dart b/benchmark/app_perf_bench.dart index 8ed6fab9..484e1baa 100644 --- a/benchmark/app_perf_bench.dart +++ b/benchmark/app_perf_bench.dart @@ -270,7 +270,10 @@ Future _runAll() async { throw StateError('the 200-table script failed: ${ddlSession.error}'); } await _scenario('erd_open_200', () async { - await _tap(_byKey('open_diagram_tab')); + // Not a tap on the toolbar button: with seven query tabs open the tab bar + // is full and the button is pushed out of the 1280x720 window, so the tap + // landed on a tab and the diagram never opened (#1546). + ws.openDiagramTab(); await _waitFor(() => ws.activeSession.isDiagram as bool, 'the Diagram tab', onTimeout: () => _dumpErdState(ws)); await _waitFor(