From 103de38eaa348e8bf2a6e282d40bbdfba49636aa Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 7 Jun 2026 20:38:31 +0530 Subject: [PATCH 1/4] Local LLM by ollama is working perfectly --- .coverage | Bin 53248 -> 69632 bytes docs/GLOSSARY.md | 4 +- docs/MCP.md | 64 ++- input.txt.example | 5 + pipeline-config.example.txt | 5 + src/website_profiling/analysis/image_probe.py | 142 +++++++ src/website_profiling/llm/agent.py | 43 +- src/website_profiling/llm/providers/ollama.py | 53 ++- src/website_profiling/mcp/server.py | 24 +- src/website_profiling/reporting/builder.py | 129 ++++++ .../tools/audit_tools/compare_slices.py | 96 +++++ .../tools/audit_tools/content.py | 34 ++ .../tools/audit_tools/crawl_lists.py | 281 +++++++++++++ .../tools/audit_tools/export_tools.py | 345 ++++++++++++++++ .../tools/audit_tools/image_tools.py | 368 ++++++++++++++++++ .../tools/audit_tools/lighthouse.py | 140 +++++++ .../tools/audit_tools/links.py | 41 ++ .../tools/audit_tools/llm_tools.py | 124 ++++++ .../tools/audit_tools/ops.py | 170 +++++++- .../tools/audit_tools/registry.py | 109 +++++- .../tools/audit_tools/report.py | 42 ++ .../tools/audit_tools/security.py | 46 +++ .../tools/audit_tools/tech.py | 48 ++- .../tools/audit_tools/tool_catalog.py | 113 ++++++ .../tools/export_artifacts.py | 215 ++++++++++ src/website_profiling/tools/export_compare.py | 53 +++ src/website_profiling/tools/export_custom.py | 248 ++++++++++++ tests/test_audit_tools_expanded.py | 139 ++++++- tests/test_builder_image_buckets.py | 55 +++ tests/test_config_schema_keys.py | 5 + tests/test_export_artifacts.py | 66 ++++ tests/test_export_compare.py | 32 ++ tests/test_export_custom.py | 50 +++ tests/test_image_probe.py | 203 ++++++++++ tests/test_image_tools.py | 149 +++++++ tests/test_mcp_registry.py | 2 +- tests/test_mcp_server_helpers.py | 4 +- tests/test_ollama_errors.py | 19 + web/app/api/chat/artifacts/[id]/route.ts | 97 +++++ web/app/client-providers.tsx | 2 + web/app/globals.css | 41 +- web/src/components/chat/ChatContextBar.tsx | 15 +- web/src/components/chat/ChatFab.tsx | 35 ++ web/src/components/chat/ChatMarkdown.tsx | 38 +- web/src/components/chat/ChatSidebar.tsx | 5 + web/src/components/chat/blocks/ChatBlocks.tsx | 15 + .../chat/blocks/ChatFileDownloadBlock.tsx | 53 +++ .../blocks/ChatImageAttentionTableBlock.tsx | 78 ++++ .../chat/blocks/ChatImageAuditBlock.tsx | 126 ++++++ .../chat/blocks/ChatImageLighthouseBlock.tsx | 44 +++ .../chat/blocks/ChatImagePagesTableBlock.tsx | 60 +++ .../components/chat/deriveChatBlocks.test.ts | 61 +++ web/src/components/chat/deriveChatBlocks.ts | 322 ++++++++++++++- .../components/chat/preprocessChatMarkdown.ts | 4 + .../components/chat/stripRedundantMarkdown.ts | 49 +++ .../components/links/tabs/TechnicalTab.tsx | 6 + web/src/lib/chatUrlState.test.ts | 16 + web/src/lib/chatUrlState.ts | 21 + web/src/lib/pipelineConfigSchema.ts | 11 + web/src/strings.json | 26 +- web/src/types/report.ts | 1 + web/src/views/Chat.tsx | 12 +- 62 files changed, 4697 insertions(+), 107 deletions(-) create mode 100644 src/website_profiling/analysis/image_probe.py create mode 100644 src/website_profiling/tools/audit_tools/crawl_lists.py create mode 100644 src/website_profiling/tools/audit_tools/export_tools.py create mode 100644 src/website_profiling/tools/audit_tools/image_tools.py create mode 100644 src/website_profiling/tools/audit_tools/llm_tools.py create mode 100644 src/website_profiling/tools/export_artifacts.py create mode 100644 src/website_profiling/tools/export_compare.py create mode 100644 src/website_profiling/tools/export_custom.py create mode 100644 tests/test_builder_image_buckets.py create mode 100644 tests/test_export_artifacts.py create mode 100644 tests/test_export_compare.py create mode 100644 tests/test_export_custom.py create mode 100644 tests/test_image_probe.py create mode 100644 tests/test_image_tools.py create mode 100644 tests/test_ollama_errors.py create mode 100644 web/app/api/chat/artifacts/[id]/route.ts create mode 100644 web/src/components/chat/ChatFab.tsx create mode 100644 web/src/components/chat/blocks/ChatFileDownloadBlock.tsx create mode 100644 web/src/components/chat/blocks/ChatImageAttentionTableBlock.tsx create mode 100644 web/src/components/chat/blocks/ChatImageAuditBlock.tsx create mode 100644 web/src/components/chat/blocks/ChatImageLighthouseBlock.tsx create mode 100644 web/src/components/chat/blocks/ChatImagePagesTableBlock.tsx diff --git a/.coverage b/.coverage index c3abd14654c8ee05d9a7d9e29af97552a1616a71..77631336c2ee31ad9dbe32cca9dde7047cb08966 100644 GIT binary patch literal 69632 zcmeHQ3v?URnI1_a$&yAhe#G%BiIs=#IB~2vX&W9(O$b?_zy>y?Y(j}Vl19=vl19wP zcHEK(QMx?#>|slH;Q;M%PnRyI^tD^IX(C1-X*6{8WM3%dk6s%^Ni!j^4=0-ZJ~E4uy>BFU-P=({lEX-|K9)Jxtgt4Um>R@Uqn$8V%oQ!sbwq{=0cy3VHg+u z*TH{zbHW$4yaUow_5EyLT}Gr|4W{q#7PgO>IDM|H3Pl)_HLglMI=>9h9s?+irwVlotyv>u;mXc$r;pS#mV zZB-@VPbzZLly1@$0{G|&L6rDboqiJ*d*@}))1Ok1=S9i9>;cRMav)t^` zi&4p1CE=V?gvypYgBvZMN=TwMq?R4YA99lAJ1SnXTt`Z0K%F&CN7Kf(v#C?MF3?x{ zQj>ME&ZfpjOSV`euwTOe_Bl%6Bb6#JuOax;v!4>vG2iKV$A=-~U3N#)`Hg4GIL<<# z4}UstskQ1^zRjkyp8%(i|7~-!jE_{Ze7Q8bW>}1aH%W=I3RM<-aIppO;S0keTr38a zp|oCp@vdGxaobbLZ!v+&ewll?s$0Jts`pijrpeH(;$}-buYAw9q!WeRBl|bo4*DsUAcb2mqbELhD+@gZib7g zg73$b6XO(}2;AuFf=a1D6`hx6InmNt?6Jz}_ojdooN7W07pprqN>z#@G3m7AimlPb zVp>Ye3CV{q(KDkLtPxT3g;WVnmfBK1W58H!ey)niTncB(KD4M!cbDxCl^ty2Kp+0Z zQF0CL%4ORv&ZgC?E!m+`4K6yTfCfE*gjm+Kv}>JHvgVzv6tXSgrJ;*dUAQpeRUG2o zA_De^MY9Ks)RnI)b&%ZMUdW|I$TRRqH_8BIfHFWCpbSt3C zd<@tuR*MtI|5o7zMmP>1bfXMV1}FoR0m=YnfHFWCpbSt3C<=?6dsUc|-obX>O1!0k-bd{>W z0uNYY>DSbdf24RYkTQ;nNijaE$(lc|L<1>Ng_L1+Nu-lBPB`d~DoPX<5Q(W&Kug1- zGr$?>0G!RH=##?h+yG$(X@rm}j>P@?r_R;Da#Ik%Mv^AMVnfhjnJEZ4J#O%e|P1Y?o8dT*lAjwMr z=~@$Yi-;kpIHLeHv>2ci6QP2ziVA!|AS{c~q*CTmUSzU{C0Nif*#=PJd!eC}!o?)g z3V1|a>%1xu6gBLS23r7UyNP-Q0&-GL2Lga4HUrirX0Y99P9yK~2!CFAMj3jY@qDm=6O&b`J z02o^cW5nZ$KuCm!)qsic07KMyfhuu18cW9%_&E^+XTS?MMEh%i11qngRf8Wgi9;b1 zIY1!lvH(H%#D)?(JOFVnVVk62NjYRc=!UdOmnrz20PrG4{s&wp7(sWoMz_3%Gy-0+ zAJ9Z-nB-4mUvj+@5c&z*Ab^+M>pt;%hY87Jc8Wp8PCF#Ok#z1W?e+jmnt#KkV?*F- zW&;GG)yzPME5(`|wgLoECvHGgH5J!c0Ofp=rYI&3v3YHm6+SJ(9XL)Gggf|y%w3is zKkB^0xuWKB=WC81T7K-_`i!d4tU`+#Q&Wlo84@VKBf4-gE&I5 zTd#lW@&5{Q5US$;<>nw%#sAA>HoJ|e9p(i;M*P3@1~x01iZnm|_nEDm5&yRjve|3R z)U6W#FBxXDikVPnj{g^%tzjkpU!<^E@jcP782`5scX{)wKq>xj-Nk0No2ge-r7-*z8uKxi>Vw>}0e3q-{`%|D9$e zUy1)60XDmlc;Emn3aE{^&Wu{riIspXN3oZe-;A5Rl=phd4ijNg+ItY$?xYM z<{#wK{2=e+IiB_YqxW;(5$^@wm0q4Z#l6Zs$Nil94)*}}MecSk$$g6J;CRogo}YR4 zc^>fG?AhhH*0aTPiKpMw3NUn|3{VCr1C#;E0A+wOKpCJ6yq_3wt!FK+1vPj74YDj> z2VXDkK5=Mz0zH_UnMOGT+lz^DGD#WK%{qxot={OE?W3J=IL<-p3vZm9ncj=`&Ti;O zh-GJHb2EioZk&B|>fm(O$s{P+kMj4Rr}7B>Jkwv>0kwjVAVN7S-wubDEFF}N=cmwAhOb)!`zw48F-+>j(cGbq z*T%=W#Wf1SI{UX@wT=LM~qwHg-tB?Wv+MCCP3H=e^ITz0{U+5sqy^H=z=N0##?3__|dn?hjYUSP2#uHp9E9oCebUiD3h@{8ZmscwSED} z&c@)WO#hn`{RQ^7BjeUf8*r9t7re2dkUomWp11;`u9N6!-i02$XFodf_z21s&`dkZ z++6r(AfMlxMkg^=Ul%ir&|8I!YZm=(>?AsnVQcCk_4>ZyM})pQ_!eHZ&+$*W8MGX( zpV*tv9Q;_mm`ikjekzChA4X{O1Z48~!#RXDxNG5-h5q5^cIBKN0lqi)4nOxw201)D z-dVRV3!GRvFYL6g-S^1sq1@hfo`bz5YlaU`=D&#wdFNse?Eh8IzR_p$$4`wH4j;V# z(I@whB}a}S`PuPP7xcQ}^v=~Me)*kWZ7K8@jwg@bIx`l1an}=_ z)Z>$T#wYWKr}kJl7Y1t@KC)+O{6yYS1AF{JtJeu~5xZ{&^?P4H&h-x1jdgrwb#(R! zLch-=zvYDj`o^13Pe#xyZ$CFNJv09H2s$(|hWZQWzyu1+&P?!jxM@k(@Lz*bm%lXG z-p9gz*mh#o#1y)BIyW(P{zx!~4!wQ2ur~vCeC&CYnLx*;erY>7d0_JG0?;Tl+2EG7 z4Z~0NqnQE%Q~ZABGnofx3mK;sPSjfe-fFk7E*Edb^Z)DlFEK)1cuROu_+McGzhCh3 zUlJTV;(yJbQXTUaKv z2{GX}{OdxF{|^6v@DIXc!gqyl3HJ(j2@eXl2w&#E{(f@xG%u6^$^d16GC&!i3{VCr z1C#;E0A+wO@V;RH2hWxoJ%+Y)uhaL|_vvEgT3yupb>ZmMMeQ10)b;4XvsxFzDqZm1 zy6~>lMQ4{TxK3TTJ9Ob%p$qtRdkkw|ri2+l@Ga3r`(j-zS)_}_ZMtY{ z)y1L~U9>jqqNPa}D;DaaxltF53v{udK^J}Xx>!}G3xBOHdIepq<8`sttBW<9E_yt= zSnbwDlS>y1YjnXnbk38K4Z9zyO~Ar}4iD#L^@w1C#;E0A+wOKpCJ6PzERilmW^BWq>m9 zK4Ji`|EKZ)`^X?PE0h7s0A+wOKpCJ6PzERilmW^BWq>k388CqXx3!a5%-qGmFaLeK z=HK{(%w3isKkB^0xx)GybISTT?>StM_j`vwiV}@W zflwmcn;Pvg??sm>F)9H<2XTbrYrXYPovRftfe8?O8EEl!wJBB7q_oyc{*~*7DYQ@^ zE+=L@@h8D_Hz`9fDwDyS+W;0pp_S&SwXEiH#MfL(01h9y{l zTBNX9@jcN{N@3O7$h*9`#tVuXF8HnFE7@F}su$X~5P!>W(fL(DG?RSGFVxzo->Ap> zE2)d^M3Z?aRk|$P!Dcs*EpkdSB1ggMjV2_WNGM52dO?iMZY7#)qfa0wpb-tER3(T@ zZUe*E$O6g9v=mjvw5%jGf0ZHXiJKc<0uO4WBsD#1ya&FHVc7eK5(g-%lv32Rfl(V_ zvs*}G#N&xTNQB16fDwp0FHj{eM`P)jGNe^-c;dY&z=_HD&FXSUj6)&w5_egEQ0!1) z6kLuuhOvHyv`tcqg0t@lvDwSsLE7XSgL03n%6df~x{iE5sEjots!4F&<$h$Jc}TmY z(Gf)rmyyv&^|M9LvB9wwK zHX+K%Kma;f>vyu*e&PVo(;3zWp1XKbAneLk$?GkBUHt`dp%g7NeA+V>%jAjxx?1djM5) zM?=gQ*D1}wiK&z^e$+`k90X0~jk=goLrT+1<&hmXagT`Ah>4Z)JeL`6B!Gv!^|4%M zfX!|sUHPg4T03!q=FU~7Mwgp|;Kz}BLQFxUw2XM0QJmB;=JT{Pj7Wz7X(@jGzl-@i zBmApy0smWmH{a}i#Jht#$=%JZ^E~6xJaz7`xi4}ZcI|fYHD9gygmcn)JN!c8e#bTT zSM4|2>)88PzwIZst=6N~Tdf_IA6Z0Z8j^eWTcnF+vR9KF?J&kv;!4&0f25N%4pgKw z_?>|cz}akyKBf8pffb|?ils&W)VpB=%S}Om5vk(*e`<7KnJEZWbt<+L5Vn!D!`bRo z$_Gf2nMf%y5R-;fm{cgvS;yK<*3FpzmzMz2wI=FTng1VJ3{Z-RP-mY1mlv6=VP*b5 z*#=PJd!b=*{y)+Rctl<6yed$d{~v4tob4v+Rn^>!&46_Yd8{gka1%gWNfe?y15-8s zA6{r0N|i3#8v$hlSzSu=|JxUkPrB4+j5a{hTS?~H=o8GH|BuxJ5Ya~BY{7cC4v>fz zD(HTk7U>3S0p&81!~x2@m|YS8V+&!7)93#qJirijUZ6_V401WZ8t?)R(f%6X%!znK z4iJdCEI`mbu@UU<@BqZQgl$q9;Rw1RZPI0GUX(lFGQkLQqTGQR(g^cM7O!^#LO)>} zRO0CC9VR4S8SUF?hvYYs&V9fb79C(o^KZEH()_<@0|cVg%s?p5|A(yrLDY#G5LIpV zbrwK5pQI_O636jB=V@nzG2ue~8GewrdhhYB=Js=+^t|f12~NjNyJqg%#8rg@LY$JUyL zH1F|v*$+r$CnU=J&3`<8u-7zGo%VRVyv8)OqVtx-MtT5>=)js$D&{PN9$al2eWxwG kjjaMKb4vQWrMK620}|0u9_%2D2jBuLA$b%0k1-DOf5zLvMF0Q* delta 1684 zcmcK4&ui0Q7zgmAeG{ybyb~r<6uLiPI8PJwFn?WknDa7rDP(DzcIn!trqMcddzZgp z3L8B*=}umY40chLi{fE|69fli>p?ahL#pe3xTVSC+X}aXjj#9k`De`|TUCzX%v=U7yNh3|EFrrlVL8_)ij#`$Q3@Z<0 zL)DU$QM{DtX@IG#5QBE8M)wRwPpC;Pu0@9`<#-w_r>;|n2@%;)qMEKMm97O@%5?op zIdvFGQj@A7Nz^!5P2vQDpW(~Ggvc>sOq{1tGzAhvW*>VD`rk?9Rb=)arpSLu{oy2X z3NpJ7Q#B=W!~=F9i70~1o`#sZ3TY&X-iFJm>(pjKGCd0WN%RDf+2uQiEc}tg%0H3C zHbd{~HQm&|>2Kmihrzq}5pKhNd_r_+G}lDp;}Pp?_IljiA7?oF*oJ0S;VqQl)zVV# zS*~L^4}flaIo4-h;AdJG$mQBg02?p|b_;-Ol{JWbcL5?UadNw`mxBV> zb{2Ms3!ee#IIoq<~Ig}y0 zGh1i}3rg*>(bV{b{M?C&pH9D5G=e4?j%bnyah7~aGSxPVvjH`<=p_h($ diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md index 64ae9695..bc8c9ca0 100644 --- a/docs/GLOSSARY.md +++ b/docs/GLOSSARY.md @@ -19,7 +19,7 @@ UI terms agencies recognize, mapped to internal keys and data sources. | Crawl summary | `charts` | Crawl aggregates | SF overview | | Internal links | `network` | Link graph | Ahrefs Internal Links | | Backlinks | `backlinks`, `gsc_links`, `gsc_links_data` | GSC Links CSV import (Google sample) | GSC Links report | -| Page previews | `gallery` | Crawl excerpts | Visual QA | +| Page previews | `gallery`, `list_site_image_urls`, `image_inventory` | Crawl excerpts + optional HTTP probe | Visual QA; size/format when probed | | Search Console | `search-performance`, `google_data` (scoped by `property_id`) | GSC API per property | Google Search Console | | Analytics (GA4) | `traffic`, `google_data` (scoped by `property_id`) | GA4 API per property | Google Analytics | | Keywords | `keywords-explorer`, `keyword_data` | Crawl + Search Console + research | Keyword tools (site-scoped) | @@ -43,7 +43,7 @@ UI terms agencies recognize, mapped to internal keys and data sources. | AI Chat | `/chat`, `/api/chat`, `chat_sessions` | LLM + read-only audit tools | Conversational site audit queries | | MCP tools | `python -m website_profiling.mcp` | Same `audit_tools` as chat | Cursor / Claude Desktop integration — see [MCP.md](MCP.md) | | Read-only session | `AUTH_DEFAULT_ROLE=client-readonly`, `/api/auth/session` | Session cookie | Client view-only access | -| Export executive summary | `export_audit_html/pdf/csv`, `executive_summary` | Report payload + optional AI | Client deliverable | +| Export executive summary | `export_audit_html/pdf/csv`, `export_audit_report` (chat/MCP), Export view | Report payload + optional AI | Client deliverable | ## Metric names diff --git a/docs/MCP.md b/docs/MCP.md index c499d43d..dd473f8e 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -43,31 +43,43 @@ Add to `.cursor/mcp.json` (or Cursor MCP settings): | `audit://glossary` | Excerpt from `docs/GLOSSARY.md` | | `audit://tools` | Tool catalog grouped by SEO domain | -## Tools (121 read-only) +## Tools (171 read-only + export) + +### Export and deliverables + +`export_audit_report`, `export_compare_csv`, `export_list_as_csv`, `compose_custom_report`, `export_custom_report`, `list_export_formats` + +Full audit exports reuse the same generators as the Export view (PDF requires `reportlab`). Export tools store files as artifacts (24h TTL); in-app chat renders download buttons via `/api/chat/artifacts/{id}`. + +### Image audit + +`get_image_audit_summary`, `list_pages_without_lazy_images`, `list_pages_with_images_missing_dimensions`, `list_site_image_urls`, `list_lighthouse_image_opportunities`, `list_largest_images`, `list_unoptimized_images`, `list_images_needing_attention` + +Size-based tools require `probe_image_inventory=true` in pipeline config when building the report. Keys: `max_image_probe_urls` (default 500), `image_probe_concurrency`, `image_probe_timeout`, `image_unoptimized_min_kb` (default 200). ### Portfolio and report -`list_properties`, `get_property`, `get_report_summary`, `get_category_scores`, `get_executive_summary`, `get_report_meta`, `get_site_level`, `list_report_history`, `get_audit_recommendations`, `get_ml_errors`, `get_ssl_expiry_info`, `list_audit_categories`, `get_category_recommendations`, `get_crawl_summary` +`list_properties`, `get_property`, `get_report_summary`, `get_category_scores`, `get_executive_summary`, `get_report_meta`, `get_site_level`, `list_report_history`, `get_audit_recommendations`, `get_ml_errors`, `get_ssl_expiry_info`, `list_audit_categories`, `get_category_recommendations`, `get_crawl_summary`, `get_portfolio_summary` ### Issues and workflow -`list_issues`, `list_issues_by_category`, `get_category_issues`, `list_issue_workflow`, `list_issues_with_ai_fixes`, `list_seo_onpage_issues` +`list_issues`, `search_issues`, `list_issues_by_category`, `get_category_issues`, `list_issue_workflow`, `list_issues_with_ai_fixes`, `list_seo_onpage_issues` ### On-page SEO -`list_content_url_issues`, `list_pages_missing_title`, `list_pages_missing_h1`, `list_pages_multiple_h1`, `list_pages_missing_meta_description`, `list_pages_meta_desc_too_short`, `list_pages_meta_desc_too_long`, `list_pages_noindex`, `get_seo_health` +`list_content_url_issues`, `list_pages_missing_title`, `list_pages_missing_h1`, `list_pages_multiple_h1`, `list_pages_missing_meta_description`, `list_pages_meta_desc_too_short`, `list_pages_meta_desc_too_long`, `list_pages_noindex`, `get_seo_health`, `list_pages_missing_canonical`, `list_canonical_mismatch`, `list_pages_with_missing_alt`, `list_pages_skipped_headings`, `list_pages_missing_viewport`, `list_pages_missing_og_image` ### Crawl and pages -`search_pages`, `search_pages_advanced`, `get_page_details`, `get_page_analysis`, `get_internal_links`, `list_redirects`, `list_broken_links`, `list_status_4xx_pages`, `list_status_5xx_pages`, `get_status_code_breakdown`, `get_response_time_stats`, `get_depth_distribution`, `get_crawl_segments`, `get_browser_diagnostics_summary`, `list_pages_with_console_errors`, `list_pages_by_fetch_method`, `get_crawl_links_table`, `get_graph_edges_sample` +`search_pages`, `search_pages_advanced`, `get_page_details`, `get_page_analysis`, `get_internal_links`, `list_redirects`, `list_broken_links`, `list_status_4xx_pages`, `list_status_5xx_pages`, `get_status_code_breakdown`, `get_response_time_stats`, `get_depth_distribution`, `get_crawl_segments`, `get_browser_diagnostics_summary`, `list_pages_with_console_errors`, `list_pages_by_fetch_method`, `get_crawl_links_table`, `get_graph_edges_sample`, `list_long_redirect_chains`, `list_robots_blocked_urls`, `get_top_pages_by_pagerank` ### Schema and technical -`get_schema_coverage`, `list_pages_without_schema`, `search_pages_by_schema_type`, `get_tech_stack_summary`, `get_security_findings` +`get_schema_coverage`, `list_pages_without_schema`, `search_pages_by_schema_type`, `get_tech_stack_summary`, `list_pages_by_technology`, `get_security_findings`, `get_security_findings_summary`, `list_security_findings_by_type` ### Links and architecture -`list_orphan_pages`, `get_top_linked_pages`, `get_top_crawled_pages`, `get_outbound_link_domains`, `get_link_graph_summary`, `get_url_fingerprints`, `get_mime_type_breakdown`, `get_title_length_distribution`, `get_domain_link_distribution`, `get_outlink_distribution` +`list_orphan_pages`, `get_top_linked_pages`, `get_top_crawled_pages`, `get_outbound_link_domains`, `get_link_graph_summary`, `get_url_fingerprints`, `list_broken_link_sources`, `get_mime_type_breakdown`, `get_title_length_distribution`, `get_domain_link_distribution`, `get_outlink_distribution` ### Indexation and international @@ -75,11 +87,11 @@ Add to `.cursor/mcp.json` (or Cursor MCP settings): ### Content and social -`get_content_analytics`, `get_content_duplicates`, `get_social_coverage`, `get_keyword_opportunities`, `get_ner_site_summary`, `list_thin_content_pages` +`get_content_analytics`, `get_content_duplicates`, `get_duplicate_cluster`, `get_social_coverage`, `get_keyword_opportunities`, `get_ner_site_summary`, `list_thin_content_pages` ### Keywords -`get_keyword_summary`, `search_keywords`, `get_striking_distance_keywords`, `get_keyword_cannibalisation`, `get_query_page_misalignment`, `get_semantic_keyword_clusters`, `get_keyword_history`, `get_keyword_serp_overlay`, `list_keywords_by_action`, `list_keywords_by_position`, `list_keywords_by_impressions` +`get_keyword_summary`, `search_keywords`, `get_striking_distance_keywords`, `get_keyword_cannibalisation`, `get_query_page_misalignment`, `get_semantic_keyword_clusters`, `get_keyword_history`, `get_keyword_serp_overlay`, `list_keywords_by_action`, `list_keywords_by_position`, `list_keywords_by_impressions`, `expand_keywords`, `generate_content_brief` ### Google @@ -91,25 +103,41 @@ Add to `.cursor/mcp.json` (or Cursor MCP settings): ### Performance -`get_lighthouse_summary`, `get_lighthouse_for_url`, `get_lighthouse_human_summary`, `get_lighthouse_diagnostics`, `get_crux_summary`, `list_slow_pages`, `list_lighthouse_poor_seo_pages` +`get_lighthouse_summary`, `get_lighthouse_for_url`, `get_lighthouse_human_summary`, `get_lighthouse_diagnostics`, `get_crux_summary`, `list_slow_pages`, `list_lighthouse_poor_seo_pages`, `list_lighthouse_poor_accessibility_pages`, `list_lighthouse_poor_best_practices_pages`, `list_lighthouse_cwv_failures` ### Drift, health, and compare -`get_health_history`, `get_category_health_history`, `compare_reports`, `compare_issue_deltas`, `compare_category_deltas`, `compare_seo_health_deltas`, `compare_lighthouse_deltas`, `compare_url_set_diff`, `compare_redirect_deltas`, `compare_link_metric_deltas` +`get_health_history`, `get_category_health_history`, `compare_reports`, `compare_issue_deltas`, `compare_category_deltas`, `compare_seo_health_deltas`, `compare_lighthouse_deltas`, `compare_url_set_diff`, `compare_redirect_deltas`, `compare_link_metric_deltas`, `compare_security_deltas`, `compare_duplicate_deltas`, `compare_tech_deltas`, `compare_content_metrics`, `compare_google_metrics`, `compare_priority_counts`, `compare_health_score_delta` ### Ops and logs -`get_integration_alerts`, `get_property_ops`, `list_crawl_runs`, `list_log_uploads`, `get_latest_log_analysis` +`get_integration_alerts`, `get_property_ops`, `list_crawl_runs`, `list_log_uploads`, `get_latest_log_analysis`, `get_log_top_paths`, `list_log_only_paths`, `list_crawl_only_paths`, `get_log_googlebot_stats`, `get_log_analysis_by_id`, `get_page_coach` + +## Future pipeline items (not yet exposed as tools) + +These require additional crawl or third-party integrations before dedicated tools are useful: + +- Google Rich Results / schema validation API +- Full backlink index and anchor-text analytics +- axe / color-contrast accessibility audits +- SERP rank tracking beyond GSC position snapshots ## Example prompts - "What indexation gaps exist between crawl and GSC?" -- "List pages missing titles or meta descriptions" -- "Show backlinks velocity and third-party overlay data" -- "List keyword cannibalisation issues" -- "Compare the latest audit to report ID 38 — what URLs were added or removed?" -- "Which pages have JS console errors or poor Lighthouse SEO scores?" -- "What does the latest access log analysis show?" +- "List pages missing canonical tags or with canonical mismatches" +- "Which paths appear in access logs but were not crawled?" +- "Compare GSC clicks vs the previous audit" +- "List pages failing Core Web Vitals thresholds" +- "Show security finding changes since report 38" +- "Which pages link to broken URLs?" +- "Generate a content brief for keyword X" +- "Download the audit as PDF" +- "Export broken links as CSV" +- "Compare report 38 to the current audit and give me a CSV diff" +- "Build a client report with executive summary, category scores, and top critical issues as PDF" +- "Which images are largest and unoptimized?" +- "List pages with images missing alt or lazy loading" ## In-app chat diff --git a/input.txt.example b/input.txt.example index 8e7b1840..4d1b4d0e 100644 --- a/input.txt.example +++ b/input.txt.example @@ -41,6 +41,11 @@ max_nodes_plot = 400 run_security_scan = true security_scan_active = false security_max_urls_probe = 20 +probe_image_inventory = false +max_image_probe_urls = 500 +image_probe_concurrency = 6 +image_probe_timeout = 8 +image_unoptimized_min_kb = 200 # --- Lighthouse --- lighthouse_url = diff --git a/pipeline-config.example.txt b/pipeline-config.example.txt index aee1441b..b1c384b3 100644 --- a/pipeline-config.example.txt +++ b/pipeline-config.example.txt @@ -42,6 +42,11 @@ max_nodes_plot = 400 run_security_scan = true security_scan_active = false security_max_urls_probe = 20 +probe_image_inventory = false +max_image_probe_urls = 500 +image_probe_concurrency = 6 +image_probe_timeout = 8 +image_unoptimized_min_kb = 200 # --- Lighthouse --- lighthouse_url = diff --git a/src/website_profiling/analysis/image_probe.py b/src/website_profiling/analysis/image_probe.py new file mode 100644 index 00000000..9bb661aa --- /dev/null +++ b/src/website_profiling/analysis/image_probe.py @@ -0,0 +1,142 @@ +"""Probe image URLs for Content-Type and size (HEAD with GET fallback).""" +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any +from urllib.parse import urlparse + +import requests + +_PARTIAL_GET_CAP = 65536 +_USER_AGENT = "WebsiteProfilingImageProbe/1.0" + + +def _normalize_image_url(url: str) -> str | None: + raw = str(url or "").strip() + if not raw or raw.lower().startswith("data:"): + return None + parsed = urlparse(raw) + if parsed.scheme not in ("http", "https"): + return None + return raw.split("#", 1)[0] + + +def _parse_size(headers: dict[str, Any]) -> int | None: + cl = headers.get("Content-Length") or headers.get("content-length") + if cl is None: + return None + try: + return int(cl) + except (TypeError, ValueError): + return None + + +def _probe_one(url: str, *, timeout: int, session: requests.Session) -> dict[str, Any]: + result: dict[str, Any] = { + "url": url, + "status": None, + "content_type": None, + "size_bytes": None, + "error": None, + } + try: + resp = session.head(url, timeout=timeout, allow_redirects=True) + if resp.status_code in (405, 501, 403): + resp = session.get(url, timeout=timeout, allow_redirects=True, stream=True) + size = _parse_size(resp.headers) + if size is None: + read = 0 + for chunk in resp.iter_content(chunk_size=8192): + if not chunk: + break + read += len(chunk) + if read >= _PARTIAL_GET_CAP: + size = read + break + else: + size = read if read else None + result["status"] = resp.status_code + result["content_type"] = (resp.headers.get("Content-Type") or "").split(";")[0].strip() or None + result["size_bytes"] = size + resp.close() + return result + result["status"] = resp.status_code + result["content_type"] = (resp.headers.get("Content-Type") or "").split(";")[0].strip() or None + result["size_bytes"] = _parse_size(resp.headers) + if result["size_bytes"] is None and resp.status_code == 200: + resp2 = session.get(url, timeout=timeout, allow_redirects=True, stream=True) + read = 0 + for chunk in resp2.iter_content(chunk_size=8192): + if not chunk: + break + read += len(chunk) + if read >= _PARTIAL_GET_CAP: + break + result["size_bytes"] = read if read else None + resp2.close() + except Exception as exc: + result["error"] = str(exc)[:200] + return result + + +def probe_image_urls( + urls: list[str], + *, + concurrency: int = 6, + timeout: int = 8, + session: requests.Session | None = None, +) -> list[dict[str, Any]]: + """Return probe results for each unique http(s) image URL.""" + seen: set[str] = set() + unique: list[str] = [] + for raw in urls: + norm = _normalize_image_url(raw) + if not norm or norm in seen: + continue + seen.add(norm) + unique.append(norm) + + owns = session is None + sess = session or requests.Session() + if owns: + sess.headers.update({"User-Agent": _USER_AGENT}) + + results: list[dict[str, Any]] = [] + workers = max(1, min(concurrency, 20)) + try: + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = {pool.submit(_probe_one, u, timeout=timeout, session=sess): u for u in unique} + for fut in as_completed(futures): + results.append(fut.result()) + finally: + if owns: + sess.close() + return results + + +def collect_image_refs_from_links(links: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Map image URL -> { source_pages: set, kinds: set }.""" + out: dict[str, dict[str, Any]] = {} + + def _add(raw: str | None, page_url: str, kind: str) -> None: + norm = _normalize_image_url(str(raw or "")) + if not norm or not page_url: + return + if norm not in out: + out[norm] = {"source_pages": set(), "kinds": set()} + out[norm]["source_pages"].add(page_url) + out[norm]["kinds"].add(kind) + + for link in links: + if not isinstance(link, dict): + continue + page_url = str(link.get("url") or "").strip() + if not page_url: + continue + pa = link.get("page_analysis") + if isinstance(pa, dict): + for u in pa.get("image_urls") or []: + _add(str(u), page_url, "content") + _add(link.get("og_image"), page_url, "og") + _add(link.get("twitter_image"), page_url, "twitter") + return out diff --git a/src/website_profiling/llm/agent.py b/src/website_profiling/llm/agent.py index 236dcc18..687f5d09 100644 --- a/src/website_profiling/llm/agent.py +++ b/src/website_profiling/llm/agent.py @@ -16,19 +16,38 @@ You help users understand crawl results, audit issues, Lighthouse scores, keywords, and Search Console data. Tool domains (prefer specific tools over generic list_issues): -- Portfolio/report: get_report_summary, get_category_scores, list_audit_categories, get_executive_summary, get_audit_recommendations, list_report_history -- Issues: list_issues, get_critical_issues, list_issues_by_category, get_category_issues, list_issues_with_ai_fixes, list_issue_workflow -- On-page: list_pages_missing_title, list_pages_noindex, list_seo_onpage_issues, list_content_url_issues -- Crawl/pages: search_pages, search_pages_advanced, get_page_details, get_page_analysis, list_status_4xx_pages, get_status_code_breakdown, get_depth_distribution -- Schema/technical: get_schema_coverage, get_seo_health, get_security_findings, get_tech_stack_summary +- Portfolio/report: get_report_summary, get_category_scores, list_audit_categories, get_executive_summary, get_audit_recommendations, list_report_history, get_portfolio_summary +- Issues: list_issues, search_issues, get_critical_issues, list_issues_by_category, get_category_issues, list_issues_with_ai_fixes, list_issue_workflow +- On-page: list_pages_missing_title, list_pages_noindex, list_seo_onpage_issues, list_content_url_issues, list_pages_missing_canonical, list_canonical_mismatch, list_pages_with_missing_alt, list_pages_missing_viewport +- Crawl/pages: search_pages, search_pages_advanced, get_page_details, get_page_analysis, list_status_4xx_pages, get_status_code_breakdown, get_depth_distribution, list_long_redirect_chains, list_robots_blocked_urls, get_top_pages_by_pagerank +- Schema/technical: get_schema_coverage, get_seo_health, get_security_findings, get_security_findings_summary, get_tech_stack_summary, list_pages_by_technology - Indexation: get_indexation_coverage, list_indexation_gaps, get_indexation_url_join -- Keywords: get_keyword_summary, get_striking_distance_keywords, list_keywords_by_position, get_keyword_serp_overlay +- Keywords: get_keyword_summary, get_striking_distance_keywords, list_keywords_by_position, get_keyword_serp_overlay, expand_keywords, generate_content_brief - Google: get_google_summary, get_gsc_top_queries, get_gsc_top_pages, get_google_integration_status, get_gsc_page_query_slice, get_ga4_page_metrics -- Links/backlinks: get_gsc_sample_links, get_backlinks_velocity, get_third_party_links_overlay -- Performance: get_lighthouse_summary, list_slow_pages, get_crux_summary, get_lighthouse_human_summary -- Content/charts: get_issue_priority_breakdown, get_mime_type_breakdown, get_title_length_distribution, get_domain_link_distribution, get_outlink_distribution, get_content_analytics, get_top_crawled_pages -- Ops/logs: get_property_ops, list_crawl_runs, get_latest_log_analysis -- Drift: compare_reports, compare_category_deltas, compare_issue_deltas, compare_url_set_diff, get_health_history, get_category_health_history +- Links/backlinks: get_gsc_sample_links, get_backlinks_velocity, get_third_party_links_overlay, list_broken_link_sources, get_page_coach +- Performance: get_lighthouse_summary, list_slow_pages, get_crux_summary, get_lighthouse_human_summary, list_lighthouse_poor_accessibility_pages, list_lighthouse_cwv_failures +- Content/charts: get_issue_priority_breakdown, get_mime_type_breakdown, get_title_length_distribution, get_domain_link_distribution, get_outlink_distribution, get_content_analytics, get_top_crawled_pages, get_duplicate_cluster +- Ops/logs: get_property_ops, list_crawl_runs, get_latest_log_analysis, get_log_top_paths, list_log_only_paths, list_crawl_only_paths, get_log_googlebot_stats +- Drift: compare_reports, compare_category_deltas, compare_issue_deltas, compare_url_set_diff, compare_google_metrics, compare_security_deltas, compare_health_score_delta, get_health_history, get_category_health_history +- Export/deliverables: export_audit_report, export_compare_csv, export_list_as_csv, compose_custom_report, export_custom_report, list_export_formats +- Images: get_image_audit_summary, list_pages_with_missing_alt, list_pages_without_lazy_images, list_pages_with_images_missing_dimensions, list_site_image_urls, list_lighthouse_image_opportunities, list_largest_images, list_unoptimized_images, list_images_needing_attention + +Image playbook: +- Overview: get_image_audit_summary first — the UI renders summary cards, page preview lists (alt/lazy/OG/dimensions), and Lighthouse image findings. Write only ### Power Insights and ### Recommended actions (interpretation). Never repeat counts, URL lists, or markdown tables of pages. +- Missing alt / lazy / OG / dimensions: get_image_audit_summary includes previews; call list_pages_* only if the user wants the full exportable list +- All image URLs: list_site_image_urls (optional kind filter) +- Lighthouse image issues: list_lighthouse_image_opportunities +- Largest / heavy files: list_largest_images (requires probe_image_inventory=true on report build) +- Unoptimized format/size: list_unoptimized_images (requires image inventory probe) +- What needs attention: list_images_needing_attention +- Export lists: export_list_as_csv with the matching list tool + +Export playbook (chat UI shows download buttons after export tools — do not paste file contents): +- Full audit PDF/HTML/CSV/JSON: export_audit_report with format pdf|html|csv|json +- Compare issue diff CSV: export_compare_csv with baseline_report_id +- Export a list as CSV: export_list_as_csv with tool_name and tool_args (e.g. list_broken_links) +- Custom client report: compose_custom_report with title and sections (executive_summary, category_scores, tool, notes), then export_custom_report format=pdf or html +- After export tools succeed, tell the user their download is ready; the UI renders file buttons automatically Visualization playbook (chat UI renders charts and tables from tool JSON automatically): - Category scores / health: get_category_scores, list_audit_categories, or get_report_summary @@ -37,7 +56,7 @@ - Audit overview / site health recap: get_report_summary (health, crawl, categories, issue counts). Keep prose to interpretation and next steps only — never repeat health score, URL counts, success rate, category scores, or priority counts in markdown; the UI renders those as cards and charts. - Distributions: get_mime_type_breakdown, get_title_length_distribution, get_domain_link_distribution, get_status_code_breakdown, get_depth_distribution - Trends over time: get_health_history, get_category_health_history -- Compare drift: compare_category_deltas, compare_issue_deltas +- Compare drift: compare_category_deltas, compare_issue_deltas, compare_google_metrics, compare_security_deltas - Lighthouse: get_lighthouse_summary - Google/GSC: get_google_summary, get_gsc_top_queries diff --git a/src/website_profiling/llm/providers/ollama.py b/src/website_profiling/llm/providers/ollama.py index ff1d2696..acfafc65 100644 --- a/src/website_profiling/llm/providers/ollama.py +++ b/src/website_profiling/llm/providers/ollama.py @@ -59,6 +59,48 @@ def normalize_messages_for_ollama(messages: list[dict[str, Any]]) -> list[dict[s return out +def _extract_ollama_error(response: Any) -> str: + raw = "" + try: + if getattr(response, "is_stream_consumed", True) is False: + body = response.read() + raw = body.decode("utf-8", errors="replace") if isinstance(body, bytes) else str(body) + else: + raw = response.text or "" + except Exception: + raw = "" + raw = raw.strip() + if not raw: + return "" + try: + data = json.loads(raw) + if isinstance(data, dict) and data.get("error"): + return str(data["error"]).strip() + except json.JSONDecodeError: + pass + return raw + + +def format_ollama_error(status_code: int, detail: str, model: str) -> str: + """Human-readable Ollama HTTP error for chat UI.""" + detail = detail.strip() + low = detail.lower() + if status_code == 404 and "model" in low and "not found" in low: + return ( + f"Ollama model '{model}' is not installed. " + f"Run `ollama pull {model}` or pick another model under Audit settings → AI." + ) + if status_code == 404: + hint = detail or "endpoint not found" + return ( + f"Ollama returned 404 for /api/chat ({hint}). " + "Check that Ollama is running, llm_base_url is correct, and your Ollama version supports chat." + ) + if detail: + return f"Ollama API error ({status_code}): {detail}" + return f"Ollama API error ({status_code})." + + class OllamaClient: def __init__(self, cfg: dict[str, str]) -> None: self._model = (cfg.get("llm_model") or "llama3.2").strip() @@ -74,16 +116,15 @@ def _client(self): return httpx.Client(timeout=self._timeout) def _raise_for_status(self, response: Any) -> None: + if int(getattr(response, "status_code", 0) or 0) >= 400: + detail = _extract_ollama_error(response) + raise RuntimeError(format_ollama_error(response.status_code, detail, self._model)) try: response.raise_for_status() except Exception as e: - detail = "" - try: - detail = response.text.strip() - except Exception: - pass + detail = _extract_ollama_error(response) if detail: - raise RuntimeError(f"Ollama API error ({response.status_code}): {detail}") from e + raise RuntimeError(format_ollama_error(response.status_code, detail, self._model)) from e raise def complete_json(self, system: str, user: str) -> dict[str, Any]: diff --git a/src/website_profiling/mcp/server.py b/src/website_profiling/mcp/server.py index c185cccd..bee74233 100644 --- a/src/website_profiling/mcp/server.py +++ b/src/website_profiling/mcp/server.py @@ -79,36 +79,44 @@ def _tools_catalog_json() -> str: "performance": [], "drift": [], "security": [], + "ops": [], + "export": [], + "images": [], } for tool in TOOL_DEFINITIONS: name = tool["name"] - if name.startswith(("list_propert", "get_propert", "get_report", "get_executive", "get_site", "list_report")): + if name.startswith("export_") or name == "compose_custom_report" or name == "list_export_formats": + domains["export"].append(name) + elif name.startswith(("get_image_", "list_pages_without_lazy", "list_pages_with_images_missing", "list_site_image", "list_lighthouse_image", "list_largest_images", "list_unoptimized_images", "list_images_needing")): + domains["images"].append(name) + elif name.startswith(("list_propert", "get_propert", "get_report", "get_executive", "get_site", "list_report", "get_portfolio")): domains["portfolio"].append(name) elif "issue" in name or "category" in name or "workflow" in name: domains["issues"].append(name) - elif name in ("search_pages", "get_page_details", "get_internal_links", "list_redirects", "list_broken_links", - "get_status_code", "get_response_time", "get_depth", "get_crawl_segments", "get_browser"): + elif name.startswith(("list_pages_", "list_canonical", "list_long_", "list_robots_", "get_top_pages_by", "search_pages", "get_page_", "list_redirects", "list_broken", "list_status_", "get_status_code", "get_response_time", "get_depth", "get_crawl_", "get_browser", "list_pages_with", "list_pages_by")): domains["crawl"].append(name) elif "schema" in name or name == "get_seo_health": domains["schema"].append(name) - elif "orphan" in name or "link" in name or "fingerprint" in name: + elif "orphan" in name or "link" in name or "fingerprint" in name or "pagerank" in name: domains["links"].append(name) elif "indexation" in name or "hreflang" in name or "language" in name: domains["indexation"].append(name) - elif "content" in name or "social" in name or "ner" in name or "thin" in name or "opportunit" in name: + elif "content" in name or "social" in name or "ner" in name or "thin" in name or "opportunit" in name or "duplicate" in name: domains["content"].append(name) - elif "keyword" in name or "cannibal" in name or "misalignment" in name or "striking" in name or "semantic" in name: + elif "keyword" in name or "cannibal" in name or "misalignment" in name or "striking" in name or "semantic" in name or name == "expand_keywords" or name == "generate_content_brief": domains["keywords"].append(name) elif "google" in name or "gsc" in name or "ga4" in name: domains["google"].append(name) elif "backlink" in name or "competitor" in name or "bing" in name or "gsc_links" in name: domains["backlinks"].append(name) - elif "lighthouse" in name or "crux" in name or "slow" in name: + elif "lighthouse" in name or "crux" in name or "slow" in name or "cwv" in name: domains["performance"].append(name) - elif "health" in name or "compare" in name or "alert" in name or "tech_stack" in name: + elif "health" in name or "compare" in name or "alert" in name or "tech_stack" in name or name == "list_pages_by_technology": domains["drift"].append(name) elif "security" in name: domains["security"].append(name) + elif "log" in name or name in ("get_property_ops", "list_crawl_runs", "list_log_uploads", "get_page_coach"): + domains["ops"].append(name) else: domains["portfolio"].append(name) return json.dumps({"tool_count": len(TOOL_DEFINITIONS), "handlers": sorted(tool_handler_names()), "domains": domains}, indent=2) diff --git a/src/website_profiling/reporting/builder.py b/src/website_profiling/reporting/builder.py index 35927766..e040bd77 100644 --- a/src/website_profiling/reporting/builder.py +++ b/src/website_profiling/reporting/builder.py @@ -978,6 +978,66 @@ def _build_keyword_opportunities(df: pd.DataFrame, config: dict[str, str] | None } +def _build_image_inventory( + links: list[dict[str, Any]], + config: Optional[dict[str, str]], +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + from ..analysis.image_probe import collect_image_refs_from_links, probe_image_urls + + refs = collect_image_refs_from_links(links) + unoptimized_min_kb = get_int(config or {}, "image_unoptimized_min_kb", 200) or 200 + summary: dict[str, Any] = { + "probed": 0, + "failed": 0, + "total_bytes": 0, + "over_threshold_count": 0, + "unoptimized_min_kb": unoptimized_min_kb, + "inventory_available": False, + } + if not get_bool(config or {}, "probe_image_inventory", False): + return [], summary + + max_urls = get_int(config or {}, "max_image_probe_urls", 500) or 500 + concurrency = get_int(config or {}, "image_probe_concurrency", 6) or 6 + probe_timeout = get_int(config or {}, "image_probe_timeout", 8) or 8 + url_list = list(refs.keys())[:max_urls] + if not url_list: + return [], summary + + print(f" Probing up to {len(url_list)} image URL(s)...", flush=True) + probed = probe_image_urls( + url_list, + concurrency=concurrency, + timeout=probe_timeout, + ) + threshold_bytes = unoptimized_min_kb * 1024 + inventory: list[dict[str, Any]] = [] + for row in probed: + url = row.get("url") + meta = refs.get(str(url or ""), {"source_pages": set(), "kinds": set()}) + size = row.get("size_bytes") + entry = { + "url": url, + "status": row.get("status"), + "content_type": row.get("content_type"), + "size_bytes": size, + "error": row.get("error"), + "source_pages": sorted(meta.get("source_pages") or []), + "kinds": sorted(meta.get("kinds") or []), + } + inventory.append(entry) + summary["probed"] += 1 + if row.get("error") or row.get("status") is None: + summary["failed"] += 1 + if size is not None: + summary["total_bytes"] += int(size) + if int(size) >= threshold_bytes: + summary["over_threshold_count"] += 1 + summary["inventory_available"] = True + print(f" Image probe complete ({summary['probed']} URLs, {summary['failed']} failed).", flush=True) + return inventory, summary + + def run_simple_report( max_fetch_for_edges: int = 300, concurrency: int = 6, @@ -1311,6 +1371,7 @@ def _bool_col(col): rec["images_total"] = _int_col("images_total") rec["images_without_alt"] = _int_col("images_without_alt") rec["img_without_lazy"] = _int_col("img_without_lazy") + rec["img_without_dimensions"] = _int_col("img_without_dimensions") rec["aria_count"] = _int_col("aria_count") rec["mixed_content_count"] = _int_col("mixed_content_count") @@ -1450,6 +1511,66 @@ def _bool_col(col): title_str = "" if pd.isna(row.get("title")) else str(row.get("title")).strip() thin_content.append({"url": u, "title": title_str, "content_length": c}) + missing_canonical: list[dict[str, Any]] = [] + canonical_mismatch: list[dict[str, Any]] = [] + missing_alt: list[dict[str, Any]] = [] + success_mask = df["status"].astype(str).str.match(r"2\d{2}", na=False) if "status" in df.columns else pd.Series([True] * len(df)) + success_df_urls = df[success_mask] if len(df) else df + if "canonical_url" in success_df_urls.columns: + for _, row in success_df_urls.iterrows(): + u = row.get("url") + if pd.isna(u) or not u: + continue + u = str(u).strip() + title_str = "" if pd.isna(row.get("title")) else str(row.get("title")).strip() + canon = "" if pd.isna(row.get("canonical_url")) else str(row.get("canonical_url")).strip() + if not canon: + missing_canonical.append({"url": u, "title": title_str}) + elif u.rstrip("/").lower() != canon.rstrip("/").lower(): + canonical_mismatch.append({"url": u, "canonical_url": canon, "title": title_str}) + if "images_without_alt" in success_df_urls.columns: + alt_missing = pd.to_numeric(success_df_urls["images_without_alt"], errors="coerce").fillna(0).astype(int) + for i, row in success_df_urls.iterrows(): + if alt_missing.loc[i] <= 0: + continue + u = row.get("url") + if pd.isna(u) or not u: + continue + missing_alt.append({ + "url": str(u).strip(), + "images_without_alt": int(alt_missing.loc[i]), + "images_total": int(pd.to_numeric(row.get("images_total"), errors="coerce") or 0), + }) + + missing_lazy: list[dict[str, Any]] = [] + missing_dimensions: list[dict[str, Any]] = [] + if "img_without_lazy" in success_df_urls.columns: + lazy_missing = pd.to_numeric(success_df_urls["img_without_lazy"], errors="coerce").fillna(0).astype(int) + for i, row in success_df_urls.iterrows(): + if lazy_missing.loc[i] <= 0: + continue + u = row.get("url") + if pd.isna(u) or not u: + continue + missing_lazy.append({ + "url": str(u).strip(), + "img_without_lazy": int(lazy_missing.loc[i]), + "images_total": int(pd.to_numeric(row.get("images_total"), errors="coerce") or 0), + }) + if "img_without_dimensions" in success_df_urls.columns: + dim_missing = pd.to_numeric(success_df_urls["img_without_dimensions"], errors="coerce").fillna(0).astype(int) + for i, row in success_df_urls.iterrows(): + if dim_missing.loc[i] <= 0: + continue + u = row.get("url") + if pd.isna(u) or not u: + continue + missing_dimensions.append({ + "url": str(u).strip(), + "img_without_dimensions": int(dim_missing.loc[i]), + "images_total": int(pd.to_numeric(row.get("images_total"), errors="coerce") or 0), + }) + content_urls = { "missing_h1": missing_h1, "missing_title": missing_title, @@ -1458,6 +1579,11 @@ def _bool_col(col): "meta_desc_short": meta_desc_short, "meta_desc_long": meta_desc_long, "thin_content": thin_content, + "missing_canonical": missing_canonical, + "canonical_mismatch": canonical_mismatch, + "missing_alt": missing_alt, + "missing_lazy": missing_lazy, + "missing_dimensions": missing_dimensions, } print(" Building content analytics...", flush=True) @@ -1481,6 +1607,7 @@ def _bool_col(col): tech_stack_summary = _build_tech_stack_summary(df) response_time_stats = _build_response_time_stats(df) depth_distribution = _build_depth_distribution(df) + image_inventory, image_inventory_summary = _build_image_inventory(links, config) report_data = { "site_name": site_display, @@ -1515,6 +1642,8 @@ def _bool_col(col): "tech_stack_summary": tech_stack_summary, "response_time_stats": response_time_stats, "depth_distribution": depth_distribution, + "image_inventory": image_inventory, + "image_inventory_summary": image_inventory_summary, "content_duplicates": ml_bundle.get("content_duplicates") or [], "language_summary": ml_bundle.get("language_summary") or {}, "ner_site_summary": ml_bundle.get("ner_site_summary") or {}, diff --git a/src/website_profiling/tools/audit_tools/compare_slices.py b/src/website_profiling/tools/audit_tools/compare_slices.py index 60c32c4f..a7b2636c 100644 --- a/src/website_profiling/tools/audit_tools/compare_slices.py +++ b/src/website_profiling/tools/audit_tools/compare_slices.py @@ -7,12 +7,19 @@ from ...reporting.compare_payload import ( build_category_scores, + build_content_metrics, + build_duplicate_deltas, + build_google_metrics, build_issue_deltas, build_lighthouse_url_deltas, build_link_metric_deltas, + build_priority_counts, build_redirect_deltas, + build_security_deltas, build_seo_health_deltas, + build_tech_deltas, build_url_set_diff, + _score_from_categories, ) from ._slice import cap_list, parse_limit from .compare_helpers import load_compare_pair @@ -134,3 +141,92 @@ def compare_link_metric_deltas(conn: Connection, ctx: AuditToolContext, args: di "total": sliced["total"], "truncated": sliced["truncated"], } + + +def _compare_list_slice( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + *, + builder, + result_key: str, + default_limit: int = 50, + max_cap: int = 100, +) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return err + assert current is not None and baseline is not None + items = builder(current, baseline) + limit = parse_limit(args.get("limit"), default_limit, max_cap) + sliced = cap_list(items if isinstance(items, list) else [], limit, max_cap=max_cap) + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + result_key: sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + } + + +def compare_security_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _compare_list_slice(conn, ctx, args, builder=build_security_deltas, result_key="security_deltas") + + +def compare_duplicate_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _compare_list_slice(conn, ctx, args, builder=build_duplicate_deltas, result_key="duplicate_deltas") + + +def compare_tech_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _compare_list_slice(conn, ctx, args, builder=build_tech_deltas, result_key="tech_deltas") + + +def compare_content_metrics(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return err + assert current is not None and baseline is not None + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "content_metrics": build_content_metrics(current, baseline), + } + + +def compare_google_metrics(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return err + assert current is not None and baseline is not None + google = build_google_metrics(current, baseline) + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "google_available": google.get("available", False), + "google_metrics": google.get("metrics") or [], + } + + +def compare_priority_counts(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return err + assert current is not None and baseline is not None + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "priority_counts": build_priority_counts(current, baseline), + } + + +def compare_health_score_delta(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return err + assert current is not None and baseline is not None + cur_health = _score_from_categories(current.get("categories") or []) + base_health = _score_from_categories(baseline.get("categories") or []) + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "health_score": { + "current": cur_health, + "baseline": base_health, + "delta": (cur_health - base_health) if cur_health is not None and base_health is not None else None, + }, + } diff --git a/src/website_profiling/tools/audit_tools/content.py b/src/website_profiling/tools/audit_tools/content.py index c4bee0a8..a69c5ca9 100644 --- a/src/website_profiling/tools/audit_tools/content.py +++ b/src/website_profiling/tools/audit_tools/content.py @@ -68,3 +68,37 @@ def list_thin_content_pages(conn: Connection, ctx: AuditToolContext, args: dict[ } sliced = cap_list(thin, limit, max_cap=50) return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def get_duplicate_cluster(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found"} + clusters = payload.get("content_duplicates") or [] + if not isinstance(clusters, list): + return {"error": "content_duplicates not in report", "missing": True} + url = str(args.get("url") or "").strip().rstrip("/").lower() + cluster_index = args.get("cluster_index") + if cluster_index is not None: + try: + idx = int(cluster_index) + except (TypeError, ValueError): + return {"error": "invalid cluster_index"} + if idx < 0 or idx >= len(clusters): + return {"error": "cluster_index out of range"} + cluster = clusters[idx] + if not isinstance(cluster, dict): + return {"error": "cluster not found"} + return {"cluster_index": idx, "cluster": cluster} + if url: + for i, cluster in enumerate(clusters): + if not isinstance(cluster, dict): + continue + rep = str(cluster.get("representative_url") or "").strip().rstrip("/").lower() + members = cluster.get("member_urls") or [] + member_norm = {str(u).strip().rstrip("/").lower() for u in members if u} + if url == rep or url in member_norm: + return {"cluster_index": i, "cluster": cluster} + return {"error": "url not found in duplicate clusters", "url": url} + return {"error": "cluster_index or url is required"} diff --git a/src/website_profiling/tools/audit_tools/crawl_lists.py b/src/website_profiling/tools/audit_tools/crawl_lists.py new file mode 100644 index 00000000..ed7a02fd --- /dev/null +++ b/src/website_profiling/tools/audit_tools/crawl_lists.py @@ -0,0 +1,281 @@ +"""Crawl-backed list tools for technical SEO, accessibility, and mobile gaps.""" +from __future__ import annotations + +from typing import Any, Callable + +import pandas as pd +from psycopg import Connection + +from ...reporting.categories import REDIRECT_CHAIN_LONG +from ._slice import cap_list, parse_limit +from .context import AuditToolContext + +_REDIRECT_CHAIN_MIN = REDIRECT_CHAIN_LONG + + +def _norm_url(url: str) -> str: + return str(url or "").strip().rstrip("/").lower() + + +def _is_2xx(status: Any) -> bool: + s = str(status or "") + return bool(s) and s[0] == "2" + + +def _truthy(val: Any) -> bool: + return str(val or "").lower() in ("true", "1", "yes") + + +def _success_df(df: pd.DataFrame) -> pd.DataFrame: + if df.empty or "status" not in df.columns: + return df + return df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] + + +def _filter_crawl_pages( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + *, + predicate: Callable[[dict[str, Any]], bool], + projection: Callable[[dict[str, Any]], dict[str, Any]], + only_2xx: bool = True, + item_key: str = "pages", +) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {item_key: [], "total": 0, "truncated": False} + work = _success_df(df) if only_2xx else df + pages: list[dict[str, Any]] = [] + for _, row in work.iterrows(): + rec = row.to_dict() + if predicate(rec): + pages.append(projection(rec)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _content_urls_list( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + bucket: str, +) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + content_urls = payload.get("content_urls") or {} + if not isinstance(content_urls, dict): + return {"error": "content_urls not in report", "missing": True, "pages": [], "total": 0, "truncated": False} + items = content_urls.get(bucket) or [] + if not isinstance(items, list): + items = [] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(items, limit, max_cap=50) + return {"bucket": bucket, "pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_pages_missing_canonical(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + bucket = _content_urls_list(conn, ctx, args, "missing_canonical") + if bucket.get("total", 0) > 0 or bucket.get("missing"): + return bucket + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=lambda r: not str(r.get("canonical_url") or "").strip(), + projection=lambda r: { + "url": str(r.get("url") or ""), + "title": str(r.get("title") or ""), + "status": str(r.get("status") or ""), + }, + ) + + +def list_canonical_mismatch(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + bucket = _content_urls_list(conn, ctx, args, "canonical_mismatch") + if bucket.get("total", 0) > 0 or bucket.get("missing"): + return bucket + + def _mismatch(r: dict[str, Any]) -> bool: + url = _norm_url(str(r.get("url") or "")) + canon = _norm_url(str(r.get("canonical_url") or "")) + return bool(url and canon and url != canon) + + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=_mismatch, + projection=lambda r: { + "url": str(r.get("url") or ""), + "canonical_url": str(r.get("canonical_url") or ""), + "title": str(r.get("title") or ""), + }, + ) + + +def list_pages_with_missing_alt(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + bucket = _content_urls_list(conn, ctx, args, "missing_alt") + if bucket.get("total", 0) > 0 or bucket.get("missing"): + return bucket + + def _missing_alt(r: dict[str, Any]) -> bool: + try: + return int(r.get("images_without_alt") or 0) > 0 + except (TypeError, ValueError): + return False + + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=_missing_alt, + projection=lambda r: { + "url": str(r.get("url") or ""), + "images_without_alt": int(r.get("images_without_alt") or 0), + "images_total": int(r.get("images_total") or 0), + }, + ) + + +def _heading_skipped(seq: str) -> bool: + parts = [p.strip() for p in str(seq or "").split(",") if p.strip()] + if not parts: + return False + levels: list[int] = [] + for h in parts: + if len(h) == 2 and h[0] == "h" and h[1] in "123456": + levels.append(int(h[1])) + for i in range(1, len(levels)): + if levels[i] > levels[i - 1] + 1: + return True + return False + + +def list_pages_skipped_headings(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=lambda r: _heading_skipped(str(r.get("heading_sequence") or "")), + projection=lambda r: { + "url": str(r.get("url") or ""), + "heading_sequence": str(r.get("heading_sequence") or ""), + "title": str(r.get("title") or ""), + }, + ) + + +def list_pages_missing_viewport(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False, "note": "no crawl data"} + if "viewport_present" not in df.columns: + return {"pages": [], "total": 0, "truncated": False, "note": "viewport_present column not in crawl"} + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=lambda r: not _truthy(r.get("viewport_present")), + projection=lambda r: { + "url": str(r.get("url") or ""), + "title": str(r.get("title") or ""), + }, + ) + + +def list_long_redirect_chains(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + def _long_chain(r: dict[str, Any]) -> bool: + try: + return int(r.get("redirect_chain_length") or 0) >= _REDIRECT_CHAIN_MIN + except (TypeError, ValueError): + return False + + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=_long_chain, + projection=lambda r: { + "url": str(r.get("url") or ""), + "status": str(r.get("status") or ""), + "redirect_chain_length": int(r.get("redirect_chain_length") or 0), + }, + only_2xx=False, + ) + + +def list_robots_blocked_urls(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=lambda r: str(r.get("status") or "") == "blocked_by_robots", + projection=lambda r: { + "url": str(r.get("url") or ""), + "status": str(r.get("status") or ""), + }, + only_2xx=False, + ) + + +def list_pages_missing_og_image(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if payload: + social = payload.get("social_coverage") or {} + if isinstance(social, dict): + missing = social.get("og_image_missing") or [] + if isinstance(missing, list) and missing: + limit = parse_limit(args.get("limit"), 30, 50) + items = [{"url": str(u)} for u in missing if u] + sliced = cap_list(items, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=lambda r: _is_2xx(r.get("status")) and not str(r.get("og_image") or "").strip(), + projection=lambda r: { + "url": str(r.get("url") or ""), + "og_image": str(r.get("og_image") or ""), + "title": str(r.get("title") or ""), + }, + ) + + +def get_top_pages_by_pagerank(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + candidates = payload.get("top_pages") or payload.get("links") or [] + if not isinstance(candidates, list): + candidates = [] + ranked: list[dict[str, Any]] = [] + for rec in candidates: + if not isinstance(rec, dict): + continue + pr = rec.get("pagerank") + if pr is None: + continue + try: + score = float(pr) + except (TypeError, ValueError): + continue + ranked.append({ + "url": rec.get("url"), + "pagerank": round(score, 5), + "inlinks": rec.get("inlinks"), + "outlinks": rec.get("outlinks"), + }) + ranked.sort(key=lambda x: float(x.get("pagerank") or 0), reverse=True) + sliced = cap_list(ranked, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} diff --git a/src/website_profiling/tools/audit_tools/export_tools.py b/src/website_profiling/tools/audit_tools/export_tools.py new file mode 100644 index 00000000..253401a2 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/export_tools.py @@ -0,0 +1,345 @@ +"""Export and deliverable tools for chat and MCP.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ..export_artifacts import ( + dicts_to_csv, + read_report_spec, + rows_from_tool_result, + save_artifact, + save_report_spec, +) +from ..export_compare import export_compare_issues_csv +from ..export_custom import ( + render_custom_report_html, + render_custom_report_pdf, + resolve_section_results, + validate_sections, +) +from ..export_audit import ( + export_audit_csv, + export_audit_html, + export_audit_json, + export_audit_pdf, +) +from ._slice import parse_limit +from .compare_helpers import load_compare_pair +from .context import AuditToolContext + +_EXPORT_FORMATS = {"pdf", "html", "csv", "json"} +_CUSTOM_FORMATS = {"html", "pdf"} +_MIME = { + "pdf": "application/pdf", + "html": "text/html; charset=utf-8", + "csv": "text/csv; charset=utf-8", + "json": "application/json; charset=utf-8", +} +_EXT = {"pdf": "pdf", "html": "html", "csv": "csv", "json": "json"} + +_LIST_EXPORT_ALLOWLIST = frozenset({ + "list_issues", + "search_issues", + "list_issues_by_category", + "list_issues_with_ai_fixes", + "list_seo_onpage_issues", + "list_content_url_issues", + "list_pages_missing_title", + "list_pages_missing_h1", + "list_pages_multiple_h1", + "list_pages_missing_meta_description", + "list_pages_meta_desc_too_short", + "list_pages_meta_desc_too_long", + "list_pages_noindex", + "list_redirects", + "list_broken_links", + "list_broken_link_sources", + "list_status_4xx_pages", + "list_status_5xx_pages", + "list_orphan_pages", + "list_thin_content_pages", + "list_pages_missing_canonical", + "list_canonical_mismatch", + "list_pages_with_missing_alt", + "list_pages_without_lazy_images", + "list_pages_with_images_missing_dimensions", + "list_site_image_urls", + "list_largest_images", + "list_unoptimized_images", + "list_images_needing_attention", + "list_pages_skipped_headings", + "list_pages_missing_viewport", + "list_long_redirect_chains", + "list_robots_blocked_urls", + "list_pages_missing_og_image", + "list_pages_by_technology", + "list_pages_with_console_errors", + "list_pages_by_fetch_method", + "list_security_findings_by_type", + "list_indexation_gaps", + "list_keywords_by_action", + "list_keywords_by_position", + "list_keywords_by_impressions", + "list_lighthouse_poor_seo_pages", + "list_lighthouse_poor_accessibility_pages", + "list_lighthouse_poor_best_practices_pages", + "list_lighthouse_cwv_failures", + "list_slow_pages", + "list_log_only_paths", + "list_crawl_only_paths", + "compare_issue_deltas", + "compare_redirect_deltas", + "compare_lighthouse_deltas", + "get_log_top_paths", + "get_top_pages_by_pagerank", + "get_top_crawled_pages", + "get_top_linked_pages", + "search_pages", + "search_pages_advanced", + "search_keywords", + "search_pages_by_schema_type", + "list_pages_without_schema", +}) + +_EXPORT_TOOL_NAMES = frozenset({ + "export_audit_report", + "export_compare_csv", + "export_list_as_csv", + "compose_custom_report", + "export_custom_report", + "list_export_formats", +}) + + +def _dispatch(name: str, args: dict[str, Any], ctx: AuditToolContext, conn: Connection) -> dict[str, Any]: + from .registry import dispatch_tool + return dispatch_tool(name, args, context=ctx, conn=conn) + + +def _artifact_from_bytes( + data: bytes | str, + *, + filename: str, + mime_type: str, + extra: dict[str, Any] | None = None, +) -> dict[str, Any]: + return save_artifact(data, filename=filename, mime_type=mime_type, meta=extra) + + +def export_audit_report(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + fmt = str(args.get("format") or "pdf").lower().strip() + if fmt not in _EXPORT_FORMATS: + return {"error": f"format must be one of: {', '.join(sorted(_EXPORT_FORMATS))}"} + report_id = scoped.report_id + try: + if fmt == "pdf": + data = export_audit_pdf(report_id) + filename = f"audit-export.{_EXT[fmt]}" + return { + **_artifact_from_bytes(data, filename=filename, mime_type=_MIME[fmt], extra={"format": fmt, "report_id": report_id}), + "format": fmt, + "report_id": report_id, + } + if fmt == "html": + data = export_audit_html(report_id) + elif fmt == "csv": + data = export_audit_csv(report_id) + else: + data = export_audit_json(report_id) + filename = f"audit-export.{_EXT[fmt]}" + return { + **_artifact_from_bytes(data, filename=filename, mime_type=_MIME[fmt], extra={"format": fmt, "report_id": report_id}), + "format": fmt, + "report_id": report_id, + } + except FileNotFoundError: + return {"error": "no report found"} + except RuntimeError as exc: + return {"error": str(exc)} + + +def export_compare_csv(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return err + assert current is not None and baseline is not None + csv_text = export_compare_issues_csv(current, baseline) + filename = f"audit-compare-{cur_rid}-vs-{base_rid}.csv" + return { + **_artifact_from_bytes(csv_text, filename=filename, mime_type=_MIME["csv"], extra={"baseline_report_id": base_rid, "report_id": cur_rid}), + "current_report_id": cur_rid, + "baseline_report_id": base_rid, + "format": "csv", + } + + +def export_list_as_csv(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + tool_name = str(args.get("tool_name") or "").strip() + if not tool_name: + return {"error": "tool_name is required"} + if tool_name not in _LIST_EXPORT_ALLOWLIST: + return {"error": f"tool_name not allowed for CSV export: {tool_name}"} + tool_args = dict(args.get("tool_args") or {}) + limit = parse_limit(args.get("limit"), 100, 500) + tool_args["limit"] = limit + scoped = ctx.with_args({**tool_args, **{k: v for k, v in args.items() if k in ("property_id", "report_id")}}) + if scoped.property_id is not None and "property_id" not in tool_args: + tool_args["property_id"] = scoped.property_id + if scoped.report_id is not None and "report_id" not in tool_args: + tool_args["report_id"] = scoped.report_id + result = _dispatch(tool_name, tool_args, scoped, conn) + if result.get("error"): + return result + rows = rows_from_tool_result(result) + if not rows: + return {"error": "tool returned no exportable rows", "tool_name": tool_name} + columns_raw = args.get("columns") + columns = [str(c) for c in columns_raw if c] if isinstance(columns_raw, list) else None + csv_text = dicts_to_csv(rows, columns) + filename = f"{tool_name}.csv" + return { + **_artifact_from_bytes( + csv_text, + filename=filename, + mime_type=_MIME["csv"], + extra={"tool_name": tool_name, "row_total": len(rows)}, + ), + "tool_name": tool_name, + "total": len(rows), + "format": "csv", + } + + +def _tool_allowed_for_custom(tool_name: str) -> bool: + if tool_name in _EXPORT_TOOL_NAMES: + return False + from .registry import tool_handler_names + return tool_name in tool_handler_names() + + +def compose_custom_report(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + title = str(args.get("title") or "").strip() + if not title: + return {"error": "title is required"} + sections_raw = args.get("sections") + sections, err = validate_sections(sections_raw) + if err: + return {"error": err} + assert sections is not None + for section in sections: + if section.get("type") == "tool": + tname = str(section.get("tool_name") or "") + if not _tool_allowed_for_custom(tname): + return {"error": f"tool not allowed in custom report: {tname}"} + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found"} + spec = { + "title": title, + "sections": sections, + "property_id": scoped.property_id, + "report_id": scoped.report_id, + } + spec_id = save_report_spec(spec) + preview_html = render_custom_report_html( + title=title, + payload=payload, + sections=sections, + section_results=[None] * len(sections), + ) + snippet = preview_html[:400].replace("\n", " ") + return { + "report_spec_id": spec_id, + "section_count": len(sections), + "preview_html_snippet": snippet, + "title": title, + } + + +def export_custom_report(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + fmt = str(args.get("format") or "html").lower().strip() + if fmt not in _CUSTOM_FORMATS: + return {"error": f"format must be one of: {', '.join(sorted(_CUSTOM_FORMATS))}"} + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found"} + spec_id = args.get("report_spec_id") + title = str(args.get("title") or "").strip() + sections: list[dict[str, Any]] | None = None + if spec_id: + spec = read_report_spec(str(spec_id)) + if not spec: + return {"error": "report_spec_id not found"} + title = str(spec.get("title") or title or "Custom Report") + raw_sections = spec.get("sections") + sections, err = validate_sections(raw_sections) + if err: + return {"error": err} + else: + sections, err = validate_sections(args.get("sections")) + if err: + return {"error": err} + if not title: + return {"error": "title is required when report_spec_id is omitted"} + assert sections is not None + for section in sections: + if section.get("type") == "tool": + tname = str(section.get("tool_name") or "") + if not _tool_allowed_for_custom(tname): + return {"error": f"tool not allowed in custom report: {tname}"} + section_results = resolve_section_results(conn, scoped, payload, sections, _dispatch) + html_doc = render_custom_report_html( + title=title, + payload=payload, + sections=sections, + section_results=section_results, + ) + safe_title = "".join(c if c.isalnum() or c in "-_" else "-" for c in title.lower())[:40] or "custom-report" + if fmt == "html": + filename = f"{safe_title}.html" + return { + **_artifact_from_bytes(html_doc, filename=filename, mime_type=_MIME["html"], extra={"format": fmt, "title": title}), + "format": fmt, + "title": title, + } + try: + pdf_bytes = render_custom_report_pdf(html_doc, title) + except RuntimeError as exc: + return {"error": str(exc)} + filename = f"{safe_title}.pdf" + return { + **_artifact_from_bytes(pdf_bytes, filename=filename, mime_type=_MIME["pdf"], extra={"format": fmt, "title": title}), + "format": fmt, + "title": title, + } + + +def list_export_formats(_conn: Connection, _ctx: AuditToolContext, _args: dict[str, Any]) -> dict[str, Any]: + return { + "formats": [ + {"tool": "export_audit_report", "format": "pdf", "description": "Full audit PDF deliverable"}, + {"tool": "export_audit_report", "format": "html", "description": "Full audit HTML preview/print"}, + {"tool": "export_audit_report", "format": "csv", "description": "Full audit CSV (URLs + issues)"}, + {"tool": "export_audit_report", "format": "json", "description": "Full audit JSON payload"}, + {"tool": "export_compare_csv", "format": "csv", "description": "Issue added/removed diff between two reports"}, + {"tool": "export_list_as_csv", "format": "csv", "description": "CSV from any allowlisted list tool result"}, + {"tool": "compose_custom_report", "description": "Save a multi-section custom report spec"}, + {"tool": "export_custom_report", "format": "html|pdf", "description": "Render composed custom report"}, + ], + "example_prompts": [ + "Download the audit as PDF", + "Export broken links as CSV", + "Compare this report to report 38 as CSV", + "Build a client report with executive summary, category scores, and broken links", + ], + "notes": [ + "PDF requires reportlab (pip install reportlab)", + "Artifacts expire after 24 hours", + "Chat UI shows download buttons after export tools run", + ], + } diff --git a/src/website_profiling/tools/audit_tools/image_tools.py b/src/website_profiling/tools/audit_tools/image_tools.py new file mode 100644 index 00000000..c1aea621 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/image_tools.py @@ -0,0 +1,368 @@ +"""Image audit tools for chat and MCP.""" +from __future__ import annotations + +from typing import Any + +import pandas as pd +from psycopg import Connection + +from ._slice import cap_list, parse_limit +from .context import AuditToolContext +from .crawl_lists import _content_urls_list, _filter_crawl_pages + +IMAGE_LIGHTHOUSE_AUDIT_IDS = frozenset({ + "uses-optimized-images", + "uses-responsive-images", + "preload-lcp-image", + "image-aspect-ratio", + "image-alt", + "efficient-animated-content", + "largest-contentful-paint", +}) + +_MODERN_IMAGE_TYPES = frozenset({"image/webp", "image/avif"}) + +_PREVIEW_LIMIT = 12 + + +def _preview_from_bucket(rows: list[Any] | None, limit: int = _PREVIEW_LIMIT) -> dict[str, Any]: + if not isinstance(rows, list) or not rows: + return {"pages": [], "total": 0, "truncated": False} + pages: list[dict[str, Any]] = [] + for row in rows[:limit]: + if isinstance(row, dict) and row.get("url"): + pages.append(dict(row)) + elif isinstance(row, str) and row.strip(): + pages.append({"url": row.strip()}) + total = len(rows) + return {"pages": pages, "total": total, "truncated": total > len(pages)} + + +def _lighthouse_image_previews(payload: dict[str, Any], limit: int = 8) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for d in payload.get("lighthouse_diagnostics") or []: + if not isinstance(d, dict): + continue + audit_id = str(d.get("lighthouse_audit_id") or "") + if audit_id not in IMAGE_LIGHTHOUSE_AUDIT_IDS: + continue + out.append({ + "title": str(d.get("title") or audit_id), + "lighthouse_audit_id": audit_id, + "url": str(d.get("url") or d.get("page") or ""), + "display_value": d.get("display_value"), + }) + if len(out) >= limit: + break + return out + + +def _int_val(val: Any) -> int: + try: + return int(val or 0) + except (TypeError, ValueError): + return 0 + + +def _inventory_from_payload(payload: dict[str, Any]) -> list[dict[str, Any]]: + raw = payload.get("image_inventory") + if not isinstance(raw, list): + return [] + return [x for x in raw if isinstance(x, dict)] + + +def get_image_audit_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found"} + df = scoped.load_crawl_df(conn) + pages_missing_alt = 0 + pages_missing_lazy = 0 + pages_missing_dims = 0 + images_total = 0 + if df is not None and not df.empty and "status" in df.columns: + work = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] + if "images_without_alt" in work.columns: + pages_missing_alt = int((pd.to_numeric(work["images_without_alt"], errors="coerce").fillna(0) > 0).sum()) + if "img_without_lazy" in work.columns: + pages_missing_lazy = int((pd.to_numeric(work["img_without_lazy"], errors="coerce").fillna(0) > 0).sum()) + if "img_without_dimensions" in work.columns: + pages_missing_dims = int((pd.to_numeric(work["img_without_dimensions"], errors="coerce").fillna(0) > 0).sum()) + if "images_total" in work.columns: + images_total = int(pd.to_numeric(work["images_total"], errors="coerce").fillna(0).sum()) + social = payload.get("social_coverage") if isinstance(payload.get("social_coverage"), dict) else {} + inv_summary = payload.get("image_inventory_summary") if isinstance(payload.get("image_inventory_summary"), dict) else {} + lh_image = 0 + for d in payload.get("lighthouse_diagnostics") or []: + if isinstance(d, dict) and str(d.get("lighthouse_audit_id") or "") in IMAGE_LIGHTHOUSE_AUDIT_IDS: + lh_image += 1 + content_urls = payload.get("content_urls") if isinstance(payload.get("content_urls"), dict) else {} + og_missing = social.get("og_image_missing") if isinstance(social.get("og_image_missing"), list) else [] + return { + "pages_missing_alt": pages_missing_alt or len(content_urls.get("missing_alt") or []), + "pages_without_lazy_images": pages_missing_lazy or len(content_urls.get("missing_lazy") or []), + "pages_missing_image_dimensions": pages_missing_dims or len(content_urls.get("missing_dimensions") or []), + "images_total_crawled": images_total, + "og_image_coverage_pct": social.get("og_image_coverage_pct"), + "og_image_missing_count": len(og_missing), + "lighthouse_image_diagnostics": lh_image, + "image_inventory_available": bool(_inventory_from_payload(payload)), + "image_inventory_summary": inv_summary, + "page_previews": { + "missing_alt": _preview_from_bucket(content_urls.get("missing_alt")), + "missing_lazy": _preview_from_bucket(content_urls.get("missing_lazy")), + "missing_dimensions": _preview_from_bucket(content_urls.get("missing_dimensions")), + "missing_og": _preview_from_bucket(og_missing), + }, + "lighthouse_image_previews": _lighthouse_image_previews(payload), + } + + +def list_pages_without_lazy_images(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + bucket = _content_urls_list(conn, ctx, args, "missing_lazy") + if bucket.get("total", 0) > 0 or bucket.get("missing"): + return bucket + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=lambda r: _int_val(r.get("img_without_lazy")) > 0, + projection=lambda r: { + "url": str(r.get("url") or ""), + "title": str(r.get("title") or ""), + "img_without_lazy": _int_val(r.get("img_without_lazy")), + "images_total": _int_val(r.get("images_total")), + }, + ) + + +def list_pages_with_images_missing_dimensions(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + bucket = _content_urls_list(conn, ctx, args, "missing_dimensions") + if bucket.get("total", 0) > 0 or bucket.get("missing"): + return bucket + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=lambda r: _int_val(r.get("img_without_dimensions")) > 0, + projection=lambda r: { + "url": str(r.get("url") or ""), + "title": str(r.get("title") or ""), + "img_without_dimensions": _int_val(r.get("img_without_dimensions")), + "images_total": _int_val(r.get("images_total")), + }, + ) + + +def list_site_image_urls(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "items": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 50, 100) + kind_filter = str(args.get("kind") or "").strip().lower() + items: list[dict[str, Any]] = [] + seen: set[tuple[str, str, str]] = set() + for link in payload.get("links") or []: + if not isinstance(link, dict): + continue + page_url = str(link.get("url") or "") + pa = link.get("page_analysis") if isinstance(link.get("page_analysis"), dict) else {} + for u in pa.get("image_urls") or []: + image_url = str(u or "").strip() + if not image_url: + continue + key = (image_url, page_url, "content") + if key in seen: + continue + if kind_filter and kind_filter != "content": + continue + seen.add(key) + items.append({"image_url": image_url, "page_url": page_url, "kind": "content"}) + for kind, field in (("og", "og_image"), ("twitter", "twitter_image")): + image_url = str(link.get(field) or "").strip() + if not image_url: + continue + if kind_filter and kind_filter != kind: + continue + key = (image_url, page_url, kind) + if key in seen: + continue + seen.add(key) + items.append({"image_url": image_url, "page_url": page_url, "kind": kind}) + sliced = cap_list(items, limit, max_cap=100) + return {"items": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_lighthouse_image_opportunities(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "diagnostics": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + diag = payload.get("lighthouse_diagnostics") or [] + filtered = [ + d for d in diag + if isinstance(d, dict) and str(d.get("lighthouse_audit_id") or "") in IMAGE_LIGHTHOUSE_AUDIT_IDS + ] + sliced = cap_list(filtered, limit, max_cap=50) + return {"diagnostics": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _threshold_kb(args: dict[str, Any], payload: dict[str, Any]) -> int: + raw = args.get("min_size_kb") + if raw is not None: + try: + return max(1, int(raw)) + except (TypeError, ValueError): + pass + summary = payload.get("image_inventory_summary") + if isinstance(summary, dict) and summary.get("unoptimized_min_kb") is not None: + try: + return max(1, int(summary["unoptimized_min_kb"])) + except (TypeError, ValueError): + pass + return 200 + + +def list_largest_images(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "items": [], "total": 0, "truncated": False} + inventory = _inventory_from_payload(payload) + if not inventory: + return { + "error": "image_inventory not in report — enable probe_image_inventory and rebuild report", + "inventory_available": False, + "items": [], + "total": 0, + "truncated": False, + } + min_kb = _threshold_kb(args, payload) + min_bytes = min_kb * 1024 + ranked = [ + item for item in inventory + if item.get("size_bytes") is not None and int(item["size_bytes"]) >= min_bytes + ] + ranked.sort(key=lambda x: int(x.get("size_bytes") or 0), reverse=True) + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(ranked, limit, max_cap=100) + return { + "items": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "inventory_available": True, + "min_size_kb": min_kb, + } + + +def list_unoptimized_images(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "items": [], "total": 0, "truncated": False} + inventory = _inventory_from_payload(payload) + if not inventory: + return { + "error": "image_inventory not in report — enable probe_image_inventory and rebuild report", + "inventory_available": False, + "items": [], + "total": 0, + "truncated": False, + } + min_kb = _threshold_kb(args, payload) + min_bytes = min_kb * 1024 + hits: list[dict[str, Any]] = [] + for item in inventory: + size = item.get("size_bytes") + if size is None: + continue + size_i = int(size) + if size_i < min_bytes: + continue + ctype = str(item.get("content_type") or "").lower().split(";")[0].strip() + if ctype in _MODERN_IMAGE_TYPES: + continue + if ctype.startswith("image/") or not ctype: + hits.append({**item, "reason": "large_non_modern_format"}) + hits.sort(key=lambda x: int(x.get("size_bytes") or 0), reverse=True) + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(hits, limit, max_cap=100) + return { + "items": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "inventory_available": True, + "min_size_kb": min_kb, + } + + +def list_images_needing_attention(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "items": [], "total": 0, "truncated": False} + inventory = _inventory_from_payload(payload) + page_issues: dict[str, set[str]] = {} + content_urls = payload.get("content_urls") if isinstance(payload.get("content_urls"), dict) else {} + for bucket, reason in ( + ("missing_alt", "page_missing_alt"), + ("missing_lazy", "page_missing_lazy"), + ("missing_dimensions", "page_missing_dimensions"), + ): + for row in content_urls.get(bucket) or []: + if isinstance(row, dict) and row.get("url"): + page_issues.setdefault(str(row["url"]), set()).add(reason) + min_kb = _threshold_kb(args, payload) + min_bytes = min_kb * 1024 + scored: list[dict[str, Any]] = [] + if inventory: + for item in inventory: + reasons: list[str] = [] + score = 0 + size = item.get("size_bytes") + ctype = str(item.get("content_type") or "").lower().split(";")[0].strip() + if size is not None and int(size) >= min_bytes: + if ctype not in _MODERN_IMAGE_TYPES: + reasons.append("large_non_modern_format") + score += 3 + else: + reasons.append("large_file") + score += 2 + if item.get("error"): + reasons.append("probe_failed") + score += 1 + for page in item.get("source_pages") or []: + for r in page_issues.get(str(page), set()): + if r not in reasons: + reasons.append(r) + score += 2 + if reasons: + scored.append({ + "url": item.get("url"), + "size_bytes": size, + "content_type": item.get("content_type"), + "source_pages": item.get("source_pages") or [], + "kinds": item.get("kinds") or [], + "reasons": reasons, + "attention_score": score, + }) + else: + for page, reasons in page_issues.items(): + scored.append({ + "url": None, + "page_url": page, + "reasons": sorted(reasons), + "attention_score": len(reasons) * 2, + }) + scored.sort(key=lambda x: int(x.get("attention_score") or 0), reverse=True) + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(scored, limit, max_cap=100) + return { + "items": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "inventory_available": bool(inventory), + } diff --git a/src/website_profiling/tools/audit_tools/lighthouse.py b/src/website_profiling/tools/audit_tools/lighthouse.py index 02d87e00..70944b93 100644 --- a/src/website_profiling/tools/audit_tools/lighthouse.py +++ b/src/website_profiling/tools/audit_tools/lighthouse.py @@ -137,3 +137,143 @@ def list_lighthouse_poor_seo_pages(conn: Connection, ctx: AuditToolContext, args poor.sort(key=lambda x: float(x.get("seo") or 0)) sliced = cap_list(poor, limit, max_cap=50) return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "threshold": threshold} + + +def _extract_lh_score(data: dict[str, Any], *keys: str) -> float | None: + for key in keys: + val = data.get(key) + if val is not None: + try: + return float(val) + except (TypeError, ValueError): + pass + scores = data.get("scores") if isinstance(data.get("scores"), dict) else {} + category_scores = data.get("category_scores") if isinstance(data.get("category_scores"), dict) else {} + metrics = data.get("median_metrics") if isinstance(data.get("median_metrics"), dict) else {} + for key in keys: + for block in (scores, category_scores, metrics): + if key in block and block[key] is not None: + try: + return float(block[key]) + except (TypeError, ValueError): + pass + alt = key.replace("-", "_") + if alt in metrics and metrics[alt] is not None: + try: + return float(metrics[alt]) + except (TypeError, ValueError): + pass + return None + + +def _list_lighthouse_poor_category( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + *, + score_keys: tuple[str, ...], + result_key: str, + threshold_arg: str, + default_threshold: int, +) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + threshold = parse_limit(args.get(threshold_arg), default_threshold, 100) + by_url = payload.get("lighthouse_by_url") or {} + poor = [] + if isinstance(by_url, dict): + for url, data in by_url.items(): + if not isinstance(data, dict): + continue + score = _extract_lh_score(data, *score_keys) + if score is not None and score < threshold: + poor.append({"url": url, result_key: score}) + poor.sort(key=lambda x: float(x.get(result_key) or 0)) + sliced = cap_list(poor, limit, max_cap=50) + return { + "pages": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "threshold": threshold, + } + + +def list_lighthouse_poor_accessibility_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + out = _list_lighthouse_poor_category( + conn, + ctx, + args, + score_keys=("accessibility", "accessibility_score"), + result_key="accessibility", + threshold_arg="accessibility_threshold", + default_threshold=50, + ) + for page in out.get("pages") or []: + if "accessibility" in page: + page["accessibility_score"] = page.pop("accessibility") + return out + + +def list_lighthouse_poor_best_practices_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + out = _list_lighthouse_poor_category( + conn, + ctx, + args, + score_keys=("best-practices", "best_practices", "best_practices_score"), + result_key="best_practices", + threshold_arg="best_practices_threshold", + default_threshold=50, + ) + for page in out.get("pages") or []: + if "best_practices" in page: + page["best_practices_score"] = page.pop("best_practices") + return out + + +def list_lighthouse_cwv_failures(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + from ...lighthouse.runner import CLS_GOOD, LCP_GOOD_MS + + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + by_url = payload.get("lighthouse_by_url") or {} + failures: list[dict[str, Any]] = [] + if isinstance(by_url, dict): + for url, data in by_url.items(): + if not isinstance(data, dict): + continue + metrics = data.get("median_metrics") if isinstance(data.get("median_metrics"), dict) else data + lcp = metrics.get("lcp_ms") + cls = metrics.get("cls") + tbt = metrics.get("tbt_ms") + failed: list[str] = [] + try: + if lcp is not None and float(lcp) > LCP_GOOD_MS: + failed.append("lcp") + except (TypeError, ValueError): + pass + try: + if cls is not None and float(cls) > CLS_GOOD: + failed.append("cls") + except (TypeError, ValueError): + pass + try: + if tbt is not None and float(tbt) > 200: + failed.append("tbt") + except (TypeError, ValueError): + pass + if failed: + failures.append({ + "url": url, + "failed_metrics": failed, + "lcp_ms": lcp, + "cls": cls, + "tbt_ms": tbt, + }) + sliced = cap_list(failures, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} diff --git a/src/website_profiling/tools/audit_tools/links.py b/src/website_profiling/tools/audit_tools/links.py index b6d305d0..b0bbfb35 100644 --- a/src/website_profiling/tools/audit_tools/links.py +++ b/src/website_profiling/tools/audit_tools/links.py @@ -76,3 +76,44 @@ def get_url_fingerprints(conn: Connection, ctx: AuditToolContext, args: dict[str fps = payload.get("url_fingerprints") or [] sliced = cap_list(fps if isinstance(fps, list) else [], limit, max_cap=50) return {"fingerprints": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_broken_link_sources(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "sources": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + issues = payload.get("issues") or {} + broken = issues.get("broken") if isinstance(issues, dict) else [] + broken_urls = { + str(b.get("url") or "").strip() + for b in (broken if isinstance(broken, list) else []) + if isinstance(b, dict) and b.get("url") + } + if not broken_urls: + return {"sources": [], "total": 0, "truncated": False} + source_map: dict[str, list[str]] = {} + edges = payload.get("graph_edges") or [] + if isinstance(edges, list): + for edge in edges: + if isinstance(edge, dict): + src = str(edge.get("from") or edge.get("source") or "").strip() + tgt = str(edge.get("to") or edge.get("target") or "").strip() + elif isinstance(edge, (list, tuple)) and len(edge) >= 2: + src, tgt = str(edge[0]).strip(), str(edge[1]).strip() + else: + continue + if tgt in broken_urls and src: + source_map.setdefault(tgt, []).append(src) + items = [] + for tgt, srcs in source_map.items(): + unique_srcs = sorted(set(srcs)) + items.append({ + "broken_url": tgt, + "source_count": len(unique_srcs), + "source_urls": unique_srcs[:10], + }) + items.sort(key=lambda x: x["source_count"], reverse=True) + sliced = cap_list(items, limit, max_cap=50) + return {"sources": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} diff --git a/src/website_profiling/tools/audit_tools/llm_tools.py b/src/website_profiling/tools/audit_tools/llm_tools.py new file mode 100644 index 00000000..7e39bf91 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/llm_tools.py @@ -0,0 +1,124 @@ +"""LLM and cross-property tool wrappers.""" +from __future__ import annotations + +import json +from typing import Any + +from psycopg import Connection + +from ...db._common import _row_field +from ...db.property_store import list_properties_public +from ...integrations.google.suggest import batch_expand +from ...llm.content_brief import generate_content_brief as build_content_brief +from ...llm.page_coach import run_page_coach +from ._slice import parse_limit +from .context import AuditToolContext + + +def generate_content_brief(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + keyword = str(args.get("keyword") or "").strip() + if not keyword: + return {"error": "keyword is required"} + rows: list[dict[str, Any]] = [] + if scoped.property_id is not None: + kw_data = scoped.load_keywords(conn) + if isinstance(kw_data, dict): + all_rows = kw_data.get("rows") or [] + if isinstance(all_rows, list): + needle = keyword.lower() + rows = [ + r for r in all_rows + if isinstance(r, dict) and needle in str(r.get("keyword") or "").lower() + ] + gaps_raw = args.get("gaps") + gaps = [str(g) for g in gaps_raw if g] if isinstance(gaps_raw, list) else None + brief = build_content_brief(keyword, rows, gaps, use_llm=False) + return {"brief": brief, "keyword": keyword, "matched_rows": len(rows)} + + +def get_page_coach(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip() + if not url: + return {"error": "url is required"} + refresh = str(args.get("refresh") or "").lower() in ("true", "1", "yes") + result = run_page_coach( + url, + refresh=refresh, + current_id=scoped.report_id, + ) + return result + + +def get_portfolio_summary(conn: Connection, _ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + limit = parse_limit(args.get("limit"), 50, 100) + props = list_properties_public(conn) + summaries: list[dict[str, Any]] = [] + for prop in props[:limit]: + if not isinstance(prop, dict): + continue + pid = prop.get("id") + if pid is None: + continue + cur = conn.execute( + """SELECT health_score, generated_at, report_id, issue_counts + FROM audit_health_snapshots + WHERE property_id = %s + ORDER BY generated_at DESC, id DESC + LIMIT 1""", + (int(pid),), + ) + row = cur.fetchone() + issue_counts = None + health_score = None + generated_at = None + report_id = None + if row: + health_score = _row_field(row, "health_score", index=0) + generated_at = _row_field(row, "generated_at", index=1) + report_id = _row_field(row, "report_id", index=2) + raw_counts = _row_field(row, "issue_counts", index=3) + if isinstance(raw_counts, str): + try: + issue_counts = json.loads(raw_counts) + except json.JSONDecodeError: + issue_counts = {} + elif isinstance(raw_counts, dict): + issue_counts = raw_counts + summaries.append({ + "property_id": pid, + "name": prop.get("name"), + "canonical_domain": prop.get("canonical_domain"), + "health_score": health_score, + "report_id": report_id, + "generated_at": generated_at.isoformat() if hasattr(generated_at, "isoformat") else str(generated_at or ""), + "issue_counts": issue_counts, + }) + return {"properties": summaries, "count": len(summaries)} + + +def expand_keywords(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + seeds_raw = args.get("seeds") + if isinstance(seeds_raw, str): + seeds = [s.strip() for s in seeds_raw.split(",") if s.strip()] + elif isinstance(seeds_raw, list): + seeds = [str(s).strip() for s in seeds_raw if str(s).strip()] + else: + return {"error": "seeds is required (list or comma-separated string)"} + if not seeds: + return {"error": "seeds is required"} + seeds = seeds[:30] + sources_raw = args.get("sources") + if isinstance(sources_raw, list): + sources = tuple(str(s).strip() for s in sources_raw if str(s).strip()) + else: + sources = ("web", "youtube", "questions") + expanded = batch_expand(seeds, sources=sources, cache_conn=conn) + return { + "property_id": scoped.property_id, + "seeds": seeds, + "expansions": expanded, + "seed_count": len(seeds), + } diff --git a/src/website_profiling/tools/audit_tools/ops.py b/src/website_profiling/tools/audit_tools/ops.py index e3c4afe8..8ba3939c 100644 --- a/src/website_profiling/tools/audit_tools/ops.py +++ b/src/website_profiling/tools/audit_tools/ops.py @@ -9,7 +9,7 @@ from ...db._common import _row_field from ...db.property_store import get_property_by_id from ...tools.alert_checker import check_all_alerts -from ._slice import parse_limit +from ._slice import cap_list, parse_limit from .context import AuditToolContext @@ -124,32 +124,160 @@ def list_log_uploads(conn: Connection, ctx: AuditToolContext, args: dict[str, An return {"uploads": uploads, "count": len(uploads), "property_id": scoped.property_id} +def _parse_analysis_field(raw: Any) -> dict[str, Any]: + if isinstance(raw, dict): + return raw + if isinstance(raw, str): + try: + parsed = json.loads(raw) + return parsed if isinstance(parsed, dict) else {} + except json.JSONDecodeError: + return {} + return {} + + +def _load_log_analysis( + conn: Connection, + property_id: int, + upload_id: int | None = None, +) -> dict[str, Any] | None: + if upload_id is not None: + cur = conn.execute( + """SELECT id, filename, line_count, analysis, uploaded_at + FROM log_file_uploads + WHERE property_id = %s AND id = %s + LIMIT 1""", + (property_id, int(upload_id)), + ) + else: + cur = conn.execute( + """SELECT id, filename, line_count, analysis, uploaded_at + FROM log_file_uploads + WHERE property_id = %s + ORDER BY uploaded_at DESC + LIMIT 1""", + (property_id,), + ) + row = cur.fetchone() + if not row: + return None + uploaded = _row_field(row, "uploaded_at", index=4) + return { + "upload_id": _row_field(row, "id", index=0), + "filename": _row_field(row, "filename", index=1), + "line_count": _row_field(row, "line_count", index=2), + "analysis": _parse_analysis_field(_row_field(row, "analysis", index=3)), + "uploaded_at": uploaded.isoformat() if hasattr(uploaded, "isoformat") else str(uploaded or ""), + } + + def get_latest_log_analysis(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: scoped = ctx.with_args(args) if scoped.property_id is None: return {"error": "property_id is required"} - cur = conn.execute( - """SELECT filename, line_count, analysis, uploaded_at - FROM log_file_uploads - WHERE property_id = %s - ORDER BY uploaded_at DESC - LIMIT 1""", - (int(scoped.property_id),), - ) - row = cur.fetchone() + row = _load_log_analysis(conn, int(scoped.property_id)) if not row: return {"error": "no log uploads found", "missing": True} - analysis = _row_field(row, "analysis", index=2) - if isinstance(analysis, str): - try: - analysis = json.loads(analysis) - except json.JSONDecodeError: - analysis = {} - uploaded = _row_field(row, "uploaded_at", index=3) return { - "filename": _row_field(row, "filename", index=0), - "line_count": _row_field(row, "line_count", index=1), - "analysis": analysis if isinstance(analysis, dict) else {}, - "uploaded_at": uploaded.isoformat() if hasattr(uploaded, "isoformat") else str(uploaded or ""), + **row, "property_id": scoped.property_id, } + + +def get_log_analysis_by_id(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required"} + upload_id = args.get("upload_id") + if upload_id is None: + return {"error": "upload_id is required"} + try: + uid = int(upload_id) + except (TypeError, ValueError): + return {"error": "invalid upload_id"} + row = _load_log_analysis(conn, int(scoped.property_id), upload_id=uid) + if not row: + return {"error": "log upload not found", "missing": True} + return {**row, "property_id": scoped.property_id} + + +def _log_compare_paths(analysis: dict[str, Any]) -> dict[str, list[str]]: + compare = analysis.get("crawl_compare") if isinstance(analysis.get("crawl_compare"), dict) else {} + log_only = compare.get("log_only_paths") or analysis.get("log_only_paths") or [] + crawl_only = compare.get("crawl_only_paths") or analysis.get("crawl_only_paths") or [] + return { + "log_only_paths": [str(p) for p in log_only if p] if isinstance(log_only, list) else [], + "crawl_only_paths": [str(p) for p in crawl_only if p] if isinstance(crawl_only, list) else [], + } + + +def get_log_top_paths(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required"} + row = _load_log_analysis(conn, int(scoped.property_id)) + if not row: + return {"error": "no log uploads found", "missing": True, "paths": [], "total": 0, "truncated": False} + analysis = row.get("analysis") or {} + paths = analysis.get("top_paths") or [] + if not isinstance(paths, list): + paths = [] + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(paths, limit, max_cap=100) + return { + "paths": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "upload_id": row.get("upload_id"), + "filename": row.get("filename"), + } + + +def list_log_only_paths(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required"} + row = _load_log_analysis(conn, int(scoped.property_id)) + if not row: + return {"error": "no log uploads found", "missing": True, "paths": [], "total": 0, "truncated": False} + paths = _log_compare_paths(row.get("analysis") or {}).get("log_only_paths") or [] + limit = parse_limit(args.get("limit"), 50, 200) + items = [{"path": p} for p in paths] + sliced = cap_list(items, limit, max_cap=200) + return {"paths": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_crawl_only_paths(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required"} + row = _load_log_analysis(conn, int(scoped.property_id)) + if not row: + return {"error": "no log uploads found", "missing": True, "paths": [], "total": 0, "truncated": False} + paths = _log_compare_paths(row.get("analysis") or {}).get("crawl_only_paths") or [] + limit = parse_limit(args.get("limit"), 50, 200) + items = [{"path": p} for p in paths] + sliced = cap_list(items, limit, max_cap=200) + return {"paths": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def get_log_googlebot_stats(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required"} + row = _load_log_analysis(conn, int(scoped.property_id)) + if not row: + return {"error": "no log uploads found", "missing": True} + analysis = row.get("analysis") or {} + parsed = int(analysis.get("parsed_lines") or 0) + bot_hits = int(analysis.get("googlebot_hits") or 0) + ratio = round(bot_hits / parsed, 4) if parsed > 0 else None + return { + "upload_id": row.get("upload_id"), + "filename": row.get("filename"), + "parsed_lines": parsed, + "googlebot_hits": bot_hits, + "googlebot_ratio": ratio, + "unique_paths": analysis.get("unique_paths"), + "status_counts": analysis.get("status_counts"), + } diff --git a/src/website_profiling/tools/audit_tools/registry.py b/src/website_profiling/tools/audit_tools/registry.py index b482964f..a291e761 100644 --- a/src/website_profiling/tools/audit_tools/registry.py +++ b/src/website_profiling/tools/audit_tools/registry.py @@ -28,22 +28,41 @@ from .compare import compare_reports from .compare_slices import ( compare_category_deltas, + compare_content_metrics, + compare_duplicate_deltas, + compare_google_metrics, + compare_health_score_delta, compare_issue_deltas, compare_lighthouse_deltas, compare_link_metric_deltas, + compare_priority_counts, compare_redirect_deltas, + compare_security_deltas, compare_seo_health_deltas, + compare_tech_deltas, compare_url_set_diff, ) from .content import ( get_content_analytics, get_content_duplicates, + get_duplicate_cluster, get_keyword_opportunities, get_ner_site_summary, get_social_coverage, list_thin_content_pages, ) from .context import AuditToolContext +from .crawl_lists import ( + get_top_pages_by_pagerank, + list_canonical_mismatch, + list_long_redirect_chains, + list_pages_missing_canonical, + list_pages_missing_og_image, + list_pages_missing_viewport, + list_pages_skipped_headings, + list_pages_with_missing_alt, + list_robots_blocked_urls, +) from .crawl import ( get_browser_diagnostics_summary, get_crawl_links_table, @@ -96,9 +115,36 @@ get_lighthouse_for_url, get_lighthouse_human_summary, get_lighthouse_summary, + list_lighthouse_cwv_failures, + list_lighthouse_poor_accessibility_pages, + list_lighthouse_poor_best_practices_pages, list_lighthouse_poor_seo_pages, list_slow_pages, ) +from .export_tools import ( + compose_custom_report, + export_audit_report, + export_compare_csv, + export_custom_report, + export_list_as_csv, + list_export_formats, +) +from .image_tools import ( + get_image_audit_summary, + list_images_needing_attention, + list_largest_images, + list_lighthouse_image_opportunities, + list_pages_with_images_missing_dimensions, + list_pages_without_lazy_images, + list_site_image_urls, + list_unoptimized_images, +) +from .llm_tools import ( + expand_keywords, + generate_content_brief, + get_page_coach, + get_portfolio_summary, +) from .onpage import ( list_content_url_issues, list_pages_meta_desc_too_long, @@ -115,14 +161,20 @@ get_outbound_link_domains, get_top_linked_pages, get_url_fingerprints, + list_broken_link_sources, list_orphan_pages, ) from .ops import ( get_google_integration_status, get_integration_alerts, get_latest_log_analysis, + get_log_analysis_by_id, + get_log_googlebot_stats, + get_log_top_paths, get_property_ops, + list_crawl_only_paths, list_crawl_runs, + list_log_only_paths, list_log_uploads, ) from .properties import get_property, list_properties @@ -134,6 +186,7 @@ get_report_summary, get_site_level, list_issues, + search_issues, ) from .report_extras import ( get_audit_recommendations, @@ -144,8 +197,12 @@ list_issues_with_ai_fixes, ) from .schema import get_schema_coverage, list_pages_without_schema, search_pages_by_schema_type -from .security import get_security_findings -from .tech import get_tech_stack_summary +from .security import ( + get_security_findings, + get_security_findings_summary, + list_security_findings_by_type, +) +from .tech import get_tech_stack_summary, list_pages_by_technology from .tool_catalog import TOOL_DEFINITIONS from .workflow import list_issue_workflow @@ -275,6 +332,54 @@ "compare_url_set_diff": compare_url_set_diff, "compare_redirect_deltas": compare_redirect_deltas, "compare_link_metric_deltas": compare_link_metric_deltas, + "compare_security_deltas": compare_security_deltas, + "compare_duplicate_deltas": compare_duplicate_deltas, + "compare_tech_deltas": compare_tech_deltas, + "compare_content_metrics": compare_content_metrics, + "compare_google_metrics": compare_google_metrics, + "compare_priority_counts": compare_priority_counts, + "compare_health_score_delta": compare_health_score_delta, + "list_pages_missing_canonical": list_pages_missing_canonical, + "list_canonical_mismatch": list_canonical_mismatch, + "list_pages_with_missing_alt": list_pages_with_missing_alt, + "list_pages_skipped_headings": list_pages_skipped_headings, + "list_pages_missing_viewport": list_pages_missing_viewport, + "list_long_redirect_chains": list_long_redirect_chains, + "list_robots_blocked_urls": list_robots_blocked_urls, + "list_pages_missing_og_image": list_pages_missing_og_image, + "get_top_pages_by_pagerank": get_top_pages_by_pagerank, + "get_log_top_paths": get_log_top_paths, + "list_log_only_paths": list_log_only_paths, + "list_crawl_only_paths": list_crawl_only_paths, + "get_log_googlebot_stats": get_log_googlebot_stats, + "get_log_analysis_by_id": get_log_analysis_by_id, + "list_lighthouse_poor_accessibility_pages": list_lighthouse_poor_accessibility_pages, + "list_lighthouse_poor_best_practices_pages": list_lighthouse_poor_best_practices_pages, + "list_lighthouse_cwv_failures": list_lighthouse_cwv_failures, + "list_pages_by_technology": list_pages_by_technology, + "get_duplicate_cluster": get_duplicate_cluster, + "get_security_findings_summary": get_security_findings_summary, + "list_security_findings_by_type": list_security_findings_by_type, + "list_broken_link_sources": list_broken_link_sources, + "search_issues": search_issues, + "generate_content_brief": generate_content_brief, + "get_page_coach": get_page_coach, + "get_portfolio_summary": get_portfolio_summary, + "expand_keywords": expand_keywords, + "export_audit_report": export_audit_report, + "export_compare_csv": export_compare_csv, + "export_list_as_csv": export_list_as_csv, + "compose_custom_report": compose_custom_report, + "export_custom_report": export_custom_report, + "list_export_formats": list_export_formats, + "get_image_audit_summary": get_image_audit_summary, + "list_pages_without_lazy_images": list_pages_without_lazy_images, + "list_pages_with_images_missing_dimensions": list_pages_with_images_missing_dimensions, + "list_site_image_urls": list_site_image_urls, + "list_lighthouse_image_opportunities": list_lighthouse_image_opportunities, + "list_largest_images": list_largest_images, + "list_unoptimized_images": list_unoptimized_images, + "list_images_needing_attention": list_images_needing_attention, } diff --git a/src/website_profiling/tools/audit_tools/report.py b/src/website_profiling/tools/audit_tools/report.py index 15d1f000..9db5a0ba 100644 --- a/src/website_profiling/tools/audit_tools/report.py +++ b/src/website_profiling/tools/audit_tools/report.py @@ -134,6 +134,48 @@ def list_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) - } +def search_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """Search issues with combined filters including message substring.""" + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "issues": [], "total": 0, "truncated": False} + + limit = args.get("limit", _ISSUE_LIMIT_DEFAULT) + try: + limit = int(limit) + except (TypeError, ValueError): + limit = _ISSUE_LIMIT_DEFAULT + limit = max(1, min(limit, _ISSUE_LIMIT_MAX)) + + priority_filter = _normalize_priority(str(args.get("priority") or "")) + category_id = str(args.get("category_id") or "").strip() + url_contains = str(args.get("url_contains") or "").strip().lower() + message_contains = str(args.get("message_contains") or "").strip().lower() + + issues = _iter_category_issues(payload) + if priority_filter: + issues = [i for i in issues if i.get("priority") == priority_filter] + if category_id: + issues = [i for i in issues if i.get("category_id") == category_id] + if url_contains: + issues = [i for i in issues if url_contains in str(i.get("url") or "").lower()] + if message_contains: + issues = [ + i for i in issues + if message_contains in str(i.get("message") or "").lower() + or message_contains in str(i.get("recommendation") or "").lower() + ] + + total = len(issues) + truncated = total > limit + return { + "issues": issues[:limit], + "total": total, + "truncated": truncated, + } + + def get_critical_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: """All Critical-priority audit issues (chat table visualization).""" return list_issues(conn, ctx, {**args, "priority": "Critical"}) diff --git a/src/website_profiling/tools/audit_tools/security.py b/src/website_profiling/tools/audit_tools/security.py index 0730390f..59b10a72 100644 --- a/src/website_profiling/tools/audit_tools/security.py +++ b/src/website_profiling/tools/audit_tools/security.py @@ -26,3 +26,49 @@ def get_security_findings(conn: Connection, ctx: AuditToolContext, args: dict[st ] sliced = cap_list(findings, limit, max_cap=50) return {"findings": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def get_security_findings_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "summary": [], "total_findings": 0} + findings = payload.get("security_findings") or [] + if not isinstance(findings, list): + findings = [] + by_type: dict[str, dict[str, Any]] = {} + for f in findings: + if not isinstance(f, dict): + continue + ftype = str(f.get("finding_type") or "unknown") + entry = by_type.setdefault(ftype, {"finding_type": ftype, "count": 0, "severities": {}}) + entry["count"] += 1 + sev = str(f.get("severity") or "unknown") + entry["severities"][sev] = entry["severities"].get(sev, 0) + 1 + summary = sorted(by_type.values(), key=lambda x: x["count"], reverse=True) + return {"summary": summary, "total_findings": len(findings), "type_count": len(summary)} + + +def list_security_findings_by_type(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "findings": [], "total": 0, "truncated": False} + finding_type = str(args.get("finding_type") or "").strip().lower() + if not finding_type: + return {"error": "finding_type is required", "findings": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 50, 50) + findings = payload.get("security_findings") or [] + if not isinstance(findings, list): + findings = [] + filtered = [ + f for f in findings + if isinstance(f, dict) and str(f.get("finding_type") or "").lower() == finding_type + ] + sliced = cap_list(filtered, limit, max_cap=50) + return { + "finding_type": finding_type, + "findings": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + } diff --git a/src/website_profiling/tools/audit_tools/tech.py b/src/website_profiling/tools/audit_tools/tech.py index 9dcc8b7c..1354fbb8 100644 --- a/src/website_profiling/tools/audit_tools/tech.py +++ b/src/website_profiling/tools/audit_tools/tech.py @@ -1,12 +1,13 @@ """Technology stack tools.""" from __future__ import annotations +import json from typing import Any from psycopg import Connection from .context import AuditToolContext -from ._slice import payload_dict_slice +from ._slice import cap_list, parse_limit, payload_dict_slice def get_tech_stack_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: @@ -15,3 +16,48 @@ def get_tech_stack_summary(conn: Connection, ctx: AuditToolContext, args: dict[s if not payload: return {"error": "no report found"} return payload_dict_slice(payload, "tech_stack_summary") + + +def list_pages_by_technology(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + tech_name = str(args.get("technology_name") or "").strip().lower() + if not tech_name: + return {"error": "technology_name is required", "pages": [], "total": 0, "truncated": False} + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + pages: list[dict[str, Any]] = [] + summary = payload.get("tech_stack_summary") or {} + if isinstance(summary, dict): + for entry in summary.get("technologies") or []: + if not isinstance(entry, dict): + continue + if str(entry.get("name") or "").lower() == tech_name: + for url in entry.get("sample_urls") or []: + pages.append({"url": url, "technology": entry.get("name")}) + break + if not pages: + df = scoped.load_crawl_df(conn) + if df is not None and not df.empty and "tech_stack" in df.columns: + for _, row in df.iterrows(): + raw = row.get("tech_stack") or "[]" + try: + techs = json.loads(str(raw)) if isinstance(raw, str) else raw + except (json.JSONDecodeError, TypeError): + techs = [] + if not isinstance(techs, list): + continue + if any(str(t).lower() == tech_name for t in techs): + pages.append({ + "url": str(row.get("url") or ""), + "technology": tech_name, + "status": str(row.get("status") or ""), + }) + sliced = cap_list(pages, limit, max_cap=50) + return { + "technology_name": tech_name, + "pages": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + } diff --git a/src/website_profiling/tools/audit_tools/tool_catalog.py b/src/website_profiling/tools/audit_tools/tool_catalog.py index 452964ec..e0569afe 100644 --- a/src/website_profiling/tools/audit_tools/tool_catalog.py +++ b/src/website_profiling/tools/audit_tools/tool_catalog.py @@ -180,4 +180,117 @@ def _tool(name: str, description: str, properties: dict[str, Any], required: lis _tool("compare_url_set_diff", "URLs added or removed from crawl vs baseline.", {"baseline_report_id": _RID, "report_id": _RID, "limit": {"type": "integer", "maximum": 200}}, ["baseline_report_id"]), _tool("compare_redirect_deltas", "Redirect chain changes vs baseline.", {"baseline_report_id": _RID, "report_id": _RID, "limit": {"type": "integer", "maximum": 100}}, ["baseline_report_id"]), _tool("compare_link_metric_deltas", "Per-URL inlink/outlink/word-count changes vs baseline.", {"baseline_report_id": _RID, "report_id": _RID, "limit": {"type": "integer", "maximum": 200}}, ["baseline_report_id"]), + _tool("compare_security_deltas", "Security finding additions and resolutions vs baseline.", {"baseline_report_id": _RID, "report_id": _RID, "limit": _LIMIT}, ["baseline_report_id"]), + _tool("compare_duplicate_deltas", "Near-duplicate cluster changes vs baseline.", {"baseline_report_id": _RID, "report_id": _RID, "limit": _LIMIT}, ["baseline_report_id"]), + _tool("compare_tech_deltas", "Technology stack changes vs baseline.", {"baseline_report_id": _RID, "report_id": _RID, "limit": _LIMIT}, ["baseline_report_id"]), + _tool("compare_content_metrics", "Content and crawl metric changes vs baseline.", {"baseline_report_id": _RID, "report_id": _RID}, ["baseline_report_id"]), + _tool("compare_google_metrics", "GSC and GA4 metric changes vs baseline.", {"baseline_report_id": _RID, "report_id": _RID}, ["baseline_report_id"]), + _tool("compare_priority_counts", "Issue priority count changes vs baseline.", {"baseline_report_id": _RID, "report_id": _RID}, ["baseline_report_id"]), + _tool("compare_health_score_delta", "Overall health score change vs baseline.", {"baseline_report_id": _RID, "report_id": _RID}, ["baseline_report_id"]), + # Crawl list tools + _tool("list_pages_missing_canonical", "2xx pages with no canonical URL.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_canonical_mismatch", "Pages where canonical URL differs from crawled URL.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_with_missing_alt", "Pages with images missing alt text.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_skipped_headings", "Pages with skipped heading levels (e.g. H1 to H3).", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_missing_viewport", "Pages missing viewport meta tag.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_long_redirect_chains", "URLs with redirect chains of 2+ hops.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_robots_blocked_urls", "URLs blocked by robots.txt during crawl.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_missing_og_image", "Pages missing Open Graph image.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("get_top_pages_by_pagerank", "Top internal pages by crawl PageRank score.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + # Log analyzer slices + _tool("get_log_top_paths", "Top hit paths from latest access log analysis.", {"property_id": _PID, "limit": {"type": "integer", "maximum": 100}}, ["property_id"]), + _tool("list_log_only_paths", "Paths in access logs but not in crawl.", {"property_id": _PID, "limit": {"type": "integer", "maximum": 200}}, ["property_id"]), + _tool("list_crawl_only_paths", "Crawled paths not seen in access logs.", {"property_id": _PID, "limit": {"type": "integer", "maximum": 200}}, ["property_id"]), + _tool("get_log_googlebot_stats", "Googlebot hit counts and ratio from access log.", {"property_id": _PID}, ["property_id"]), + _tool("get_log_analysis_by_id", "Access log analysis for a specific upload_id.", {"property_id": _PID, "upload_id": {"type": "integer"}}, ["property_id", "upload_id"]), + # Lighthouse depth + _tool("list_lighthouse_poor_accessibility_pages", "Pages with Lighthouse accessibility below threshold.", {"property_id": _PID, "report_id": _RID, "accessibility_threshold": {"type": "integer"}, "limit": _LIMIT}), + _tool("list_lighthouse_poor_best_practices_pages", "Pages with Lighthouse best-practices below threshold.", {"property_id": _PID, "report_id": _RID, "best_practices_threshold": {"type": "integer"}, "limit": _LIMIT}), + _tool("list_lighthouse_cwv_failures", "Pages failing LCP, CLS, or TBT thresholds.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + # Drill-down + _tool("list_pages_by_technology", "Pages using a detected technology name.", {"property_id": _PID, "report_id": _RID, "technology_name": {"type": "string"}, "limit": _LIMIT}, ["technology_name"]), + _tool("get_duplicate_cluster", "Duplicate content cluster by index or member URL.", {"property_id": _PID, "report_id": _RID, "cluster_index": {"type": "integer"}, "url": _URL}), + _tool("get_security_findings_summary", "Security findings grouped by finding_type.", {"property_id": _PID, "report_id": _RID}), + _tool("list_security_findings_by_type", "Security findings filtered by finding_type.", {"property_id": _PID, "report_id": _RID, "finding_type": {"type": "string"}, "limit": _LIMIT}, ["finding_type"]), + _tool("list_broken_link_sources", "Pages linking to broken URLs.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("search_issues", "Search audit issues by message, URL, category, or priority.", {"property_id": _PID, "report_id": _RID, "message_contains": {"type": "string"}, "url_contains": {"type": "string"}, "category_id": {"type": "string"}, "priority": {"type": "string"}, "limit": _LIMIT}), + # LLM / portfolio + _tool("generate_content_brief", "Generate a content brief for a target keyword.", {"property_id": _PID, "keyword": {"type": "string"}, "gaps": {"type": "array", "items": {"type": "string"}}}, ["keyword"]), + _tool("get_page_coach", "LLM internal linking coach for one URL.", {"url": _URL, "property_id": _PID, "report_id": _RID, "refresh": {"type": "boolean"}}, ["url"]), + _tool("get_portfolio_summary", "Multi-property health score rollup.", {"limit": {"type": "integer", "maximum": 100}}), + _tool("expand_keywords", "Expand seed keywords via Google Suggest.", {"property_id": _PID, "seeds": {"type": "array", "items": {"type": "string"}}, "sources": {"type": "array", "items": {"type": "string"}}}, ["seeds"]), + # Export & deliverables + _tool( + "export_audit_report", + "Export full audit report as PDF, HTML, CSV, or JSON. Returns download artifact metadata.", + {"property_id": _PID, "report_id": _RID, "format": {"type": "string", "enum": ["pdf", "html", "csv", "json"]}}, + ), + _tool( + "export_compare_csv", + "Export issue added/removed CSV diff between current and baseline report.", + {"property_id": _PID, "report_id": _RID, "baseline_report_id": {"type": "integer"}}, + ["baseline_report_id"], + ), + _tool( + "export_list_as_csv", + "Export rows from an allowlisted list tool as CSV.", + { + "property_id": _PID, + "report_id": _RID, + "tool_name": {"type": "string"}, + "tool_args": {"type": "object"}, + "columns": {"type": "array", "items": {"type": "string"}}, + "limit": {"type": "integer", "maximum": 500}, + }, + ["tool_name"], + ), + _tool( + "compose_custom_report", + "Compose a multi-section custom report spec (tool, executive_summary, category_scores, notes sections).", + { + "property_id": _PID, + "report_id": _RID, + "title": {"type": "string"}, + "sections": {"type": "array", "items": {"type": "object"}}, + }, + ["title", "sections"], + ), + _tool( + "export_custom_report", + "Render a composed custom report to HTML or PDF.", + { + "property_id": _PID, + "report_id": _RID, + "report_spec_id": {"type": "string"}, + "title": {"type": "string"}, + "sections": {"type": "array", "items": {"type": "object"}}, + "format": {"type": "string", "enum": ["html", "pdf"]}, + }, + ), + _tool("list_export_formats", "List supported export tools, formats, and example prompts.", {}), + # Image audit + _tool("get_image_audit_summary", "Site-wide image audit totals: alt, lazy-load, dimensions, OG, Lighthouse image diagnostics.", {"property_id": _PID, "report_id": _RID}), + _tool("list_pages_without_lazy_images", "Pages with images not using loading=lazy.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_with_images_missing_dimensions", "Pages with images missing width/height (CLS risk).", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool( + "list_site_image_urls", + "Unique image URLs from crawl (on-page, OG, Twitter) with source page.", + {"property_id": _PID, "report_id": _RID, "kind": {"type": "string", "enum": ["content", "og", "twitter"]}, "limit": {"type": "integer", "maximum": 100}}, + ), + _tool("list_lighthouse_image_opportunities", "Lighthouse diagnostics related to images and LCP.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool( + "list_largest_images", + "Largest probed images by file size (requires probe_image_inventory on report build).", + {"property_id": _PID, "report_id": _RID, "min_size_kb": {"type": "integer"}, "limit": {"type": "integer", "maximum": 100}}, + ), + _tool( + "list_unoptimized_images", + "Large images not in WebP/AVIF (requires image inventory probe).", + {"property_id": _PID, "report_id": _RID, "min_size_kb": {"type": "integer"}, "limit": {"type": "integer", "maximum": 100}}, + ), + _tool( + "list_images_needing_attention", + "Ranked images/pages with composite attention reasons (size, format, alt/lazy/dimension issues).", + {"property_id": _PID, "report_id": _RID, "min_size_kb": {"type": "integer"}, "limit": {"type": "integer", "maximum": 100}}, + ), ] diff --git a/src/website_profiling/tools/export_artifacts.py b/src/website_profiling/tools/export_artifacts.py new file mode 100644 index 00000000..fd2c0151 --- /dev/null +++ b/src/website_profiling/tools/export_artifacts.py @@ -0,0 +1,215 @@ +"""Temporary export artifact store (DATA_DIR/exports) with TTL.""" +from __future__ import annotations + +import csv +import io +import json +import os +import re +import time +import uuid +from datetime import datetime, timezone +from typing import Any + +_ARTIFACT_ID_RE = re.compile(r"^[a-f0-9-]{36}$") +_SPEC_ID_RE = re.compile(r"^[a-f0-9-]{36}$") +_TTL_SECONDS = 24 * 60 * 60 +_INLINE_MAX_BYTES = 512 * 1024 +_LIST_ROW_KEYS = ( + "pages", + "items", + "paths", + "issues", + "issue_deltas", + "rows", + "keywords", + "queries", + "links", + "findings", + "technologies", + "clusters", + "deltas", + "results", +) + + +def _data_dir() -> str: + return (os.environ.get("DATA_DIR") or os.getcwd()).strip() or os.getcwd() + + +def exports_dir() -> str: + path = os.path.join(_data_dir(), "exports") + os.makedirs(path, exist_ok=True) + return path + + +def specs_dir() -> str: + path = os.path.join(exports_dir(), "specs") + os.makedirs(path, exist_ok=True) + return path + + +def _meta_path(artifact_id: str) -> str: + return os.path.join(exports_dir(), f"{artifact_id}.meta.json") + + +def _data_path(artifact_id: str) -> str: + return os.path.join(exports_dir(), f"{artifact_id}.bin") + + +def sweep_expired_artifacts() -> int: + """Remove artifacts older than TTL. Returns count removed.""" + root = exports_dir() + now = time.time() + removed = 0 + for name in os.listdir(root): + if not name.endswith(".meta.json"): + continue + meta_path = os.path.join(root, name) + try: + with open(meta_path, encoding="utf-8") as f: + meta = json.load(f) + created = float(meta.get("created_at_epoch") or 0) + if created and now - created > _TTL_SECONDS: + aid = meta.get("artifact_id") or name.replace(".meta.json", "") + delete_artifact(str(aid)) + removed += 1 + except (OSError, json.JSONDecodeError, TypeError, ValueError): + continue + return removed + + +def save_artifact( + data: bytes | str, + *, + filename: str, + mime_type: str, + meta: dict[str, Any] | None = None, +) -> dict[str, Any]: + sweep_expired_artifacts() + artifact_id = str(uuid.uuid4()) + if isinstance(data, str): + raw = data.encode("utf-8") + else: + raw = data + created = datetime.now(timezone.utc) + record: dict[str, Any] = { + "artifact_id": artifact_id, + "filename": filename, + "mime_type": mime_type, + "size_bytes": len(raw), + "created_at": created.isoformat(), + "created_at_epoch": created.timestamp(), + } + if meta: + record["extra"] = meta + with open(_meta_path(artifact_id), "w", encoding="utf-8") as f: + json.dump(record, f) + with open(_data_path(artifact_id), "wb") as f: + f.write(raw) + envelope = artifact_envelope(artifact_id, record) + if len(raw) <= _INLINE_MAX_BYTES and mime_type.startswith(("text/", "application/json")): + envelope["content"] = raw.decode("utf-8", errors="replace") + return envelope + + +def artifact_envelope(artifact_id: str, record: dict[str, Any]) -> dict[str, Any]: + return { + "artifact_id": artifact_id, + "filename": record.get("filename"), + "mime_type": record.get("mime_type"), + "size_bytes": record.get("size_bytes"), + "download_path": f"/api/chat/artifacts/{artifact_id}", + } + + +def read_artifact_meta(artifact_id: str) -> dict[str, Any] | None: + if not _ARTIFACT_ID_RE.match(artifact_id): + return None + path = _meta_path(artifact_id) + if not os.path.isfile(path): + return None + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def read_artifact_bytes(artifact_id: str) -> tuple[dict[str, Any], bytes] | None: + meta = read_artifact_meta(artifact_id) + if not meta: + return None + data_path = _data_path(artifact_id) + if not os.path.isfile(data_path): + return None + with open(data_path, "rb") as f: + return meta, f.read() + + +def delete_artifact(artifact_id: str) -> None: + for path in (_meta_path(artifact_id), _data_path(artifact_id)): + try: + if os.path.isfile(path): + os.remove(path) + except OSError: + pass + + +def save_report_spec(spec: dict[str, Any]) -> str: + spec_id = str(uuid.uuid4()) + spec["report_spec_id"] = spec_id + spec["created_at"] = datetime.now(timezone.utc).isoformat() + path = os.path.join(specs_dir(), f"{spec_id}.json") + with open(path, "w", encoding="utf-8") as f: + json.dump(spec, f) + return spec_id + + +def read_report_spec(spec_id: str) -> dict[str, Any] | None: + if not _SPEC_ID_RE.match(spec_id): + return None + path = os.path.join(specs_dir(), f"{spec_id}.json") + if not os.path.isfile(path): + return None + with open(path, encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else None + + +def rows_from_tool_result(result: dict[str, Any]) -> list[dict[str, Any]]: + if result.get("error"): + return [] + for key in _LIST_ROW_KEYS: + raw = result.get(key) + if isinstance(raw, list) and raw: + rows: list[dict[str, Any]] = [] + for item in raw: + if isinstance(item, dict): + rows.append(item) + elif item is not None: + rows.append({"value": item}) + if rows: + return rows + return [] + + +def dicts_to_csv(rows: list[dict[str, Any]], columns: list[str] | None = None) -> str: + if not rows: + return "" + if columns: + fieldnames = [c for c in columns if c] + else: + keys: list[str] = [] + seen: set[str] = set() + for row in rows: + for k in row: + if k not in seen: + seen.add(k) + keys.append(k) + fieldnames = keys + if not fieldnames: + return "" + buf = io.StringIO() + w = csv.DictWriter(buf, fieldnames=fieldnames, extrasaction="ignore") + w.writeheader() + for row in rows: + w.writerow({k: row.get(k, "") for k in fieldnames}) + return buf.getvalue() diff --git a/src/website_profiling/tools/export_compare.py b/src/website_profiling/tools/export_compare.py new file mode 100644 index 00000000..bf5d120e --- /dev/null +++ b/src/website_profiling/tools/export_compare.py @@ -0,0 +1,53 @@ +"""Compare report issue diff CSV export.""" +from __future__ import annotations + +import csv +import io +from typing import Any + + +def _issue_key(cat: str, issue: dict[str, Any]) -> str: + return f"{cat}|{issue.get('url') or ''}|{issue.get('message') or ''}" + + +def _collect_issues(payload: dict[str, Any]) -> dict[str, tuple[str, dict[str, Any]]]: + out: dict[str, tuple[str, dict[str, Any]]] = {} + for cat in payload.get("categories") or []: + if not isinstance(cat, dict): + continue + name = str(cat.get("name") or cat.get("id") or "") + for issue in cat.get("issues") or []: + if not isinstance(issue, dict): + continue + key = _issue_key(name, issue) + out[key] = (name, issue) + return out + + +def export_compare_issues_csv(current: dict[str, Any], baseline: dict[str, Any]) -> str: + issues_a = _collect_issues(current) + issues_b = _collect_issues(baseline) + buf = io.StringIO() + w = csv.writer(buf) + w.writerow(["change", "category", "priority", "url", "message", "recommendation"]) + for key, (cat, issue) in issues_a.items(): + if key not in issues_b: + w.writerow([ + "removed", + cat, + issue.get("priority") or "", + issue.get("url") or "", + issue.get("message") or "", + issue.get("recommendation") or "", + ]) + for key, (cat, issue) in issues_b.items(): + if key not in issues_a: + w.writerow([ + "added", + cat, + issue.get("priority") or "", + issue.get("url") or "", + issue.get("message") or "", + issue.get("recommendation") or "", + ]) + return buf.getvalue() diff --git a/src/website_profiling/tools/export_custom.py b/src/website_profiling/tools/export_custom.py new file mode 100644 index 00000000..95208e56 --- /dev/null +++ b/src/website_profiling/tools/export_custom.py @@ -0,0 +1,248 @@ +"""Custom composed report HTML/PDF builder.""" +from __future__ import annotations + +import html +import io +import re +from typing import Any, Callable + +from psycopg import Connection + +from .export_artifacts import rows_from_tool_result +from .export_audit import ( + _category_cards_html, + _executive_export_data, + _executive_source_label, + _executive_summary_html, + _format_report_date, + _overall_score, + _report_html_styles, +) + +_MAX_SECTIONS = 12 +_NOTES_MAX_LEN = 8000 + +_SECTION_TABLE_KEYS = ( + "pages", + "items", + "paths", + "issues", + "issue_deltas", + "rows", + "keywords", + "queries", + "links", + "findings", + "deltas", +) + + +def _sanitize_notes(text: str) -> str: + cleaned = re.sub(r"]*>.*?", "", text, flags=re.I | re.S) + cleaned = cleaned.replace("<", "<").replace(">", ">") + return cleaned[:_NOTES_MAX_LEN] + + +def _table_from_rows(rows: list[dict[str, Any]], max_rows: int = 50) -> str: + if not rows: + return '

No data.

' + sample = rows[:max_rows] + keys: list[str] = [] + seen: set[str] = set() + for row in sample: + for k in row: + if k not in seen: + seen.add(k) + keys.append(k) + if not keys: + return '

No columns.

' + head = "".join(f"{html.escape(k)}" for k in keys[:8]) + body_rows = [] + for row in sample: + cells = "".join( + f"{html.escape(str(row.get(k, ''))[:500])}" for k in keys[:8] + ) + body_rows.append(f"{cells}") + note = "" + if len(rows) > max_rows: + note = f'

Showing {max_rows} of {len(rows)} rows.

' + return ( + f'
{head}' + f"{''.join(body_rows)}
{note}" + ) + + +def _section_html_tool_result(heading: str, result: dict[str, Any]) -> str: + h = html.escape(heading) + if result.get("error"): + return f"

{h}

{html.escape(str(result['error']))}

" + rows = rows_from_tool_result(result) + if rows: + return f"

{h}

{_table_from_rows(rows)}
" + for key in _SECTION_TABLE_KEYS: + raw = result.get(key) + if isinstance(raw, list) and raw and isinstance(raw[0], dict): + return f"

{h}

{_table_from_rows(raw)}
" + preview = html.escape(str(result)[:2000]) + return f"

{h}

{preview}
" + + +def _section_html_executive(payload: dict[str, Any]) -> str: + return f"

Executive summary

{_executive_summary_html(payload)}
" + + +def _section_html_categories(payload: dict[str, Any]) -> str: + cards = _category_cards_html(payload.get("categories") or []) + overall = _overall_score(payload) + score_txt = str(overall) if overall is not None else "—" + return ( + f'

Category scores

' + f'

Overall health: {html.escape(score_txt)}/100

' + f'
{cards}
' + ) + + +def _section_html_notes(heading: str, markdown: str) -> str: + body = _sanitize_notes(markdown).replace("\n", "
") + return f"

{html.escape(heading)}

{body}
" + + +def render_custom_report_html( + *, + title: str, + payload: dict[str, Any], + sections: list[dict[str, Any]], + section_results: list[dict[str, Any] | None], +) -> str: + site = html.escape(str(payload.get("site_name") or "Site Audit")) + generated = html.escape(_format_report_date(str(payload.get("report_generated_at") or ""))) + title_esc = html.escape(title) + parts: list[str] = [] + for section, result in zip(sections, section_results): + stype = str(section.get("type") or "") + if stype == "executive_summary": + parts.append(_section_html_executive(payload)) + elif stype == "category_scores": + parts.append(_section_html_categories(payload)) + elif stype == "notes": + parts.append(_section_html_notes( + str(section.get("heading") or "Notes"), + str(section.get("markdown") or ""), + )) + elif stype == "tool" and result is not None: + parts.append(_section_html_tool_result( + str(section.get("heading") or section.get("tool_name") or "Section"), + result, + )) + body = "\n".join(parts) + styles = _report_html_styles() + return f""" + + + + {title_esc} — {site} + + + +
+
+
Site Audit
+

{title_esc}

+

{site} · {generated}

+
+
{body}
+
+ +""" + + +def render_custom_report_pdf(html_doc: str, title: str) -> bytes: + try: + from reportlab.lib import colors + from reportlab.lib.pagesizes import letter + from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle + from reportlab.lib.units import inch + from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer + except ImportError as exc: + raise RuntimeError("PDF export requires reportlab (pip install reportlab)") from exc + + buf = io.BytesIO() + doc = SimpleDocTemplate(buf, pagesize=letter, topMargin=0.55 * inch, bottomMargin=0.55 * inch) + styles = getSampleStyleSheet() + title_style = ParagraphStyle( + "CustomTitle", + parent=styles["Heading1"], + fontSize=18, + textColor=colors.HexColor("#0f172a"), + ) + story: list[Any] = [ + Paragraph(html.escape(title), title_style), + Spacer(1, 0.2 * inch), + Paragraph( + "Custom report generated from selected audit sections. " + "Open the HTML export for full tables and formatting.", + styles["Normal"], + ), + ] + text = re.sub(r"<[^>]+>", " ", html_doc) + text = re.sub(r"\s+", " ", text).strip() + chunk_size = 3000 + for i in range(0, min(len(text), 12000), chunk_size): + story.append(Paragraph(html.escape(text[i : i + chunk_size]), styles["Normal"])) + story.append(Spacer(1, 0.1 * inch)) + doc.build(story) + return buf.getvalue() + + +def validate_sections(sections: Any) -> tuple[list[dict[str, Any]] | None, str | None]: + if not isinstance(sections, list) or not sections: + return None, "sections must be a non-empty array" + if len(sections) > _MAX_SECTIONS: + return None, f"sections max {_MAX_SECTIONS}" + normalized: list[dict[str, Any]] = [] + for raw in sections: + if not isinstance(raw, dict): + return None, "each section must be an object" + stype = str(raw.get("type") or "") + if stype == "tool": + if not raw.get("tool_name"): + return None, "tool sections require tool_name" + normalized.append(raw) + elif stype in ("executive_summary", "category_scores"): + normalized.append({"type": stype}) + elif stype == "notes": + if not raw.get("markdown"): + return None, "notes sections require markdown" + normalized.append(raw) + else: + return None, f"unknown section type: {stype}" + return normalized, None + + +def resolve_section_results( + conn: Connection, + ctx: Any, + payload: dict[str, Any], + sections: list[dict[str, Any]], + dispatch_fn: Callable[..., dict[str, Any]], +) -> list[dict[str, Any] | None]: + results: list[dict[str, Any] | None] = [] + for section in sections: + stype = section.get("type") + if stype in ("executive_summary", "category_scores", "notes"): + results.append(None) + continue + if stype == "tool": + tool_args = dict(section.get("tool_args") or {}) + if ctx.property_id is not None and "property_id" not in tool_args: + tool_args["property_id"] = ctx.property_id + if ctx.report_id is not None and "report_id" not in tool_args: + tool_args["report_id"] = ctx.report_id + results.append(dispatch_fn(str(section["tool_name"]), tool_args, context=ctx, conn=conn)) + continue + results.append(None) + return results diff --git a/tests/test_audit_tools_expanded.py b/tests/test_audit_tools_expanded.py index 00fbd9e4..83e51408 100644 --- a/tests/test_audit_tools_expanded.py +++ b/tests/test_audit_tools_expanded.py @@ -161,7 +161,7 @@ def conn() -> MagicMock: def test_handler_schema_parity() -> None: names = {t["name"] for t in TOOL_DEFINITIONS} assert names == tool_handler_names() - assert len(TOOL_DEFINITIONS) == 123 + assert len(TOOL_DEFINITIONS) == 171 def test_slice_helpers() -> None: @@ -408,3 +408,140 @@ def test_compare_reports(conn: MagicMock, ctx: AuditToolContext) -> None: side_effect=[None, baseline], ): assert "not found" in dispatch_tool("compare_reports", {"baseline_report_id": 1}, context=ctx, conn=conn)["error"] + + +def test_new_gap_closure_tools(conn: MagicMock, ctx: AuditToolContext) -> None: + payload = _full_payload() + payload["content_urls"] = { + **payload.get("content_urls", {}), + "missing_canonical": [{"url": "https://ex.com/a", "title": "A"}], + "canonical_mismatch": [{"url": "https://ex.com/b", "canonical_url": "https://ex.com/other", "title": "B"}], + "missing_alt": [{"url": "https://ex.com/c", "images_without_alt": 2, "images_total": 3}], + } + payload["social_coverage"] = {"og_image_missing": ["https://ex.com/d"]} + payload["security_findings"] = [ + {"url": "https://ex.com", "severity": "High", "finding_type": "hsts", "message": "missing"}, + ] + payload["lighthouse_by_url"] = { + "https://ex.com/slow": { + "median_metrics": {"lcp_ms": 4000, "cls": 0.2, "tbt_ms": 300, "accessibility_score": 40, "best_practices_score": 45}, + "accessibility": 40, + "best-practices": 45, + }, + } + payload["graph_edges"] = [{"from": "https://ex.com/src", "to": "https://ex.com/broken"}] + payload["issues"] = {"broken": [{"url": "https://ex.com/broken", "status": "404"}]} + payload["tech_stack_summary"] = {"technologies": [{"name": "WordPress", "count": 2, "sample_urls": ["https://ex.com/wp"]}]} + df = pd.DataFrame([ + { + "url": "https://ex.com/a", + "status": "200", + "canonical_url": "", + "images_without_alt": 1, + "images_total": 2, + "heading_sequence": "h1,h3", + "viewport_present": "false", + "redirect_chain_length": 2, + "og_image": "", + "pagerank": 0.5, + "tech_stack": '["WordPress"]', + }, + ]) + log_row = { + "upload_id": 1, + "filename": "access.log", + "line_count": 100, + "analysis": { + "top_paths": [{"path": "/hot", "hits": 50}], + "parsed_lines": 100, + "googlebot_hits": 10, + "crawl_compare": {"log_only_paths": ["/log-only"], "crawl_only_paths": ["/crawl-only"]}, + }, + } + with patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df): + assert dispatch_tool("list_pages_missing_canonical", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_canonical_mismatch", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_pages_with_missing_alt", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_pages_without_lazy_images", {}, context=ctx, conn=conn)["total"] >= 0 + assert dispatch_tool("list_pages_with_images_missing_dimensions", {}, context=ctx, conn=conn)["total"] >= 0 + assert dispatch_tool("get_image_audit_summary", {}, context=ctx, conn=conn)["pages_missing_alt"] >= 0 + assert dispatch_tool("list_pages_skipped_headings", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_pages_missing_viewport", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_long_redirect_chains", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_pages_missing_og_image", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("get_top_pages_by_pagerank", {}, context=ctx, conn=conn)["total"] >= 0 + assert dispatch_tool("get_security_findings_summary", {}, context=ctx, conn=conn)["total_findings"] == 1 + assert dispatch_tool("list_security_findings_by_type", {"finding_type": "hsts"}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_broken_link_sources", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_pages_by_technology", {"technology_name": "WordPress"}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("get_duplicate_cluster", {"cluster_index": 0}, context=ctx, conn=conn)["cluster_index"] == 0 + assert dispatch_tool("search_issues", {"message_contains": "title"}, context=ctx, conn=conn)["total"] >= 0 + assert dispatch_tool("list_lighthouse_poor_accessibility_pages", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_lighthouse_poor_best_practices_pages", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_lighthouse_cwv_failures", {}, context=ctx, conn=conn)["total"] == 1 + with patch("website_profiling.tools.audit_tools.ops._load_log_analysis", return_value=log_row): + assert dispatch_tool("get_log_top_paths", {"property_id": 1}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_log_only_paths", {"property_id": 1}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_crawl_only_paths", {"property_id": 1}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("get_log_googlebot_stats", {"property_id": 1}, context=ctx, conn=conn)["googlebot_hits"] == 10 + with patch("website_profiling.tools.audit_tools.compare_helpers.read_report_payload", return_value=payload): + assert "security_deltas" in dispatch_tool("compare_security_deltas", {"baseline_report_id": 1}, context=ctx, conn=conn) + assert "health_score" in dispatch_tool("compare_health_score_delta", {"baseline_report_id": 1}, context=ctx, conn=conn) + with patch("website_profiling.tools.audit_tools.llm_tools.list_properties_public", return_value=[{"id": 1, "name": "ex.com", "canonical_domain": "ex.com"}]): + conn.execute = MagicMock(return_value=MagicMock(fetchone=MagicMock(return_value={"health_score": 80, "generated_at": datetime.now(timezone.utc), "report_id": 1, "issue_counts": "{}"}))) + assert dispatch_tool("get_portfolio_summary", {}, conn=conn)["count"] == 1 + with patch("website_profiling.tools.audit_tools.llm_tools.batch_expand", return_value={"widgets": {"web": ["widgets near me"]}}): + assert dispatch_tool("expand_keywords", {"seeds": ["widgets"]}, context=ctx, conn=conn)["seed_count"] == 1 + assert dispatch_tool("generate_content_brief", {"keyword": "widgets"}, context=ctx, conn=conn)["brief"]["keyword"] == "widgets" + + +def test_export_tools(conn: MagicMock, ctx: AuditToolContext, tmp_path, monkeypatch) -> None: + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + payload = _full_payload() + with patch.object(Ctx, "load_payload", return_value=payload), patch( + "website_profiling.tools.audit_tools.export_tools.export_audit_csv", + return_value="url,status\nhttps://ex.com,200\n", + ): + out = dispatch_tool("export_audit_report", {"format": "csv"}, context=ctx, conn=conn) + assert out.get("artifact_id") + assert out.get("filename", "").endswith(".csv") + formats = dispatch_tool("list_export_formats", {}, context=ctx, conn=conn) + assert formats.get("formats") + with patch.object(Ctx, "load_payload", return_value=payload), patch( + "website_profiling.tools.audit_tools.export_tools._dispatch", + return_value={"pages": [{"url": "https://ex.com/broken", "status": "404"}], "total": 1, "truncated": False}, + ): + csv_out = dispatch_tool( + "export_list_as_csv", + {"tool_name": "list_broken_links", "tool_args": {}}, + context=ctx, + conn=conn, + ) + assert csv_out.get("artifact_id") + assert csv_out.get("total") == 1 + with patch.object(Ctx, "load_payload", return_value=payload): + spec = dispatch_tool( + "compose_custom_report", + { + "title": "Client", + "sections": [{"type": "category_scores"}, {"type": "notes", "heading": "N", "markdown": "Hi"}], + }, + context=ctx, + conn=conn, + ) + assert spec.get("report_spec_id") + with patch( + "website_profiling.tools.audit_tools.export_tools.resolve_section_results", + return_value=[None, None], + ): + html_out = dispatch_tool( + "export_custom_report", + {"report_spec_id": spec["report_spec_id"], "format": "html"}, + context=ctx, + conn=conn, + ) + assert html_out.get("artifact_id") + with patch("website_profiling.tools.audit_tools.export_tools.load_compare_pair") as mock_pair: + mock_pair.return_value = (payload, payload, 2, 1, None) + cmp_out = dispatch_tool("export_compare_csv", {"baseline_report_id": 1}, context=ctx, conn=conn) + assert cmp_out.get("artifact_id") diff --git a/tests/test_builder_image_buckets.py b/tests/test_builder_image_buckets.py new file mode 100644 index 00000000..43f6a303 --- /dev/null +++ b/tests/test_builder_image_buckets.py @@ -0,0 +1,55 @@ +"""Tests for builder image inventory and ref collection.""" +from __future__ import annotations + +from unittest.mock import patch + +from website_profiling.analysis.image_probe import collect_image_refs_from_links +from website_profiling.reporting.builder import _build_image_inventory + + +def test_build_image_inventory_disabled() -> None: + links = [{"url": "https://ex.com/", "page_analysis": {"image_urls": ["https://cdn.ex.com/a.png"]}}] + inventory, summary = _build_image_inventory(links, {"probe_image_inventory": "false"}) + assert inventory == [] + assert summary["inventory_available"] is False + assert summary["probed"] == 0 + + +def test_build_image_inventory_with_probe_mock() -> None: + links = [ + { + "url": "https://ex.com/page", + "page_analysis": {"image_urls": ["https://cdn.ex.com/heavy.png"]}, + }, + ] + probed = [ + { + "url": "https://cdn.ex.com/heavy.png", + "status": 200, + "content_type": "image/png", + "size_bytes": 300_000, + "error": None, + }, + ] + with patch("website_profiling.analysis.image_probe.probe_image_urls", return_value=probed): + inventory, summary = _build_image_inventory( + links, + { + "probe_image_inventory": "true", + "max_image_probe_urls": "500", + "image_unoptimized_min_kb": "200", + }, + ) + assert summary["inventory_available"] is True + assert summary["probed"] == 1 + assert summary["over_threshold_count"] == 1 + assert len(inventory) == 1 + assert inventory[0]["source_pages"] == ["https://ex.com/page"] + assert inventory[0]["kinds"] == ["content"] + + +def test_collect_refs_skips_invalid() -> None: + refs = collect_image_refs_from_links([ + {"url": "https://ex.com/", "page_analysis": {"image_urls": ["data:image/png;base64,x"]}}, + ]) + assert refs == {} diff --git a/tests/test_config_schema_keys.py b/tests/test_config_schema_keys.py index 24533f08..a000d69a 100644 --- a/tests/test_config_schema_keys.py +++ b/tests/test_config_schema_keys.py @@ -41,6 +41,11 @@ "run_security_scan", "security_scan_active", "security_max_urls_probe", + "probe_image_inventory", + "max_image_probe_urls", + "image_probe_concurrency", + "image_probe_timeout", + "image_unoptimized_min_kb", "lighthouse_url", "lighthouse_mode", "lighthouse_strategy", diff --git a/tests/test_export_artifacts.py b/tests/test_export_artifacts.py new file mode 100644 index 00000000..c2ac3c99 --- /dev/null +++ b/tests/test_export_artifacts.py @@ -0,0 +1,66 @@ +"""Tests for export artifact store.""" +from __future__ import annotations + +import json +import os +import time + +import pytest + +from website_profiling.tools import export_artifacts + + +@pytest.fixture +def artifact_dir(tmp_path, monkeypatch): + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + return tmp_path + + +def test_save_and_read_artifact(artifact_dir) -> None: + env = export_artifacts.save_artifact(b"hello", filename="test.txt", mime_type="text/plain") + assert env["artifact_id"] + assert env["size_bytes"] == 5 + assert env["download_path"].startswith("/api/chat/artifacts/") + loaded = export_artifacts.read_artifact_bytes(env["artifact_id"]) + assert loaded is not None + meta, data = loaded + assert meta["filename"] == "test.txt" + assert data == b"hello" + + +def test_inline_content_for_small_text(artifact_dir) -> None: + env = export_artifacts.save_artifact("a,b\n1,2", filename="t.csv", mime_type="text/csv") + assert env.get("content") == "a,b\n1,2" + + +def test_read_missing_artifact(artifact_dir) -> None: + assert export_artifacts.read_artifact_bytes("00000000-0000-0000-0000-000000000000") is None + + +def test_dicts_to_csv() -> None: + csv_text = export_artifacts.dicts_to_csv( + [{"url": "https://ex.com", "status": "404"}], + columns=["url", "status"], + ) + assert "url,status" in csv_text + assert "https://ex.com" in csv_text + + +def test_save_report_spec(artifact_dir) -> None: + spec_id = export_artifacts.save_report_spec({"title": "T", "sections": []}) + spec = export_artifacts.read_report_spec(spec_id) + assert spec is not None + assert spec["title"] == "T" + + +def test_sweep_expired_artifacts(artifact_dir) -> None: + env = export_artifacts.save_artifact(b"x", filename="old.bin", mime_type="application/octet-stream") + meta_path = os.path.join(export_artifacts.exports_dir(), f"{env['artifact_id']}.meta.json") + with open(meta_path, encoding="utf-8") as f: + meta = json.load(f) + meta["created_at_epoch"] = time.time() - export_artifacts._TTL_SECONDS - 10 + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + removed = export_artifacts.sweep_expired_artifacts() + assert removed >= 1 + assert export_artifacts.read_artifact_bytes(env["artifact_id"]) is None diff --git a/tests/test_export_compare.py b/tests/test_export_compare.py new file mode 100644 index 00000000..98402218 --- /dev/null +++ b/tests/test_export_compare.py @@ -0,0 +1,32 @@ +"""Tests for compare CSV export.""" +from __future__ import annotations + +from website_profiling.tools.export_compare import export_compare_issues_csv + + +def test_export_compare_issues_csv_added_removed() -> None: + current = { + "categories": [ + { + "name": "Tech", + "issues": [ + {"url": "https://ex.com/a", "message": "gone", "priority": "High", "recommendation": "fix"}, + ], + }, + ], + } + baseline = { + "categories": [ + { + "name": "Tech", + "issues": [ + {"url": "https://ex.com/b", "message": "new", "priority": "Medium", "recommendation": "fix b"}, + ], + }, + ], + } + csv_text = export_compare_issues_csv(current, baseline) + assert "removed" in csv_text + assert "added" in csv_text + assert "https://ex.com/a" in csv_text + assert "https://ex.com/b" in csv_text diff --git a/tests/test_export_custom.py b/tests/test_export_custom.py new file mode 100644 index 00000000..b0cd0ff3 --- /dev/null +++ b/tests/test_export_custom.py @@ -0,0 +1,50 @@ +"""Tests for custom report builder.""" +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from website_profiling.tools.export_custom import ( + render_custom_report_html, + render_custom_report_pdf, + validate_sections, +) + + +def test_validate_sections_ok() -> None: + sections, err = validate_sections([ + {"type": "executive_summary"}, + {"type": "notes", "heading": "Summary", "markdown": "Hello"}, + {"type": "tool", "heading": "Broken", "tool_name": "list_broken_links", "tool_args": {}}, + ]) + assert err is None + assert sections is not None + assert len(sections) == 3 + + +def test_validate_sections_rejects_unknown_type() -> None: + _, err = validate_sections([{"type": "unknown"}]) + assert err is not None + + +def test_render_custom_report_html() -> None: + payload = {"site_name": "Example", "report_generated_at": "2026-06-07T12:00:00Z", "categories": []} + html_doc = render_custom_report_html( + title="Client Report", + payload=payload, + sections=[{"type": "notes", "heading": "Notes", "markdown": "Line one"}], + section_results=[None], + ) + assert "Client Report" in html_doc + assert "Example" in html_doc + assert "Line one" in html_doc + + +def test_render_custom_report_pdf_smoke() -> None: + html_doc = "

Test

" + try: + pdf = render_custom_report_pdf(html_doc, "Test") + except RuntimeError as exc: + pytest.skip(str(exc)) + assert pdf[:4] == b"%PDF" diff --git a/tests/test_image_probe.py b/tests/test_image_probe.py new file mode 100644 index 00000000..10307521 --- /dev/null +++ b/tests/test_image_probe.py @@ -0,0 +1,203 @@ +"""Tests for image URL probing.""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from website_profiling.analysis.image_probe import ( + _PARTIAL_GET_CAP, + _USER_AGENT, + _parse_size, + _probe_one, + collect_image_refs_from_links, + probe_image_urls, +) + + +def test_collect_image_refs_dedupes_and_kinds() -> None: + links = [ + { + "url": "https://ex.com/a", + "og_image": "https://cdn.ex.com/og.png", + "twitter_image": "https://cdn.ex.com/og.png", + "page_analysis": {"image_urls": ["https://cdn.ex.com/hero.jpg", "https://cdn.ex.com/hero.jpg"]}, + }, + { + "url": "https://ex.com/b", + "page_analysis": {"image_urls": ["https://cdn.ex.com/hero.jpg"]}, + }, + ] + refs = collect_image_refs_from_links(links) + assert "https://cdn.ex.com/hero.jpg" in refs + assert refs["https://cdn.ex.com/hero.jpg"]["kinds"] == {"content"} + assert set(refs["https://cdn.ex.com/hero.jpg"]["source_pages"]) == {"https://ex.com/a", "https://ex.com/b"} + assert refs["https://cdn.ex.com/og.png"]["kinds"] == {"og", "twitter"} + + +def test_collect_image_refs_skips_invalid_links() -> None: + refs = collect_image_refs_from_links([ + "not-a-dict", + {"url": "", "page_analysis": {"image_urls": ["https://cdn.ex.com/x.png"]}}, + {"url": "https://ex.com/ok", "og_image": "https://cdn.ex.com/og.png"}, + ]) + assert list(refs) == ["https://cdn.ex.com/og.png"] + + +def test_parse_size_invalid_content_length() -> None: + assert _parse_size({"Content-Length": "not-a-number"}) is None + assert _parse_size({}) is None + assert _parse_size({"content-length": "42"}) == 42 + + +def test_probe_skips_data_and_non_http() -> None: + urls = ["data:image/png;base64,abc", "ftp://ex.com/x.png", "https://ex.com/a.png"] + session = MagicMock() + session.head.return_value = MagicMock( + status_code=200, + headers={"Content-Type": "image/png", "Content-Length": "100"}, + ) + with patch("website_profiling.analysis.image_probe.requests.Session", return_value=session): + results = probe_image_urls(urls, concurrency=1, timeout=5, session=session) + assert len(results) == 1 + assert results[0]["url"] == "https://ex.com/a.png" + assert results[0]["size_bytes"] == 100 + + +def test_probe_dedupes_urls() -> None: + session = MagicMock() + session.head.return_value = MagicMock( + status_code=200, + headers={"Content-Type": "image/jpeg", "Content-Length": "50"}, + ) + urls = ["https://ex.com/x.jpg", "https://ex.com/x.jpg#frag"] + results = probe_image_urls(urls, concurrency=1, timeout=5, session=session) + assert len(results) == 1 + assert session.head.call_count == 1 + + +def test_probe_get_fallback_on_405() -> None: + session = MagicMock() + head_resp = MagicMock(status_code=405, headers={}) + get_resp = MagicMock( + status_code=200, + headers={"Content-Type": "image/png"}, + iter_content=lambda chunk_size=8192: [b"x" * 100], + ) + get_resp.close = MagicMock() + session.head.return_value = head_resp + session.get.return_value = get_resp + results = probe_image_urls(["https://ex.com/img.png"], concurrency=1, timeout=5, session=session) + assert len(results) == 1 + assert results[0]["status"] == 200 + assert results[0]["size_bytes"] == 100 + session.get.assert_called_once() + + +def test_probe_get_fallback_counts_bytes_when_no_content_length() -> None: + session = MagicMock() + session.head.return_value = MagicMock(status_code=501, headers={}) + get_resp = MagicMock( + status_code=200, + headers={"Content-Type": "image/jpeg"}, + iter_content=lambda chunk_size=8192: [b"a" * 10], + ) + get_resp.close = MagicMock() + session.get.return_value = get_resp + result = _probe_one("https://ex.com/no-cl.jpg", timeout=5, session=session) + assert result["size_bytes"] == 10 + + +def test_probe_get_fallback_empty_chunk_stops_read() -> None: + session = MagicMock() + session.head.return_value = MagicMock(status_code=501, headers={}) + get_resp = MagicMock( + status_code=200, + headers={"Content-Type": "image/jpeg"}, + iter_content=lambda chunk_size=8192: [b"", b"ignored"], + ) + get_resp.close = MagicMock() + session.get.return_value = get_resp + result = _probe_one("https://ex.com/empty-chunk.jpg", timeout=5, session=session) + assert result["size_bytes"] is None + + +def test_probe_get_fallback_hits_partial_cap() -> None: + session = MagicMock() + session.head.return_value = MagicMock(status_code=403, headers={}) + big = b"x" * (_PARTIAL_GET_CAP + 1) + get_resp = MagicMock( + status_code=200, + headers={"Content-Type": "image/png"}, + iter_content=lambda chunk_size=8192: [big], + ) + get_resp.close = MagicMock() + session.get.return_value = get_resp + result = _probe_one("https://ex.com/big.png", timeout=5, session=session) + assert result["size_bytes"] == _PARTIAL_GET_CAP + 1 + + +def test_probe_head_200_streams_when_no_content_length() -> None: + session = MagicMock() + session.head.return_value = MagicMock(status_code=200, headers={"Content-Type": "image/webp"}) + get_resp = MagicMock( + status_code=200, + headers={"Content-Type": "image/webp"}, + iter_content=lambda chunk_size=8192: [b"webp-bytes"], + ) + get_resp.close = MagicMock() + session.get.return_value = get_resp + result = _probe_one("https://ex.com/stream.webp", timeout=5, session=session) + assert result["size_bytes"] == len(b"webp-bytes") + session.get.assert_called_once() + + +def test_probe_head_200_partial_get_cap() -> None: + session = MagicMock() + session.head.return_value = MagicMock(status_code=200, headers={}) + huge = b"y" * (_PARTIAL_GET_CAP + 500) + get_resp = MagicMock( + status_code=200, + headers={}, + iter_content=lambda chunk_size=8192: [huge], + ) + get_resp.close = MagicMock() + session.get.return_value = get_resp + result = _probe_one("https://ex.com/huge.jpg", timeout=5, session=session) + assert result["size_bytes"] == _PARTIAL_GET_CAP + 500 + + +def test_probe_head_200_empty_stream_chunk() -> None: + session = MagicMock() + session.head.return_value = MagicMock(status_code=200, headers={"Content-Type": "image/png"}) + get_resp = MagicMock( + status_code=200, + headers={"Content-Type": "image/png"}, + iter_content=lambda chunk_size=8192: [b""], + ) + get_resp.close = MagicMock() + session.get.return_value = get_resp + result = _probe_one("https://ex.com/empty-stream.png", timeout=5, session=session) + assert result["size_bytes"] is None + session.get.assert_called_once() + + +def test_probe_records_error() -> None: + session = MagicMock() + session.head.side_effect = TimeoutError("timed out") + results = probe_image_urls(["https://ex.com/fail.png"], concurrency=1, timeout=1, session=session) + assert results[0]["error"] + assert results[0]["size_bytes"] is None + + +def test_probe_creates_and_closes_own_session() -> None: + mock_session = MagicMock() + mock_session.head.return_value = MagicMock( + status_code=200, + headers={"Content-Type": "image/png", "Content-Length": "1"}, + ) + with patch("website_profiling.analysis.image_probe.requests.Session", return_value=mock_session) as session_cls: + results = probe_image_urls(["https://ex.com/a.png"], concurrency=1, timeout=5) + session_cls.assert_called_once() + mock_session.headers.update.assert_called_once_with({"User-Agent": _USER_AGENT}) + mock_session.close.assert_called_once() + assert results[0]["size_bytes"] == 1 + diff --git a/tests/test_image_tools.py b/tests/test_image_tools.py new file mode 100644 index 00000000..72611529 --- /dev/null +++ b/tests/test_image_tools.py @@ -0,0 +1,149 @@ +"""Tests for image audit tools.""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from website_profiling.tools.audit_tools import dispatch_tool +from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx +from website_profiling.tools.audit_tools.image_tools import IMAGE_LIGHTHOUSE_AUDIT_IDS + + +@pytest.fixture +def ctx() -> Ctx: + return Ctx(property_id=1, report_id=1) + + +@pytest.fixture +def conn() -> MagicMock: + return MagicMock() + + +def _payload() -> dict: + return { + "social_coverage": { + "og_image_coverage_pct": 80, + "og_image_missing": ["https://ex.com/no-og"], + }, + "content_urls": { + "missing_alt": [{"url": "https://ex.com/alt", "images_without_alt": 2, "images_total": 3}], + "missing_lazy": [{"url": "https://ex.com/lazy", "img_without_lazy": 1, "images_total": 2}], + "missing_dimensions": [{"url": "https://ex.com/dim", "img_without_dimensions": 2, "images_total": 4}], + }, + "lighthouse_diagnostics": [ + {"lighthouse_audit_id": "uses-optimized-images", "title": "Serve images in next-gen formats"}, + {"lighthouse_audit_id": "render-blocking-resources", "title": "Eliminate render-blocking"}, + {"lighthouse_audit_id": "image-alt", "title": "Image elements do not have alt"}, + ], + "links": [ + { + "url": "https://ex.com/", + "og_image": "https://cdn.ex.com/og.png", + "twitter_image": "https://cdn.ex.com/tw.png", + "page_analysis": {"image_urls": ["https://cdn.ex.com/hero.jpg"]}, + }, + ], + "image_inventory": [ + { + "url": "https://cdn.ex.com/big.png", + "size_bytes": 500_000, + "content_type": "image/png", + "source_pages": ["https://ex.com/"], + "kinds": ["content"], + }, + { + "url": "https://cdn.ex.com/modern.webp", + "size_bytes": 400_000, + "content_type": "image/webp", + "source_pages": ["https://ex.com/"], + "kinds": ["content"], + }, + ], + "image_inventory_summary": {"probed": 2, "failed": 0, "unoptimized_min_kb": 200}, + } + + +def test_get_image_audit_summary(conn: MagicMock, ctx: Ctx) -> None: + df = pd.DataFrame([ + {"url": "https://ex.com/a", "status": "200", "images_without_alt": 1, "img_without_lazy": 0, + "img_without_dimensions": 0, "images_total": 5}, + ]) + with patch.object(Ctx, "load_payload", return_value=_payload()), patch.object(Ctx, "load_crawl_df", return_value=df): + summary = dispatch_tool("get_image_audit_summary", {}, context=ctx, conn=conn) + assert summary["pages_missing_alt"] >= 1 + assert summary["pages_without_lazy_images"] >= 1 + assert summary["pages_missing_image_dimensions"] >= 1 + assert summary["images_total_crawled"] == 5 + assert summary["lighthouse_image_diagnostics"] == 2 + assert summary["image_inventory_available"] is True + assert "page_previews" in summary + assert summary["page_previews"]["missing_lazy"]["total"] >= 1 + + +def test_list_site_image_urls(conn: MagicMock, ctx: Ctx) -> None: + with patch.object(Ctx, "load_payload", return_value=_payload()): + all_items = dispatch_tool("list_site_image_urls", {}, context=ctx, conn=conn) + assert all_items["total"] == 3 + og_only = dispatch_tool("list_site_image_urls", {"kind": "og"}, context=ctx, conn=conn) + assert og_only["total"] == 1 + assert og_only["items"][0]["kind"] == "og" + + +def test_list_lighthouse_image_opportunities(conn: MagicMock, ctx: Ctx) -> None: + with patch.object(Ctx, "load_payload", return_value=_payload()): + result = dispatch_tool("list_lighthouse_image_opportunities", {}, context=ctx, conn=conn) + assert result["total"] == 2 + ids = {d["lighthouse_audit_id"] for d in result["diagnostics"]} + assert ids <= IMAGE_LIGHTHOUSE_AUDIT_IDS + assert "render-blocking-resources" not in ids + + +def test_lazy_and_dimensions_lists_prefer_buckets(conn: MagicMock, ctx: Ctx) -> None: + with patch.object(Ctx, "load_payload", return_value=_payload()), patch.object(Ctx, "load_crawl_df", return_value=None): + lazy = dispatch_tool("list_pages_without_lazy_images", {}, context=ctx, conn=conn) + dims = dispatch_tool("list_pages_with_images_missing_dimensions", {}, context=ctx, conn=conn) + assert lazy["total"] == 1 + assert dims["total"] == 1 + + +def test_inventory_tools(conn: MagicMock, ctx: Ctx) -> None: + with patch.object(Ctx, "load_payload", return_value=_payload()): + largest = dispatch_tool("list_largest_images", {"min_size_kb": 200}, context=ctx, conn=conn) + assert largest["inventory_available"] is True + assert largest["total"] >= 1 + unopt = dispatch_tool("list_unoptimized_images", {}, context=ctx, conn=conn) + assert unopt["total"] == 1 + assert unopt["items"][0]["content_type"] == "image/png" + attention = dispatch_tool("list_images_needing_attention", {}, context=ctx, conn=conn) + assert attention["total"] >= 1 + assert attention["items"][0].get("reasons") + + +def test_inventory_missing_when_not_probed(conn: MagicMock, ctx: Ctx) -> None: + payload = _payload() + payload.pop("image_inventory") + with patch.object(Ctx, "load_payload", return_value=payload): + result = dispatch_tool("list_largest_images", {}, context=ctx, conn=conn) + assert result["inventory_available"] is False + assert "probe_image_inventory" in result["error"] + + +def test_all_image_tools_dispatch(conn: MagicMock, ctx: Ctx) -> None: + df = pd.DataFrame([{"url": "https://ex.com/a", "status": "200", "img_without_lazy": 1, "img_without_dimensions": 1}]) + tools = [ + "get_image_audit_summary", + "list_pages_without_lazy_images", + "list_pages_with_images_missing_dimensions", + "list_site_image_urls", + "list_lighthouse_image_opportunities", + "list_largest_images", + "list_unoptimized_images", + "list_images_needing_attention", + ] + with patch.object(Ctx, "load_payload", return_value=_payload()), patch.object(Ctx, "load_crawl_df", return_value=df): + for name in tools: + result = dispatch_tool(name, {}, context=ctx, conn=conn) + assert isinstance(result, dict) + assert "error" not in result diff --git a/tests/test_mcp_registry.py b/tests/test_mcp_registry.py index 95aada69..585c74da 100644 --- a/tests/test_mcp_registry.py +++ b/tests/test_mcp_registry.py @@ -7,7 +7,7 @@ def test_tool_definitions_schema() -> None: - assert len(TOOL_DEFINITIONS) == 123 + assert len(TOOL_DEFINITIONS) == 171 for tool in TOOL_DEFINITIONS: assert tool.get("name") assert tool.get("description") diff --git a/tests/test_mcp_server_helpers.py b/tests/test_mcp_server_helpers.py index 9e19e99d..01a73b2a 100644 --- a/tests/test_mcp_server_helpers.py +++ b/tests/test_mcp_server_helpers.py @@ -61,7 +61,7 @@ def test_read_glossary_excerpt_missing(monkeypatch) -> None: def test_tools_catalog_json_includes_security_tools() -> None: catalog = json.loads(mcp_server._tools_catalog_json()) - assert catalog["tool_count"] >= 123 + assert catalog["tool_count"] >= 171 assert "get_security_findings" in catalog["domains"]["security"] @@ -168,7 +168,7 @@ async def __aexit__(self, *_args): assert captured["name"] == "site-audit" assert captured["ran"] is True tools = asyncio.run(captured["list_tools"]()) # type: ignore[arg-type] - assert len(tools) >= 123 + assert len(tools) >= 171 resources = asyncio.run(captured["list_resources"]()) # type: ignore[arg-type] assert any(r["uri"] == "audit://property/7" for r in resources) diff --git a/tests/test_ollama_errors.py b/tests/test_ollama_errors.py new file mode 100644 index 00000000..c029149c --- /dev/null +++ b/tests/test_ollama_errors.py @@ -0,0 +1,19 @@ +"""Ollama API error formatting.""" +from __future__ import annotations + +from website_profiling.llm.providers.ollama import format_ollama_error + + +def test_format_model_not_found() -> None: + msg = format_ollama_error( + 404, + "model 'llama3.2' not found", + "llama3.2", + ) + assert "not installed" in msg + assert "ollama pull llama3.2" in msg + + +def test_format_generic_404() -> None: + msg = format_ollama_error(404, "", "m") + assert "/api/chat" in msg diff --git a/web/app/api/chat/artifacts/[id]/route.ts b/web/app/api/chat/artifacts/[id]/route.ts new file mode 100644 index 00000000..942ac29c --- /dev/null +++ b/web/app/api/chat/artifacts/[id]/route.ts @@ -0,0 +1,97 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { spawn } from 'child_process'; +import path from 'path'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { requireApiAuthForChat } from '@/server/auth'; +import { resolvePythonExecutable } from '@/server/resolvePython'; +import type { ApiRouteHandlerWithParams } from '@/types/api'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +const REPO_ROOT = process.env.WEBSITE_PROFILING_ROOT || path.resolve(process.cwd(), '..'); + +const ARTIFACT_SCRIPT = ` +import json +import sys +from website_profiling.tools.export_artifacts import read_artifact_bytes +aid = sys.argv[1] +result = read_artifact_bytes(aid) +if not result: + print(json.dumps({"error": "not found"})) +else: + meta, data = result + import base64 + print(json.dumps({ + "filename": meta.get("filename"), + "mime_type": meta.get("mime_type"), + "data_base64": base64.b64encode(data).decode("ascii"), + })) +`; + +export const GET: ApiRouteHandlerWithParams<{ id: string }> = async ( + request: NextRequest, + context: { params: Promise<{ id: string }> }, +): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + const authDenied = requireApiAuthForChat(request); + if (authDenied) return authDenied; + + const { id } = await context.params; + if (!id || !/^[a-f0-9-]{36}$/.test(id)) { + return NextResponse.json({ error: 'Invalid artifact id' }, { status: 400 }); + } + + const python = resolvePythonExecutable(process.env.PYTHON, REPO_ROOT); + + return new Promise((resolve) => { + const proc = spawn(python, ['-c', ARTIFACT_SCRIPT, id], { + cwd: REPO_ROOT, + env: { + ...process.env, + PYTHONPATH: path.join(REPO_ROOT, 'src'), + PYTHONIOENCODING: 'utf-8', + }, + }); + let out = ''; + let err = ''; + proc.stdout.on('data', (c: Buffer | string) => { + out += c.toString(); + }); + proc.stderr.on('data', (c) => { + err += c.toString(); + }); + proc.on('close', (code) => { + if (code !== 0) { + resolve(NextResponse.json({ error: err.trim() || 'Artifact read failed' }, { status: 500 })); + return; + } + try { + const parsed = JSON.parse(out.trim()) as { + error?: string; + filename?: string; + mime_type?: string; + data_base64?: string; + }; + if (parsed.error || !parsed.data_base64) { + resolve(NextResponse.json({ error: 'Artifact not found' }, { status: 404 })); + return; + } + const body = Buffer.from(parsed.data_base64, 'base64'); + const filename = parsed.filename || 'export.bin'; + const mime = parsed.mime_type || 'application/octet-stream'; + resolve( + new NextResponse(body, { + headers: { + 'Content-Type': mime, + 'Content-Disposition': `attachment; filename="${filename}"`, + }, + }), + ); + } catch { + resolve(NextResponse.json({ error: 'Invalid artifact response' }, { status: 500 })); + } + }); + }); +}; diff --git a/web/app/client-providers.tsx b/web/app/client-providers.tsx index 4562ecbb..83de93fd 100644 --- a/web/app/client-providers.tsx +++ b/web/app/client-providers.tsx @@ -5,6 +5,7 @@ import '@/patchConsole'; import { ThemeProvider } from '@/context/ThemeProvider'; import { PipelineProvider } from '@/context/PipelineContext'; import { SessionProvider } from '@/context/SessionContext'; +import ChatFab from '@/components/chat/ChatFab'; import PipelineRunnerFab from '@/components/pipeline/PipelineRunnerFab'; function LoadingFallback() { @@ -22,6 +23,7 @@ export default function ClientProviders({ children }: { children: ReactNode }): }> {children} + diff --git a/web/app/globals.css b/web/app/globals.css index bbf227d5..0d43cddd 100644 --- a/web/app/globals.css +++ b/web/app/globals.css @@ -424,9 +424,48 @@ code { .chat-prose-blockquote { margin: 0.75rem 0; - padding-left: 0.75rem; + padding: 0.75rem 1rem; border-left: 3px solid var(--app-border); + border-radius: 0 0.375rem 0.375rem 0; color: var(--app-text-subtle); + background: rgb(255 255 255 / 0.02); +} + +.chat-prose-insight { + margin: 1rem 0; + padding: 0.75rem 1rem; + border-left: 3px solid rgb(245 158 11 / 0.55); + border-radius: 0 0.375rem 0.375rem 0; + background: rgb(245 158 11 / 0.08); + color: var(--app-text); +} + +.chat-prose-insight-title { + margin-top: 1.25rem; + margin-bottom: 0.5rem; + padding: 0.5rem 0.75rem; + border-radius: 0.375rem; + border-left: 3px solid rgb(245 158 11 / 0.55); + background: rgb(245 158 11 / 0.08); + color: rgb(253 230 138 / 0.95); +} + +.chat-prose > .chat-prose-insight-title:first-child { + margin-top: 0; +} + +.chat-prose-ol { + counter-reset: chat-rec; +} + +.chat-prose-ol > .chat-prose-li { + margin: 0.5rem 0; + padding-left: 0.25rem; +} + +.chat-prose-ol > .chat-prose-li::marker { + font-weight: 600; + color: var(--accent, rgb(96 165 250)); } .chat-prose-hr { diff --git a/web/src/components/chat/ChatContextBar.tsx b/web/src/components/chat/ChatContextBar.tsx index 2fdb3242..12dc45e4 100644 --- a/web/src/components/chat/ChatContextBar.tsx +++ b/web/src/components/chat/ChatContextBar.tsx @@ -1,6 +1,6 @@ 'use client'; -import { Globe, PanelLeft } from 'lucide-react'; +import { Globe } from 'lucide-react'; import { formatChatPropertyLabel } from '@/lib/chatPropertyLabel'; import { strings } from '@/lib/strings'; import type { PropertyOption } from '@/components/chat/ChatSidebar'; @@ -12,7 +12,6 @@ export interface ChatContextBarProps { propertyId: number | null; sessionTitle?: string | null; loading?: boolean; - onExpandSidebar?: () => void; } export default function ChatContextBar({ @@ -20,7 +19,6 @@ export default function ChatContextBar({ propertyId, sessionTitle, loading, - onExpandSidebar, }: ChatContextBarProps) { const domainLabel = property ? formatChatPropertyLabel(property) @@ -30,17 +28,6 @@ export default function ChatContextBar({ return (
- {onExpandSidebar ? ( - - ) : null}
diff --git a/web/src/components/chat/ChatFab.tsx b/web/src/components/chat/ChatFab.tsx new file mode 100644 index 00000000..bd2e7ec0 --- /dev/null +++ b/web/src/components/chat/ChatFab.tsx @@ -0,0 +1,35 @@ +'use client'; + +import { MessageSquare } from 'lucide-react'; +import Link from 'next/link'; +import { usePathname, useSearchParams } from 'next/navigation'; +import { buildChatFabHref, isChatFabVisiblePath } from '@/lib/chatUrlState'; +import { strings } from '@/lib/strings'; + +const s = strings.components.chat; + +/** + * Floating entry to AI chat from domain-scoped report views (e.g. /dashboard?domain=…). + */ +export default function ChatFab() { + const pathname = usePathname(); + const searchParams = useSearchParams(); + const domain = searchParams.get('domain') ?? searchParams.get('brand'); + + if (!isChatFabVisiblePath(pathname)) { + return null; + } + + const href = buildChatFabHref(domain); + + return ( + + + + ); +} diff --git a/web/src/components/chat/ChatMarkdown.tsx b/web/src/components/chat/ChatMarkdown.tsx index e02af26f..dff69c73 100644 --- a/web/src/components/chat/ChatMarkdown.tsx +++ b/web/src/components/chat/ChatMarkdown.tsx @@ -9,6 +9,24 @@ import { oneDark } from 'react-syntax-highlighter/dist/esm/styles/prism'; import type { Components } from 'react-markdown'; import { preprocessChatMarkdown } from '@/components/chat/preprocessChatMarkdown'; +function flattenText(node: unknown): string { + if (node == null || typeof node === 'boolean') return ''; + if (typeof node === 'string' || typeof node === 'number') return String(node); + if (Array.isArray(node)) return node.map(flattenText).join(''); + if (typeof node === 'object' && node !== null && 'props' in node) { + const props = (node as { props?: { children?: unknown } }).props; + return flattenText(props?.children); + } + return ''; +} + +function insightHeadingClass(children: unknown, base: string): string { + const text = flattenText(children); + return /💡|power insights|recommended actions|quick wins|priority fixes/i.test(text) + ? `${base} chat-prose-insight-title` + : base; +} + export interface ChatMarkdownProps { content: string; streaming?: boolean; @@ -19,9 +37,9 @@ export default function ChatMarkdown({ content, streaming }: ChatMarkdownProps) const components = useMemo( () => ({ - h1: ({ children }) =>

{children}

, - h2: ({ children }) =>

{children}

, - h3: ({ children }) =>
{children}
, + h1: ({ children }) =>

{children}

, + h2: ({ children }) =>

{children}

, + h3: ({ children }) =>
{children}
, p: ({ children }) =>

{children}

, ul: ({ children }) =>
    {children}
, ol: ({ children }) =>
    {children}
, @@ -33,9 +51,17 @@ export default function ChatMarkdown({ content, streaming }: ChatMarkdownProps) ), strong: ({ children }) => {children}, em: ({ children }) => {children}, - blockquote: ({ children }) => ( -
{children}
- ), + blockquote: ({ children }) => { + const text = String(children ?? ''); + const isInsight = /💡|power insights|key takeaway/i.test(text); + return ( +
+ {children} +
+ ); + }, hr: () =>
, table: ({ children }) => (
diff --git a/web/src/components/chat/ChatSidebar.tsx b/web/src/components/chat/ChatSidebar.tsx index c7c3f805..2b5a5b5c 100644 --- a/web/src/components/chat/ChatSidebar.tsx +++ b/web/src/components/chat/ChatSidebar.tsx @@ -8,6 +8,7 @@ import { Home, Link as LinkIcon, MessageSquarePlus, + PanelLeft, Settings, Terminal, Trash2, @@ -177,6 +178,10 @@ export default function ChatSidebar({ + setExpanded(true)}> + + + diff --git a/web/src/components/chat/blocks/ChatBlocks.tsx b/web/src/components/chat/blocks/ChatBlocks.tsx index 1f79dee9..023c8933 100644 --- a/web/src/components/chat/blocks/ChatBlocks.tsx +++ b/web/src/components/chat/blocks/ChatBlocks.tsx @@ -2,6 +2,7 @@ import type { ChatBlock } from '@/components/chat/deriveChatBlocks'; import { blockKey } from '@/components/chat/deriveChatBlocks'; +import ChatFileDownloadBlock from './ChatFileDownloadBlock'; import ChatCategoryScoresBlock from './ChatCategoryScoresBlock'; import ChatCompareCategoryBlock from './ChatCompareCategoryBlock'; import ChatGoogleSummaryBlock from './ChatGoogleSummaryBlock'; @@ -11,6 +12,10 @@ import ChatIssueTableBlock from './ChatIssueTableBlock'; import ChatLabelValueChartBlock from './ChatLabelValueChartBlock'; import ChatLighthouseBlock from './ChatLighthouseBlock'; import ChatStatusBreakdownBlock from './ChatStatusBreakdownBlock'; +import ChatImageAuditBlock from './ChatImageAuditBlock'; +import ChatImagePagesTableBlock from './ChatImagePagesTableBlock'; +import ChatImageAttentionTableBlock from './ChatImageAttentionTableBlock'; +import ChatImageLighthouseBlock from './ChatImageLighthouseBlock'; export interface ChatBlocksProps { blocks: ChatBlock[]; @@ -42,6 +47,16 @@ export default function ChatBlocks({ blocks }: ChatBlocksProps) { return ; case 'google_summary': return ; + case 'file_download': + return ; + case 'image_audit_summary': + return ; + case 'image_pages_table': + return ; + case 'image_attention_table': + return ; + case 'image_lighthouse_list': + return ; default: return null; } diff --git a/web/src/components/chat/blocks/ChatFileDownloadBlock.tsx b/web/src/components/chat/blocks/ChatFileDownloadBlock.tsx new file mode 100644 index 00000000..c086bc8a --- /dev/null +++ b/web/src/components/chat/blocks/ChatFileDownloadBlock.tsx @@ -0,0 +1,53 @@ +'use client'; + +import { Download, FileText } from 'lucide-react'; +import { apiUrl, getPublicBasePath } from '@/lib/publicBase'; +import type { ChatBlock } from '@/components/chat/deriveChatBlocks'; + +type FileDownloadBlock = Extract; + +function resolveHref(url: string): string { + if (url.startsWith('http://') || url.startsWith('https://')) return url; + if (url.startsWith('/api/')) return `${getPublicBasePath()}${url}`; + return apiUrl(url.replace(/^\//, '')); +} + +function formatLabel(filename: string, mimeType?: string): string { + const lower = filename.toLowerCase(); + if (lower.endsWith('.pdf')) return 'Download PDF'; + if (lower.endsWith('.csv')) return 'Download CSV'; + if (lower.endsWith('.json')) return 'Download JSON'; + if (lower.endsWith('.html')) return 'Download HTML'; + if (mimeType?.includes('pdf')) return 'Download PDF'; + if (mimeType?.includes('csv')) return 'Download CSV'; + return 'Download file'; +} + +export default function ChatFileDownloadBlock({ block }: { block: FileDownloadBlock }) { + return ( +
+

Export ready

+
+ {block.files.map((file) => { + const href = resolveHref(file.url); + const isPdf = file.filename.toLowerCase().endsWith('.pdf') || file.mime_type?.includes('pdf'); + return ( + + {isPdf ? : } + {file.label || formatLabel(file.filename, file.mime_type)} + + ); + })} +
+
+ ); +} diff --git a/web/src/components/chat/blocks/ChatImageAttentionTableBlock.tsx b/web/src/components/chat/blocks/ChatImageAttentionTableBlock.tsx new file mode 100644 index 00000000..befa15c5 --- /dev/null +++ b/web/src/components/chat/blocks/ChatImageAttentionTableBlock.tsx @@ -0,0 +1,78 @@ +'use client'; + +import type { ChatBlock } from '@/components/chat/deriveChatBlocks'; +import { formatChatUrlDisplay } from '@/lib/formatChatUrl'; + +type Block = Extract; + +const DISPLAY_LIMIT = 10; + +function formatBytes(n: number): string { + if (n >= 1_048_576) return `${(n / 1_048_576).toFixed(1)} MB`; + if (n >= 1024) return `${Math.round(n / 1024)} KB`; + return `${n} B`; +} + +function formatReason(r: string): string { + return r.replace(/_/g, ' '); +} + +export default function ChatImageAttentionTableBlock({ block }: { block: Block }) { + const shown = block.items.slice(0, DISPLAY_LIMIT); + const remaining = (block.total ?? block.items.length) - shown.length; + + return ( +
+

+ {block.title} +

+
    + {shown.map((item, i) => { + const href = item.url || item.pageUrl; + const label = href ? formatChatUrlDisplay(href) : 'Page-level issue'; + return ( +
  • +
    + {href ? ( + + {label} + + ) : ( + {label} + )} + {item.sizeBytes != null && Number.isFinite(item.sizeBytes) ? ( + + {formatBytes(item.sizeBytes)} + + ) : null} +
    + {item.reasons.length ? ( +
    + {item.reasons.map((r) => ( + + {formatReason(r)} + + ))} +
    + ) : null} +
    • + ); + })} +
+ {remaining > 0 || block.truncated ? ( +

+ {remaining > 0 ? `${remaining} more not shown` : 'Results truncated'} +

+ ) : null} +
+ ); +} diff --git a/web/src/components/chat/blocks/ChatImageAuditBlock.tsx b/web/src/components/chat/blocks/ChatImageAuditBlock.tsx new file mode 100644 index 00000000..c1cdbf9b --- /dev/null +++ b/web/src/components/chat/blocks/ChatImageAuditBlock.tsx @@ -0,0 +1,126 @@ +'use client'; + +import { ImageIcon } from 'lucide-react'; +import { SimpleBarChart } from '@/components/charts/SimpleBarChart'; +import type { ChatBlock } from '@/components/chat/deriveChatBlocks'; +import { strings } from '@/lib/strings'; + +type Block = Extract; +const ib = strings.components.chat.blocks.imageAudit; + +function StatCard({ + label, + value, + tone, +}: { + label: string; + value: number; + tone: 'ok' | 'warn' | 'neutral'; +}) { + const toneClass = + tone === 'ok' + ? 'border-emerald-500/30 bg-emerald-500/10 text-emerald-200' + : tone === 'warn' + ? 'border-amber-500/30 bg-amber-500/10 text-amber-100' + : 'border-default bg-brand-800/40 text-foreground'; + + return ( +
+

{label}

+

{value.toLocaleString()}

+
+ ); +} + +export default function ChatImageAuditBlock({ block }: { block: Block }) { + const chartItems = [ + { label: ib.missingAlt, value: block.pagesMissingAlt }, + { label: ib.noLazyLoad, value: block.pagesWithoutLazy }, + { label: ib.missingDimensions, value: block.pagesMissingDimensions }, + { label: ib.lighthouseIssues, value: block.lighthouseImageDiagnostics }, + ].filter((i) => i.value > 0); + + return ( +
+
+
+ +
+
+

{ib.title}

+

{ib.subtitle}

+
+
+

+ {block.imagesTotal.toLocaleString()} +

+

{ib.totalImages}

+
+
+ +
+ 0 ? 'warn' : 'ok'} + /> + 0 ? 'warn' : 'ok'} + /> + 0 ? 'warn' : 'ok'} + /> + 0 ? 'warn' : 'ok'} + /> +
+ +
+ {block.ogCoveragePct != null ? ( + + {ib.ogCoverage}:{' '} + + {block.ogCoveragePct % 1 === 0 + ? block.ogCoveragePct + : block.ogCoveragePct.toFixed(1)} + % + + {block.ogMissingCount != null && block.ogMissingCount > 0 + ? ` · ${block.ogMissingCount} missing` + : ''} + + ) : null} + + {ib.sizeProbe}:{' '} + + {block.inventoryAvailable ? ib.probeOn : ib.probeOff} + + {block.inventoryAvailable && block.inventoryProbed != null + ? ` · ${block.inventoryProbed} URLs` + : ''} + +
+ + {chartItems.length > 0 ? ( +
+

{ib.issueBreakdown}

+ i.label)} + values={chartItems.map((i) => i.value)} + ariaLabel={ib.issueBreakdown} + /> +
+ ) : null} +
+ ); +} diff --git a/web/src/components/chat/blocks/ChatImageLighthouseBlock.tsx b/web/src/components/chat/blocks/ChatImageLighthouseBlock.tsx new file mode 100644 index 00000000..315bd838 --- /dev/null +++ b/web/src/components/chat/blocks/ChatImageLighthouseBlock.tsx @@ -0,0 +1,44 @@ +'use client'; + +import type { ChatBlock } from '@/components/chat/deriveChatBlocks'; +import { formatChatUrlDisplay } from '@/lib/formatChatUrl'; +import { strings } from '@/lib/strings'; + +type Block = Extract; +const ib = strings.components.chat.blocks.imageAudit; + +export default function ChatImageLighthouseBlock({ block }: { block: Block }) { + return ( +
+

+ {ib.lighthouseTitle} +

+ + {block.total > block.items.length ? ( +

+ {block.total - block.items.length} more Lighthouse image findings +

+ ) : null} +
+ ); +} diff --git a/web/src/components/chat/blocks/ChatImagePagesTableBlock.tsx b/web/src/components/chat/blocks/ChatImagePagesTableBlock.tsx new file mode 100644 index 00000000..0429a8fb --- /dev/null +++ b/web/src/components/chat/blocks/ChatImagePagesTableBlock.tsx @@ -0,0 +1,60 @@ +'use client'; + +import type { ChatBlock } from '@/components/chat/deriveChatBlocks'; +import { useChatFollowUp } from '@/components/chat/ChatFollowUpContext'; +import { formatChatUrlDisplay } from '@/lib/formatChatUrl'; +import { format, strings } from '@/lib/strings'; + +type Block = Extract; +const cb = strings.components.chat.blocks; +const ib = cb.imageAudit; + +const DISPLAY_LIMIT = 12; + +export default function ChatImagePagesTableBlock({ block }: { block: Block }) { + const { suggestFollowUp } = useChatFollowUp(); + const shown = block.pages.slice(0, DISPLAY_LIMIT); + const remaining = (block.total ?? block.pages.length) - shown.length; + + return ( +
+

+ {block.title} + {block.total != null ? ( + ({block.total}) + ) : null} +

+ + {remaining > 0 || block.truncated ? ( +
+ {remaining > 0 ? format(ib.morePages, { count: remaining }) : cb.showAll} + +
+ ) : null} +
+ ); +} diff --git a/web/src/components/chat/deriveChatBlocks.test.ts b/web/src/components/chat/deriveChatBlocks.test.ts index 836380f6..88cbea55 100644 --- a/web/src/components/chat/deriveChatBlocks.test.ts +++ b/web/src/components/chat/deriveChatBlocks.test.ts @@ -179,6 +179,67 @@ describe('deriveChatBlocks', () => { 'label_value:MIME types', ); }); + + it('builds image_audit_summary from get_image_audit_summary', () => { + const blocks = deriveChatBlocks([ + doneTool('get_image_audit_summary', { + images_total_crawled: 106, + pages_missing_alt: 1, + pages_without_lazy_images: 9, + pages_missing_image_dimensions: 0, + og_image_coverage_pct: 95, + og_image_missing_count: 2, + lighthouse_image_diagnostics: 3, + image_inventory_available: false, + }), + ]); + expect(blocks.map((b) => b.type)).toContain('image_audit_summary'); + const summary = blocks.find((b) => b.type === 'image_audit_summary'); + if (summary?.type === 'image_audit_summary') { + expect(summary.imagesTotal).toBe(106); + expect(summary.pagesWithoutLazy).toBe(9); + } + }); + + it('builds image_pages_table from list_pages_without_lazy_images', () => { + const blocks = deriveChatBlocks([ + doneTool('list_pages_without_lazy_images', { + pages: [{ url: 'https://ex.com/a', title: 'A', img_without_lazy: 2, images_total: 5 }], + total: 1, + }), + ]); + expect(blocks[0]?.type).toBe('image_pages_table'); + }); + + it('builds preview page tables from get_image_audit_summary', () => { + const blocks = deriveChatBlocks([ + doneTool('get_image_audit_summary', { + images_total_crawled: 106, + pages_missing_alt: 1, + pages_without_lazy_images: 9, + pages_missing_image_dimensions: 0, + lighthouse_image_diagnostics: 1, + image_inventory_available: false, + page_previews: { + missing_alt: { + pages: [{ url: 'https://ex.com/docs', images_without_alt: 1, images_total: 1 }], + total: 1, + }, + missing_lazy: { + pages: [{ url: 'https://ex.com/', img_without_lazy: 2, images_total: 54 }], + total: 9, + truncated: true, + }, + missing_og: { pages: [{ url: 'https://ex.com/og' }], total: 11, truncated: true }, + }, + lighthouse_image_previews: [{ title: 'LCP image', lighthouse_audit_id: 'largest-contentful-paint' }], + }), + ]); + const types = blocks.map((b) => b.type); + expect(types).toContain('image_audit_summary'); + expect(types.filter((t) => t === 'image_pages_table').length).toBeGreaterThanOrEqual(2); + expect(types).toContain('image_lighthouse_list'); + }); }); describe('toolEventsToActivity', () => { diff --git a/web/src/components/chat/deriveChatBlocks.ts b/web/src/components/chat/deriveChatBlocks.ts index d2a45629..27f597f5 100644 --- a/web/src/components/chat/deriveChatBlocks.ts +++ b/web/src/components/chat/deriveChatBlocks.ts @@ -91,6 +91,49 @@ export type ChatBlock = ctr?: number; queries: GoogleQueryRow[]; pages: GooglePageRow[]; + } + | { + type: 'file_download'; + files: { label?: string; url: string; mime_type?: string; filename: string }[]; + } + | { + type: 'image_audit_summary'; + imagesTotal: number; + pagesMissingAlt: number; + pagesWithoutLazy: number; + pagesMissingDimensions: number; + ogCoveragePct?: number; + ogMissingCount?: number; + lighthouseImageDiagnostics: number; + inventoryAvailable: boolean; + inventoryProbed?: number; + inventoryFailed?: number; + } + | { + type: 'image_pages_table'; + title: string; + pages: { url: string; title?: string; detail?: string }[]; + total?: number; + truncated?: boolean; + } + | { + type: 'image_lighthouse_list'; + items: { title: string; auditId?: string; url?: string; displayValue?: string }[]; + total: number; + } + | { + type: 'image_attention_table'; + title: string; + items: { + url?: string; + pageUrl?: string; + sizeBytes?: number | null; + contentType?: string; + reasons: string[]; + score?: number; + }[]; + total?: number; + truncated?: boolean; }; const SUMMARY_TOOLS = new Set(['get_report_summary', 'get_executive_summary']); @@ -119,6 +162,35 @@ const GOOGLE_SUMMARY_TOOLS = new Set([ 'get_gsc_top_queries', 'get_gsc_top_pages', ]); +const EXPORT_TOOLS = new Set([ + 'export_audit_report', + 'export_compare_csv', + 'export_list_as_csv', + 'export_custom_report', +]); + +const IMAGE_SUMMARY_TOOL = 'get_image_audit_summary'; + +const IMAGE_PREVIEW_TITLES: Record = { + missing_alt: 'Pages missing alt text', + missing_lazy: 'Pages without lazy-loaded images', + missing_dimensions: 'Pages missing width/height', + missing_og: 'Pages missing OG image', +}; + +const IMAGE_PAGE_TABLE_TOOLS: Record = { + list_pages_with_missing_alt: IMAGE_PREVIEW_TITLES.missing_alt, + list_pages_without_lazy_images: IMAGE_PREVIEW_TITLES.missing_lazy, + list_pages_with_images_missing_dimensions: IMAGE_PREVIEW_TITLES.missing_dimensions, + list_pages_missing_og_image: IMAGE_PREVIEW_TITLES.missing_og, +}; + +const IMAGE_INVENTORY_TABLE_TOOLS: Record = { + list_largest_images: 'Largest images', + list_unoptimized_images: 'Unoptimized images', +}; + +const IMAGE_ATTENTION_TOOL = 'list_images_needing_attention'; function asRecord(v: unknown): Record | null { return v && typeof v === 'object' && !Array.isArray(v) ? (v as Record) : null; @@ -130,6 +202,14 @@ export function blockKey(block: ChatBlock): string { return `label_value:${block.title}`; case 'health_trend': return block.categoryId ? `health_trend:${block.categoryId}` : 'health_trend'; + case 'file_download': + return `file_download:${block.files.map((f) => f.filename).join(',')}`; + case 'image_pages_table': + return `image_pages:${block.title}`; + case 'image_attention_table': + return `image_attention:${block.title}`; + case 'image_lighthouse_list': + return 'image_lighthouse'; default: return block.type; } @@ -517,9 +597,234 @@ function parseGooglePages(raw: unknown): GooglePageRow[] { return out; } -type BlockParser = (name: string, result: Record) => ChatBlock | null; +function blockFromImageSummaryPreviews( + name: string, + result: Record, +): ChatBlock[] | null { + if (name !== IMAGE_SUMMARY_TOOL || result.error) return null; + const previews = asRecord(result.page_previews); + if (!previews) return null; + const blocks: ChatBlock[] = []; + for (const [key, title] of Object.entries(IMAGE_PREVIEW_TITLES)) { + const bucket = asRecord(previews[key]); + const pagesRaw = bucket?.pages; + if (!Array.isArray(pagesRaw) || !pagesRaw.length) continue; + const pages = pagesRaw + .map((raw) => { + const row = asRecord(raw); + if (!row) return null; + const url = String(row.url || '').trim(); + if (!url) return null; + return { + url, + title: row.title != null ? String(row.title) : undefined, + detail: pageDetailFromRow(row), + }; + }) + .filter((p): p is NonNullable => p != null); + if (!pages.length) continue; + blocks.push({ + type: 'image_pages_table', + title, + pages, + total: typeof bucket?.total === 'number' ? bucket.total : pages.length, + truncated: Boolean(bucket?.truncated), + }); + } + const lhRaw = result.lighthouse_image_previews; + if (Array.isArray(lhRaw) && lhRaw.length) { + const items = lhRaw + .map((raw) => { + const row = asRecord(raw); + if (!row) return null; + return { + title: String(row.title || row.lighthouse_audit_id || 'Lighthouse'), + auditId: row.lighthouse_audit_id != null ? String(row.lighthouse_audit_id) : undefined, + url: row.url != null ? String(row.url) : undefined, + displayValue: row.display_value != null ? String(row.display_value) : undefined, + }; + }) + .filter((i): i is NonNullable => i != null); + if (items.length) { + blocks.push({ + type: 'image_lighthouse_list', + items, + total: Number(result.lighthouse_image_diagnostics) || items.length, + }); + } + } + return blocks.length ? blocks : null; +} + +function blockFromImageSummary(name: string, result: Record): ChatBlock | null { + if (name !== IMAGE_SUMMARY_TOOL) return null; + if (result.error) return null; + const inv = asRecord(result.image_inventory_summary); + return { + type: 'image_audit_summary', + imagesTotal: Number(result.images_total_crawled) || 0, + pagesMissingAlt: Number(result.pages_missing_alt) || 0, + pagesWithoutLazy: Number(result.pages_without_lazy_images) || 0, + pagesMissingDimensions: Number(result.pages_missing_image_dimensions) || 0, + ogCoveragePct: + result.og_image_coverage_pct != null ? Number(result.og_image_coverage_pct) : undefined, + ogMissingCount: + result.og_image_missing_count != null ? Number(result.og_image_missing_count) : undefined, + lighthouseImageDiagnostics: Number(result.lighthouse_image_diagnostics) || 0, + inventoryAvailable: Boolean(result.image_inventory_available), + inventoryProbed: inv?.probed != null ? Number(inv.probed) : undefined, + inventoryFailed: inv?.failed != null ? Number(inv.failed) : undefined, + }; +} + +function pageDetailFromRow(row: Record): string | undefined { + const parts: string[] = []; + if (row.images_without_alt != null) parts.push(`${row.images_without_alt} missing alt`); + if (row.img_without_lazy != null) parts.push(`${row.img_without_lazy} without lazy load`); + if (row.img_without_dimensions != null) parts.push(`${row.img_without_dimensions} missing dimensions`); + if (row.images_total != null && parts.length) { + return `${parts.join(' · ')} (${row.images_total} total)`; + } + return parts.join(' · ') || undefined; +} + +function blockFromImagePagesTable(name: string, result: Record): ChatBlock | null { + const title = IMAGE_PAGE_TABLE_TOOLS[name]; + if (!title) return null; + if (result.error) return null; + const pagesRaw = result.pages; + if (!Array.isArray(pagesRaw) || !pagesRaw.length) return null; + const pages = pagesRaw + .map((raw) => { + const row = asRecord(raw); + if (!row) return null; + const url = String(row.url || '').trim(); + if (!url) return null; + return { + url, + title: row.title != null ? String(row.title) : undefined, + detail: pageDetailFromRow(row), + }; + }) + .filter((p): p is NonNullable => p != null); + if (!pages.length) return null; + return { + type: 'image_pages_table', + title, + pages, + total: typeof result.total === 'number' ? result.total : pages.length, + truncated: Boolean(result.truncated), + }; +} + +function formatBytes(n: number): string { + if (n >= 1_048_576) return `${(n / 1_048_576).toFixed(1)} MB`; + if (n >= 1024) return `${Math.round(n / 1024)} KB`; + return `${n} B`; +} + +function blockFromImageInventoryTable(name: string, result: Record): ChatBlock | null { + const title = IMAGE_INVENTORY_TABLE_TOOLS[name]; + if (!title) return null; + if (result.error) return null; + const itemsRaw = result.items; + if (!Array.isArray(itemsRaw) || !itemsRaw.length) return null; + const items = itemsRaw + .map((raw) => { + const row = asRecord(raw); + if (!row) return null; + const url = String(row.url || '').trim(); + if (!url) return null; + const size = row.size_bytes != null ? Number(row.size_bytes) : null; + const ctype = row.content_type != null ? String(row.content_type) : undefined; + const reasons: string[] = []; + if (row.reason) reasons.push(String(row.reason)); + if (size != null && Number.isFinite(size)) { + reasons.push(formatBytes(size)); + } + if (ctype) reasons.push(ctype); + return { + url, + sizeBytes: size, + contentType: ctype, + reasons, + score: row.attention_score != null ? Number(row.attention_score) : undefined, + }; + }) + .filter((i): i is NonNullable => i != null); + if (!items.length) return null; + return { + type: 'image_attention_table', + title, + items, + total: typeof result.total === 'number' ? result.total : items.length, + truncated: Boolean(result.truncated), + }; +} + +function blockFromImageAttention(name: string, result: Record): ChatBlock | null { + if (name !== IMAGE_ATTENTION_TOOL) return null; + if (result.error) return null; + const itemsRaw = result.items; + if (!Array.isArray(itemsRaw) || !itemsRaw.length) return null; + const items = itemsRaw + .map((raw) => { + const row = asRecord(raw); + if (!row) return null; + const reasons = Array.isArray(row.reasons) + ? row.reasons.map((r) => String(r)).filter(Boolean) + : []; + if (!reasons.length && !row.url && !row.page_url) return null; + return { + url: row.url != null ? String(row.url) : undefined, + pageUrl: row.page_url != null ? String(row.page_url) : undefined, + sizeBytes: row.size_bytes != null ? Number(row.size_bytes) : null, + contentType: row.content_type != null ? String(row.content_type) : undefined, + reasons, + score: row.attention_score != null ? Number(row.attention_score) : undefined, + }; + }) + .filter((i): i is NonNullable => i != null); + if (!items.length) return null; + return { + type: 'image_attention_table', + title: 'Images needing attention', + items, + total: typeof result.total === 'number' ? result.total : items.length, + truncated: Boolean(result.truncated), + }; +} + +function blockFromFileDownload(name: string, result: Record): ChatBlock | null { + if (!EXPORT_TOOLS.has(name)) return null; + if (result.error) return null; + const artifactId = String(result.artifact_id || ''); + const filename = String(result.filename || 'export.bin'); + if (!artifactId) return null; + const mimeType = result.mime_type != null ? String(result.mime_type) : undefined; + const fmt = result.format != null ? String(result.format).toUpperCase() : undefined; + return { + type: 'file_download', + files: [ + { + filename, + mime_type: mimeType, + url: `/api/chat/artifacts/${artifactId}`, + label: fmt ? `Download ${fmt}` : undefined, + }, + ], + }; +} + +type BlockParser = (name: string, result: Record) => ChatBlock | ChatBlock[] | null; const BLOCK_PARSERS: BlockParser[] = [ + blockFromFileDownload, + blockFromImageSummary, + blockFromImageSummaryPreviews, + blockFromImagePagesTable, + blockFromImageInventoryTable, + blockFromImageAttention, blockFromSummary, blockFromReportCrawlStatus, blockFromIssueTable, @@ -542,12 +847,15 @@ export function deriveChatBlocks(toolActivity: ToolActivityItem[]): ChatBlock[] if (item.status !== 'done' || !item.result) continue; const result = item.result; for (const parser of BLOCK_PARSERS) { - const block = parser(item.name, result); - if (!block) continue; - const key = blockKey(block); - if (seen.has(key)) continue; - seen.add(key); - blocks.push(block); + const parsed = parser(item.name, result); + if (!parsed) continue; + const candidates = Array.isArray(parsed) ? parsed : [parsed]; + for (const block of candidates) { + const key = blockKey(block); + if (seen.has(key)) continue; + seen.add(key); + blocks.push(block); + } } } diff --git a/web/src/components/chat/preprocessChatMarkdown.ts b/web/src/components/chat/preprocessChatMarkdown.ts index ce0d901b..594685a2 100644 --- a/web/src/components/chat/preprocessChatMarkdown.ts +++ b/web/src/components/chat/preprocessChatMarkdown.ts @@ -13,6 +13,10 @@ export function preprocessChatMarkdown(content: string): string { 'Issues to fix', 'Recommendations', 'Next steps', + 'Power Insights', + 'Recommended actions', + 'Quick wins', + 'Priority fixes', ]; for (const title of sectionHeadings) { diff --git a/web/src/components/chat/stripRedundantMarkdown.ts b/web/src/components/chat/stripRedundantMarkdown.ts index 6b0b0b88..5ce91afb 100644 --- a/web/src/components/chat/stripRedundantMarkdown.ts +++ b/web/src/components/chat/stripRedundantMarkdown.ts @@ -15,12 +15,18 @@ function isIssueTable(headerRow: string): boolean { ); } +function isImageMetricTable(headerRow: string): boolean { + const h = headerRow.toLowerCase(); + return /metric|value|images|alt|lazy|dimension|og/i.test(h); +} + function shouldStripTable(headerRow: string, blocks: ChatBlock[]): boolean { for (const block of blocks) { if (block.type === 'category_scores' && isCategoryScoreTable(headerRow)) return true; if (block.type === 'issue_table' && isIssueTable(headerRow)) return true; if (block.type === 'compare_category_deltas' && isCategoryScoreTable(headerRow)) return true; if (block.type === 'google_summary' && /query|clicks|page/i.test(headerRow)) return true; + if (block.type === 'image_audit_summary' && isImageMetricTable(headerRow)) return true; } return false; } @@ -118,11 +124,54 @@ function stripOverviewProse(content: string, blocks: ChatBlock[]): string { return out.replace(/\n{3,}/g, '\n\n').trim(); } +function stripImageAuditProse(content: string, blocks: ChatBlock[]): string { + const hasImageViz = blocks.some( + (b) => + b.type === 'image_audit_summary' || + b.type === 'image_pages_table' || + b.type === 'image_lighthouse_list', + ); + if (!hasImageViz) return content; + + let out = content; + out = out.replace(/^[^\n#]*📸[^\n]*\n?/m, ''); + out = out.replace( + /^.*\b(?:total images crawled|pages missing alt|pages with non-lazy|pages missing image dimensions|og image coverage|lighthouse image).*(?:\d+|%).*$/gim, + '', + ); + out = out.replace(/\n?#{1,3}\s*headline numbers[^\n]*\n[\s\S]*?(?=\n#{1,3}\s|$)/gi, '\n'); + + // Drop per-issue URL enumerations when page tables render the same data + if (blocks.some((b) => b.type === 'image_pages_table')) { + out = out.replace( + /\n?#{1,5}\s*\d+\.\s*(?:missing alt|lazy|og|lighthouse|dimensions)[^\n]*\n[\s\S]*?(?=\n#{1,5}\s|\n#{1,3}\s|$)/gi, + '\n', + ); + const lines = out.split('\n'); + const filtered = lines.filter((line) => { + const t = line.trim(); + if (!t) return true; + if (/^https?:\/\//i.test(t)) return false; + if (/^[-*]\s+https?:\/\//i.test(t)) return false; + if (/^[-*]\s+\/[\w/-]+/.test(t)) return false; + if (/non-lazy\s*\/\s*\d+\s+total/i.test(t)) return false; + if (/without alt/i.test(t) && /\d+\s+of\s+\d+/i.test(t)) return false; + if (/pages missing:/i.test(t)) return false; + if (/^[-*]\s+[\w-]+,\s+[\w-]+/.test(t) && t.length < 120) return false; + return true; + }); + out = filtered.join('\n'); + } + + return out.replace(/\n{3,}/g, '\n\n').trim(); +} + /** Remove GFM tables and prose duplicated by structured chat blocks. */ export function stripRedundantMarkdown(content: string, blocks: ChatBlock[]): string { if (!content.trim() || !blocks.length) return content; let out = stripTables(content, blocks); out = stripIssueProse(out, blocks); out = stripOverviewProse(out, blocks); + out = stripImageAuditProse(out, blocks); return out.replace(/\n{3,}/g, '\n\n').trim(); } diff --git a/web/src/components/links/tabs/TechnicalTab.tsx b/web/src/components/links/tabs/TechnicalTab.tsx index ecb86e9c..c2d3c028 100644 --- a/web/src/components/links/tabs/TechnicalTab.tsx +++ b/web/src/components/links/tabs/TechnicalTab.tsx @@ -109,6 +109,12 @@ export default function TechnicalTab({ link }: TechnicalTabProps) { label={lt.noLazyLoad} color={(link.img_without_lazy ?? 0) > 0 ? 'bg-yellow-500' : 'bg-green-500'} /> + 0 ? 'bg-orange-500' : 'bg-green-500'} + />
{lt.ariaElements} {link.aria_count ?? 0} diff --git a/web/src/lib/chatUrlState.test.ts b/web/src/lib/chatUrlState.test.ts index ec77ac54..bf02f184 100644 --- a/web/src/lib/chatUrlState.test.ts +++ b/web/src/lib/chatUrlState.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from 'vitest'; import { applyChatUrlContext, + buildChatFabHref, buildChatSearchQuery, + isChatFabVisiblePath, parseChatUrlContext, } from './chatUrlState'; @@ -29,4 +31,18 @@ describe('chatUrlState', () => { const current = 'property=3&session=42'; expect(buildChatSearchQuery(current, { propertyId: 3, sessionId: 42 })).toBe(current); }); + + it('buildChatFabHref includes domain query for chat deep link', () => { + expect(buildChatFabHref('codefrydev.in')).toBe('/chat?domain=codefrydev.in'); + expect(buildChatFabHref('')).toBe('/chat'); + expect(buildChatFabHref(null)).toBe('/chat'); + }); + + it('isChatFabVisiblePath matches report routes only', () => { + expect(isChatFabVisiblePath('/dashboard')).toBe(true); + expect(isChatFabVisiblePath('/issues')).toBe(true); + expect(isChatFabVisiblePath('/home')).toBe(false); + expect(isChatFabVisiblePath('/chat')).toBe(false); + expect(isChatFabVisiblePath('/pipeline')).toBe(false); + }); }); diff --git a/web/src/lib/chatUrlState.ts b/web/src/lib/chatUrlState.ts index cbf44e15..f7f0c486 100644 --- a/web/src/lib/chatUrlState.ts +++ b/web/src/lib/chatUrlState.ts @@ -1,3 +1,5 @@ +import { pathSlugToViewId } from '@/routes'; + const STORAGE_KEY = 'chat:last-context:v1'; export interface ChatUrlContext { @@ -56,6 +58,25 @@ export function buildChatSearchQuery( return params.toString(); } +/** Report views that carry ?domain= (not home, chat, or pipeline). */ +export function isChatFabVisiblePath(pathname: string): boolean { + if (pathname === '/chat' || pathname.startsWith('/chat/')) return false; + if (pathname === '/pipeline' || pathname.startsWith('/pipeline/')) return false; + if (pathname === '/home') return false; + const slug = pathname.replace(/^\//, '').split('/')[0] ?? ''; + const viewId = pathSlugToViewId(slug); + return viewId != null && viewId !== 'home'; +} + +/** Deep link into chat scoped to the current site domain. */ +export function buildChatFabHref(domain: string | null | undefined): string { + const trimmed = (domain ?? '').trim(); + if (!trimmed) return '/chat'; + const params = new URLSearchParams(); + params.set('domain', trimmed); + return `/chat?${params.toString()}`; +} + export function applyChatUrlContext( params: URLSearchParams, ctx: ChatUrlContext, diff --git a/web/src/lib/pipelineConfigSchema.ts b/web/src/lib/pipelineConfigSchema.ts index 0e64427a..2d348a50 100644 --- a/web/src/lib/pipelineConfigSchema.ts +++ b/web/src/lib/pipelineConfigSchema.ts @@ -244,6 +244,17 @@ export const PIPELINE_CONFIG_SECTIONS: PipelineConfigSection[] = [ help: 'Authorized testing only. Sends controlled probes beyond passive header checks.', }, { key: 'security_max_urls_probe', label: 'Security max URLs to probe', type: 'number', defaultValue: '20' }, + { + key: 'probe_image_inventory', + label: 'Probe image URLs (size/MIME)', + type: 'bool', + defaultValue: false, + help: 'HEAD/GET discovered image URLs during report build for largest/unoptimized image tools.', + }, + { key: 'max_image_probe_urls', label: 'Max image URLs to probe', type: 'number', defaultValue: '500' }, + { key: 'image_probe_concurrency', label: 'Image probe concurrency', type: 'number', defaultValue: '6' }, + { key: 'image_probe_timeout', label: 'Image probe timeout (seconds)', type: 'number', defaultValue: '8' }, + { key: 'image_unoptimized_min_kb', label: 'Unoptimized image min size (KB)', type: 'number', defaultValue: '200' }, ], }, { diff --git a/web/src/strings.json b/web/src/strings.json index 3de47c8d..a10a9413 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -2161,7 +2161,9 @@ "What is our schema markup coverage?", "Which pages have thin content?", "Show content duplicate clusters", - "What changed in health score over time?" + "What changed in health score over time?", + "Download the audit as PDF", + "Which images are largest and unoptimized?" ], "toolActivityTitle": "Tools used", "toolRunning": "running…", @@ -2183,6 +2185,8 @@ "ollamaLabel": "Ollama", "ollamaNoModel": "no model selected", "ollamaUnreachable": "Cannot reach Ollama", + "fabTitle": "AI Chat", + "fabAria": "Open AI chat for this site", "ollamaToolsMode": "native tools", "ollamaReactMode": "JSON tool mode", "ollamaModelMissing": "Configured model not in local or cloud catalog", @@ -2230,7 +2234,24 @@ "ctr": "CTR", "topQueries": "Top queries", "topPages": "Top pages", - "askTopQuery": "Analyze search performance for query: {query}" + "askTopQuery": "Analyze search performance for query: {query}", + "imageAudit": { + "title": "Image audit", + "subtitle": "Crawl + social + Lighthouse image signals", + "totalImages": "images crawled", + "missingAlt": "Missing alt", + "noLazyLoad": "No lazy load", + "missingDimensions": "No dimensions", + "lighthouseIssues": "LH image issues", + "ogCoverage": "OG image coverage", + "sizeProbe": "Size probe", + "probeOn": "enabled", + "probeOff": "not in report", + "issueBreakdown": "Page-level image issues", + "lighthouseTitle": "Lighthouse image opportunities", + "morePages": "{count} more not shown", + "exportList": "Export the full list of {topic} as CSV" + } }, "navHome": "Home", "navGsc": "Search Console", @@ -2511,6 +2532,7 @@ "totalImages": "Total Images", "missingAlt": "Missing Alt", "noLazyLoad": "No Lazy Load", + "noDimensions": "No width/height", "ariaElements": "ARIA Elements", "assetBarLabels": [ "Scripts", diff --git a/web/src/types/report.ts b/web/src/types/report.ts index 6baa7a2a..0ee044b4 100644 --- a/web/src/types/report.ts +++ b/web/src/types/report.ts @@ -723,6 +723,7 @@ export interface LinkDetail extends Omit { @@ -131,7 +138,7 @@ export default function ChatPage() { } finally { setLoadingProperties(false); } - }, [configState.active_property_id, configState.start_url]); + }, [configState.active_property_id, configState.start_url, searchParams]); const resolveSessionFromUrl = useCallback(async (sid: number, pid: number | null) => { try { @@ -507,7 +514,6 @@ export default function ChatPage() { propertyId={propertyId} sessionTitle={activeSession?.title} loading={loadingProperties} - onExpandSidebar={layout.expanded ? undefined : () => layout.setExpanded(true)} /> {!llmEnabled ? (
From 4abe769a746ee3e8d145fad048c77e3a3e0a9bbc Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 7 Jun 2026 20:57:47 +0530 Subject: [PATCH 2/4] Mostly UI --- web/app/page.tsx | 31 +- .../components/GoogleIntegrationsPanel.tsx | 307 ++++++++++++++---- web/src/components/IntegrationsModal.tsx | 16 +- web/src/components/LandingShell.tsx | 86 +++++ .../components/landing/LandingGoogleSetup.tsx | 145 +++++++++ .../components/landing/LandingPathStrip.tsx | 49 +++ .../landing/LandingSectionHeader.tsx | 37 +++ web/src/strings.json | 170 ++++++++++ web/src/views/Home.tsx | 110 +++++-- web/src/views/Landing.tsx | 281 ++++++++++++++++ 10 files changed, 1121 insertions(+), 111 deletions(-) create mode 100644 web/src/components/LandingShell.tsx create mode 100644 web/src/components/landing/LandingGoogleSetup.tsx create mode 100644 web/src/components/landing/LandingPathStrip.tsx create mode 100644 web/src/components/landing/LandingSectionHeader.tsx create mode 100644 web/src/views/Landing.tsx diff --git a/web/app/page.tsx b/web/app/page.tsx index 9c5c576f..ff7050b0 100644 --- a/web/app/page.tsx +++ b/web/app/page.tsx @@ -1,23 +1,16 @@ -import { redirect } from 'next/navigation'; +import type { Metadata } from 'next'; +import LandingPage from '@/views/Landing'; +import { strings } from '@/lib/strings'; + +const vl = strings.views.landing; + +export const metadata: Metadata = { + title: vl.metaTitle, + description: vl.metaDescription, +}; export const dynamic = 'force-dynamic'; -export default async function RootPage({ - searchParams, -}: { - searchParams: Promise>; -}) { - const params = await searchParams; - const sp = new URLSearchParams(); - for (const [key, value] of Object.entries(params)) { - if (typeof value === 'string') { - sp.set(key, value); - } else if (Array.isArray(value)) { - for (const v of value) { - sp.append(key, v); - } - } - } - const q = sp.toString(); - redirect(q ? `/home?${q}` : '/home'); +export default function RootPage() { + return ; } diff --git a/web/src/components/GoogleIntegrationsPanel.tsx b/web/src/components/GoogleIntegrationsPanel.tsx index 120a8094..84602e6b 100644 --- a/web/src/components/GoogleIntegrationsPanel.tsx +++ b/web/src/components/GoogleIntegrationsPanel.tsx @@ -11,6 +11,7 @@ import { KeyRound, Link2, BarChart3, + Settings2, } from 'lucide-react'; import type { GooglePropertiesResponse, GoogleStatusResponse, IntegrationToast } from '@/types/api'; import { apiUrl } from '@/lib/publicBase'; @@ -26,8 +27,12 @@ import { pickInitialPropertyId, siteUrlFromProperty } from '@/lib/googleProperty import { deriveSiteNameFromStartUrl } from '@/lib/domainSlug'; import type { PropertyListItem } from '@/types/api'; import Button from '@/components/Button'; +import ViewTabs from '@/components/ViewTabs'; +import { ViewTabPanel } from '@/components/ViewTabPanel'; import { useReadOnlySession } from '@/hooks/useReadOnlySession'; +type IntegrationsTabId = 'connect' | 'properties' | 'imports' | 'settings'; + const GCP_GUIDE_URL = 'https://developers.google.com/workspace/guides/get-started'; @@ -221,6 +226,8 @@ function InputField({ export interface GoogleIntegrationsPanelProps { initialToast?: IntegrationToast | null; showTitle?: boolean; + /** Tabbed layout for modal / large containers. */ + layout?: 'default' | 'tabbed'; /** When omitted, resolved from pipeline/report Site URL. */ propertyId?: number | null; /** Site URL used to resolve the property row when propertyId is omitted. */ @@ -233,6 +240,7 @@ export interface GoogleIntegrationsPanelProps { export default function GoogleIntegrationsPanel({ initialToast, showTitle = true, + layout = 'default', propertyId: propertyIdProp, startUrl: startUrlProp = '', }: GoogleIntegrationsPanelProps) { @@ -381,6 +389,7 @@ export default function GoogleIntegrationsPanel({ const linksFileInputRef = useRef(null); // Advanced accordion (paste refresh token) + const [activeTab, setActiveTab] = useState('connect'); const [showAdvanced, setShowAdvanced] = useState(false); const [refreshToken, setRefreshToken] = useState(''); const [savingToken, setSavingToken] = useState(false); @@ -778,19 +787,24 @@ export default function GoogleIntegrationsPanel({ const step2Done = Boolean(connected); const needsProperty = effectivePropertyId == null && !startUrl.trim(); + const isTabbed = layout === 'tabbed'; - return ( -
- {readOnly ? ( -

- {strings.app.readonlyBanner} -

- ) : null} -

- Google Client ID/Secret and service account keys are stored in the database. Each site keeps its own - OAuth connection and Search Console / Analytics property IDs. -

+ const readOnlyBanner = readOnly ? ( +

+ {strings.app.readonlyBanner} +

+ ) : null; + + const infoBannerText = + 'Google Client ID/Secret and service account keys are stored in the database. Each site keeps its own OAuth connection and Search Console / Analytics property IDs.'; + const infoBanner = ( +

+ {infoBannerText} +

+ ); + + const propertySelector = (
) : null}
+ ); - {needsProperty ? ( -

- Set a Site URL under Crawl settings so this audit can link Google Search Console and Analytics to the - correct domain. -

- ) : null} - {showTitle ? ( -
-
- - - -
-

Connect Search Console & Analytics

-

- Connect Search Console and Analytics 4, then choose properties to sync with your reports. -

-
+ const needsPropertyWarning = needsProperty ? ( +

+ Set a Site URL under Crawl settings so this audit can link Google Search Console and Analytics to the + correct domain. +

+ ) : null; + + const titleBlock = showTitle ? ( +
+
+ + + +
+

Connect Search Console & Analytics

+

+ Connect Search Console and Analytics 4, then choose properties to sync with your reports. +

+
+
+ +
+ ) : null; + + const compactContextBar = isTabbed ? ( +
+
+
+ + + +
+

Site & connection

+

Credentials are shared; OAuth is per site.

-
- ) : null} - - {toast ? ( -
- {toast.type === 'success' ? ( - + +
+
+
+ + {loadingPropertyRows ? ( +
+ + Loading sites… +
+ ) : propertyRows.length === 0 && effectivePropertyId == null ? ( +

{s.googlePropertySelectorEmpty}

+ ) : propertyRows.length === 0 && startUrl.trim() ? ( +

+ Site: {startUrl.trim()} +

) : ( - + )} - {toast.message} -
- ) : null} - - {loadingStatus ? ( -
- - Loading connection status…
+ {selectedProperty && status ? ( +
+

+ {format(s.googlePropertyContextTitle, { name: selectedProperty.name })} +

+

+ {format(s.googlePropertyGscGa4, { + gsc: status.gscSiteUrl || '—', + ga4: status.ga4PropertyId || '—', + })} +

+
+ ) : null} +
+
+ ) : null; + + const toastBlock = toast ? ( +
+ {toast.type === 'success' ? ( + ) : ( - <> + + )} + {toast.message} +
+ ) : null; + + const credentialsStep = (
+ ); + const connectStep = ( )} + ); + + const loadingBlock = ( +
+ + Loading connection status… +
+ ); - {connected ? ( + const connectNotReadyMessage = ( +

+ Connect your Google account in the Connect tab first. +

+ ); + + const propertiesStep = connected ? ( ) : null} - ) : null} + ) : null; - {effectivePropertyId != null && endpoints.linksImport ? ( + const linksStep = effectivePropertyId != null && endpoints.linksImport ? ( ) : null} - ) : null} - - - - {effectivePropertyId != null ? ( - - ) : null} + ) : null; - {effectivePropertyId != null ? ( + const advancedSection = effectivePropertyId != null ? (
+ ) : null; + + const integrationTabs = [ + { id: 'connect' as const, label: 'Connect', icon: }, + { id: 'properties' as const, label: 'Properties & sync', icon: }, + { id: 'imports' as const, label: 'Imports', icon: }, + { id: 'settings' as const, label: 'Settings', icon: }, + ]; + + if (isTabbed) { + return ( +
+
+ {readOnlyBanner} + {compactContextBar} + {needsPropertyWarning} + {toastBlock} + setActiveTab(tabId as IntegrationsTabId)} + ariaLabel="Integration sections" + idPrefix="integrations" + className="border-t border-muted/60 pt-3" + /> +
+
+ {loadingStatus ? ( + loadingBlock + ) : ( + <> + {activeTab === 'connect' ? ( + +

+ {infoBannerText} +

+
+ {credentialsStep} + {connectStep} +
+ + ) : null} + {activeTab === 'properties' ? ( + + {propertiesStep ?? connectNotReadyMessage} + + ) : null} + {activeTab === 'imports' ? ( + + {linksStep} + + + ) : null} + {activeTab === 'settings' ? ( + + {effectivePropertyId != null ? ( + + ) : ( +

+ Select a site above to configure schedules and alerts. +

+ )} + {advancedSection} + + ) : null} + + )} +
+
+ ); + } + + return ( +
+ {readOnlyBanner} + {infoBanner} + {propertySelector} + {needsPropertyWarning} + {titleBlock} + {toastBlock} + {loadingStatus ? ( + loadingBlock + ) : ( +
+ {credentialsStep} + {connectStep} + {propertiesStep} + {linksStep} + + {effectivePropertyId != null ? ( + ) : null} - + {advancedSection} +
)}
); diff --git a/web/src/components/IntegrationsModal.tsx b/web/src/components/IntegrationsModal.tsx index 90e9e342..8410c086 100644 --- a/web/src/components/IntegrationsModal.tsx +++ b/web/src/components/IntegrationsModal.tsx @@ -25,11 +25,16 @@ export default function IntegrationsModal({ open, onClose, initialToast }: Integ className="fixed inset-0 z-50 flex items-center justify-center p-4 bg-black/60 backdrop-blur-sm" role="presentation" > -
+
-
- -

Google Integrations

+
+
+ +

Google Integrations

+
+

+ Connect Search Console, Analytics, and related data sources for this site. +

-
+
diff --git a/web/src/components/LandingShell.tsx b/web/src/components/LandingShell.tsx new file mode 100644 index 00000000..d46642e8 --- /dev/null +++ b/web/src/components/LandingShell.tsx @@ -0,0 +1,86 @@ +'use client'; + +import Link from 'next/link'; +import type { ReactNode } from 'react'; +import AppLogo from '@/components/AppLogo'; +import ThemeToggle from '@/components/ThemeToggle'; +import { strings } from '@/lib/strings'; + +export interface LandingShellProps { + children: ReactNode; + footer?: ReactNode; +} + +const NAV_ITEMS = [ + { href: '#how-it-works', labelKey: 'navHowItWorks' as const }, + { href: '#quick-start', labelKey: 'navQuickStart' as const }, + { href: '#google-setup', labelKey: 'navGoogleSetup' as const }, + { href: '#features', labelKey: 'navFeatures' as const }, +]; + +export default function LandingShell({ children, footer }: LandingShellProps) { + const vl = strings.views.landing; + const app = strings.app; + + return ( +
+
+
+ + + {app.productName} + + +
+ + + {vl.navOpenApp} + + + {vl.navRunAudit} + +
+
+ +
+ +
{children}
+ + {footer ? ( +
{footer}
+ ) : null} +
+ ); +} diff --git a/web/src/components/landing/LandingGoogleSetup.tsx b/web/src/components/landing/LandingGoogleSetup.tsx new file mode 100644 index 00000000..1220270f --- /dev/null +++ b/web/src/components/landing/LandingGoogleSetup.tsx @@ -0,0 +1,145 @@ +'use client'; + +import { useState } from 'react'; +import Link from 'next/link'; +import { ChevronDown, ExternalLink } from 'lucide-react'; +import Button from '@/components/Button'; +import { strings } from '@/lib/strings'; + +type GuideSection = { + title: string; + items: string[]; + linkLabel?: string; + linkUrl?: string; +}; + +const SECTION_ORDER = [ + 'prerequisites', + 'gcpProject', + 'enableApis', + 'oauthConsent', + 'oauthClient', + 'serviceAccount', + 'gscProperty', + 'ga4Property', + 'inApp', +] as const; + +export default function LandingGoogleSetup() { + const vl = strings.views.landing; + const sections = vl.googleSetupSections as Record; + const [openIds, setOpenIds] = useState>(() => new Set(['prerequisites', 'oauthClient'])); + + const toggle = (id: string) => { + setOpenIds((prev) => { + const next = new Set(prev); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + }; + + const expandAll = () => setOpenIds(new Set(SECTION_ORDER)); + const collapseAll = () => setOpenIds(new Set()); + + return ( +
+
+
+

+ {vl.sectionSetupGuide} +

+
+
+

{vl.googleSetupTitle}

+

+ {vl.googleSetupSubtitle} +

+

+ {vl.googleSetupNote} +

+
+
+ + + + + +
+
+
+ +
+ {SECTION_ORDER.map((id, index) => { + const section = sections[id]; + if (!section) return null; + const expanded = openIds.has(id); + const panelId = `google-setup-panel-${id}`; + return ( +
+ + {expanded ? ( +
+
    + {section.items.map((item) => ( +
  1. {item}
    2. + ))} +
+ {section.linkLabel && section.linkUrl ? ( + + {section.linkLabel} + + + ) : null} +
+ ) : null} +
+ ); + })} +
+
+
+ ); +} diff --git a/web/src/components/landing/LandingPathStrip.tsx b/web/src/components/landing/LandingPathStrip.tsx new file mode 100644 index 00000000..e0173f06 --- /dev/null +++ b/web/src/components/landing/LandingPathStrip.tsx @@ -0,0 +1,49 @@ +'use client'; + +import { ArrowRight, BarChart2, Download, Play, Settings2 } from 'lucide-react'; +import { strings } from '@/lib/strings'; + +const vl = strings.views.landing; + +const STEPS = [ + { id: 'quick-start', icon: Download, label: vl.pathStepInstall, hint: vl.pathStepInstallHint }, + { id: 'how-it-works', icon: Play, label: vl.pathStepCrawl, hint: vl.pathStepCrawlHint }, + { id: 'google-setup', icon: Settings2, label: vl.pathStepGoogle, hint: vl.pathStepGoogleHint }, + { id: 'features', icon: BarChart2, label: vl.pathStepReport, hint: vl.pathStepReportHint }, +] as const; + +export default function LandingPathStrip() { + return ( +
+

+ {vl.pathTitle} +

+
+ {STEPS.map(({ id, icon: Icon, label, hint }, index) => ( +
+ + + + + + + {label} + + {hint} + + + {index < STEPS.length - 1 ? ( + + ) : null} +
+ ))} +
+
+ ); +} diff --git a/web/src/components/landing/LandingSectionHeader.tsx b/web/src/components/landing/LandingSectionHeader.tsx new file mode 100644 index 00000000..060c706f --- /dev/null +++ b/web/src/components/landing/LandingSectionHeader.tsx @@ -0,0 +1,37 @@ +import type { ReactNode } from 'react'; + +interface LandingSectionHeaderProps { + eyebrow?: string; + title: string; + subtitle?: string; + action?: ReactNode; + centered?: boolean; +} + +export default function LandingSectionHeader({ + eyebrow, + title, + subtitle, + action, + centered = true, +}: LandingSectionHeaderProps) { + const align = centered ? 'text-center' : 'text-left'; + return ( +
+
+ {eyebrow ? ( +

+ {eyebrow} +

+ ) : null} +

{title}

+ {subtitle ? ( +

{subtitle}

+ ) : null} +
+ {action ?
{action}
: null} +
+ ); +} diff --git a/web/src/strings.json b/web/src/strings.json index a10a9413..5c51dfdb 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -672,6 +672,175 @@ } }, "views": { + "landing": { + "metaTitle": "Site Audit — Self-hosted technical SEO", + "metaDescription": "Crawl your sites, find technical SEO issues, connect Search Console and Analytics, and export client reports — free and self-hosted.", + "navFeatures": "Features", + "navHowItWorks": "How it works", + "navQuickStart": "Quick start", + "navGoogleSetup": "Google setup", + "navOpenApp": "Dashboard", + "navRunAudit": "Run audit", + "heroBadge": "Open source · Self-hosted", + "heroTitle": "Technical SEO audits you control", + "heroSubtitle": "Crawl sites, surface real issues, connect Search Console and Analytics, and ship client reports — on infrastructure you own.", + "ctaDashboard": "Open dashboard", + "ctaRunAudit": "Run your first audit", + "ctaGoogleGuide": "Google setup guide", + "scrollHint": "New here? Follow the steps below.", + "pathTitle": "Get started", + "pathStepInstall": "Install", + "pathStepInstallHint": "Docker or local dev", + "pathStepCrawl": "Crawl", + "pathStepCrawlHint": "Run your first audit", + "pathStepGoogle": "Connect Google", + "pathStepGoogleHint": "GSC + GA4 OAuth", + "pathStepReport": "Review", + "pathStepReportHint": "Issues & exports", + "sectionGettingStarted": "Getting started", + "sectionCapabilities": "Capabilities", + "sectionSetupGuide": "Setup guide", + "featuresTitle": "Everything you need for technical SEO", + "featuresSubtitle": "Familiar workflows from Screaming Frog, SEMrush, and Ahrefs — in one open-source tool you run yourself.", + "featureCrawlTitle": "Site crawl", + "featureCrawlDescription": "Crawl every URL, status codes, redirect chains, and site structure — like Screaming Frog.", + "featureIssuesTitle": "Issues & health", + "featureIssuesDescription": "Prioritized technical issues and site health scores — like SEMrush Site Audit.", + "featureOnPageTitle": "On-page & performance", + "featureOnPageDescription": "Titles, meta, content checks, and Lighthouse Core Web Vitals — like PageSpeed Insights.", + "featureSearchTitle": "Search & traffic", + "featureSearchDescription": "Search Console queries and GA4 traffic when you connect Google.", + "featureKeywordsTitle": "Keywords & indexation", + "featureKeywordsDescription": "Keyword explorer, query–page alignment, and indexation coverage.", + "featureLinksTitle": "Links & logs", + "featureLinksDescription": "Backlinks from GSC Links import and access-log analysis — Ahrefs-style insights from your data.", + "featureAiTitle": "AI assistant", + "featureAiDescription": "Ask questions about your audit in chat or via MCP tools in Cursor and Claude Desktop.", + "featureSelfHostedTitle": "Self-hosted", + "featureSelfHostedDescription": "Run with Docker or locally. Your crawl data stays on your infrastructure.", + "howItWorksTitle": "How it works", + "howItWorksSubtitle": "Three steps from first crawl to client-ready report.", + "step1Title": "Run a crawl", + "step1Description": "Enter a start URL, choose crawl depth, and launch a full site audit from Run audit.", + "step2Title": "Connect Google", + "step2Description": "Create OAuth credentials in Google Cloud, then connect Search Console and GA4 per property.", + "step2Link": "Google setup guide", + "step3Title": "Review reports", + "step3Description": "Open your property from the dashboard, explore issues, and export HTML or PDF for clients.", + "step3Link": "Open dashboard", + "quickStartTitle": "Quick start", + "quickStartSubtitle": "Get running in minutes with Docker or local development.", + "quickStartDockerLabel": "Docker", + "quickStartDockerCommand": "docker compose up --build", + "quickStartLocalLabel": "Local development", + "quickStartLocalSetup": "./local-run setup", + "quickStartLocalRun": "./local-run", + "quickStartDocsHint": "See CONTRIBUTING.md and docs/MCP.md in the repository for full setup and AI chat configuration.", + "googleSetupTitle": "Google Search Console & Analytics setup", + "googleSetupSubtitle": "Step-by-step guide to create a Google Cloud project, OAuth credentials, and connect GSC + GA4 to your properties.", + "googleSetupCta": "Open Integrations", + "googleSetupNote": "OAuth Client ID/Secret are stored once for the whole app. Each site property keeps its own Google sign-in and GSC/GA4 property IDs.", + "googleSetupSections": { + "prerequisites": { + "title": "Before you start", + "items": [ + "A Google account with access to the site's Search Console property (Owner or Full user).", + "A GA4 property for the same site (Admin access to read Analytics data).", + "Verify the site in Search Console (domain or URL-prefix property) before connecting.", + "In Site Audit, set a Site URL under Run audit → Crawl so the property can be linked." + ] + }, + "gcpProject": { + "title": "Create a Google Cloud project", + "items": [ + "Go to Google Cloud Console and create a new project (or pick an existing one).", + "Note the project name — you will enable APIs and create credentials here.", + "Billing is not required for Search Console and Analytics Data API reads within normal quotas." + ], + "linkLabel": "Google Cloud Console", + "linkUrl": "https://console.cloud.google.com/" + }, + "enableApis": { + "title": "Enable required APIs", + "items": [ + "In APIs & Services → Library, enable Google Search Console API.", + "Enable Google Analytics Data API (for GA4 runReport).", + "Wait a minute after enabling before creating credentials or testing." + ], + "linkLabel": "Enable APIs", + "linkUrl": "https://console.cloud.google.com/apis/library" + }, + "oauthConsent": { + "title": "Configure OAuth consent screen", + "items": [ + "Open APIs & Services → OAuth consent screen.", + "Choose External (or Internal if using Google Workspace for your org only).", + "Add app name, support email, and developer contact.", + "Add scopes: .../auth/webmasters.readonly and .../auth/analytics.readonly.", + "Add your Google account as a test user while the app is in Testing mode." + ], + "linkLabel": "OAuth consent screen", + "linkUrl": "https://console.cloud.google.com/apis/credentials/consent" + }, + "oauthClient": { + "title": "Create OAuth client (recommended)", + "items": [ + "Go to APIs & Services → Credentials → Create credentials → OAuth client ID.", + "Application type: Web application.", + "Authorized redirect URI (local dev): http://localhost:3000/api/integrations/google/callback", + "For production, add the same path on your public origin (or set GOOGLE_REDIRECT_URI).", + "Copy the Client ID and Client Secret into Site Audit → Integrations → Connect tab → Save credentials.", + "Per site: pick the property, click Connect with Google, and sign in with an account that has GSC + GA4 access." + ], + "linkLabel": "Create credentials", + "linkUrl": "https://console.cloud.google.com/apis/credentials" + }, + "serviceAccount": { + "title": "Alternative: service account (advanced)", + "items": [ + "IAM & Admin → Service accounts → Create → Keys → Add key → JSON.", + "Grant the service account access: add its email as a user in Search Console (property settings) and GA4 (Property access management) with Viewer role.", + "Upload the JSON key via the credentials upload API or pipeline settings (service account replaces OAuth app credentials at the app level).", + "Service accounts cannot use the Connect with Google button — use this only if your workflow requires non-interactive access." + ], + "linkLabel": "Service accounts", + "linkUrl": "https://console.cloud.google.com/iam-admin/serviceaccounts" + }, + "gscProperty": { + "title": "Search Console property", + "items": [ + "In Google Search Console, confirm the property URL matches how you crawl (e.g. https://www.example.com/ or sc-domain:example.com).", + "After OAuth connect, open Integrations → Properties & sync → Load properties and select the GSC site.", + "If the site is missing, the signed-in user may lack access or the property is not verified." + ], + "linkLabel": "Search Console", + "linkUrl": "https://search.google.com/search-console" + }, + "ga4Property": { + "title": "Google Analytics 4 property", + "items": [ + "In GA4 Admin → Property settings, copy the numeric Property ID (e.g. 123456789).", + "Do not use the G-XXXXXXXX Measurement ID — Site Audit needs the numeric property ID.", + "In Integrations → Properties & sync, select the GA4 property or paste the numeric ID, then Save properties.", + "Use Test connection, then Fetch data now to pull queries and traffic into reports." + ], + "linkLabel": "GA4 Admin", + "linkUrl": "https://analytics.google.com/" + }, + "inApp": { + "title": "Connect inside Site Audit", + "items": [ + "Open a report and click the gear icon (Integrations), or go to Run audit → Search Console & Analytics tab.", + "Connect tab: save Client ID + Secret, then Connect with Google for the selected site.", + "Properties & sync tab: choose GSC site, GA4 property ID, and date range; save and fetch.", + "Imports tab: optionally upload GSC Links CSV exports for backlink samples." + ] + } + }, + "footerOpenApp": "Open app", + "footerRunAudit": "Run audit", + "footerChat": "AI chat" + }, "home": { "title": "Properties", "subtitle": "Client sites from crawls and audit runs. Search by domain or start URL.", @@ -681,6 +850,7 @@ "totalUrlsLabel": "URLs crawled", "avgHealthLabel": "Avg site health", "brandLabel": "Property", + "groupPropertyCount": "{count} properties", "healthScoreLabel": "Site health", "crawlUrlLabel": "Site URL", "urlCountLabel": "URLs", diff --git a/web/src/views/Home.tsx b/web/src/views/Home.tsx index a07e4996..38fe082c 100644 --- a/web/src/views/Home.tsx +++ b/web/src/views/Home.tsx @@ -1,10 +1,12 @@ -import { Building2, ExternalLink, Globe, ArrowRight, Search, Settings2, Trash2 } from 'lucide-react'; +import { Building2, ChevronDown, ExternalLink, Globe, ArrowRight, Search, Trash2 } from 'lucide-react'; import { useMemo, useState, useEffect, useCallback } from 'react'; +import AppLogo from '@/components/AppLogo'; import { PageLayout, Card } from '../components'; import HealthSparkline from '@/components/HealthSparkline'; import { Skeleton, SkeletonDomainCard } from '../components/Skeleton'; import { useReport } from '../context/useReport'; import { format, strings } from '../lib/strings'; +import { extractHostname } from '@/lib/domainSlug'; import { apiUrl, reportApi } from '../lib/publicBase'; import type { PortfolioGroup, ReportCategory, ViewProps } from '@/types'; @@ -30,7 +32,15 @@ function healthScoreClass(score: number): string { return 'text-rose-700 dark:text-rose-400'; } -export default function Home({ onNavigate, onOpenIntegrations }: ViewProps) { +function portfolioRootDomain(group: PortfolioGroup): string { + const host = extractHostname(group.crawlUrl) || group.domainName.trim().toLowerCase(); + if (!host) return group.domainName || 'unknown'; + const parts = host.split('.').filter(Boolean); + if (parts.length <= 2) return host; + return parts.slice(-2).join('.'); +} + +export default function Home({ onNavigate }: ViewProps) { const { reportList, crawlRuns, loadCrawlPreview, refreshReports } = useReport(); const vh = strings.views.home; const sj = strings.common; @@ -42,6 +52,16 @@ export default function Home({ onNavigate, onOpenIntegrations }: ViewProps) { const [deletingKey, setDeletingKey] = useState(null); const [deleteError, setDeleteError] = useState(null); const [healthHistoryByDomain, setHealthHistoryByDomain] = useState>({}); + const [collapsedGroups, setCollapsedGroups] = useState>(() => new Set()); + + const toggleGroupCollapsed = useCallback((rootDomain: string) => { + setCollapsedGroups((prev) => { + const next = new Set(prev); + if (next.has(rootDomain)) next.delete(rootDomain); + else next.add(rootDomain); + return next; + }); + }, []); const portfolioCardKey = (group: PortfolioGroup) => `${group.domainParam}-${group.crawlOnly ? 'crawl' : 'report'}-${group.reportId ?? 'nr'}-${group.crawlRunId ?? 'nc'}-${group.generatedAtMs}`; @@ -174,6 +194,22 @@ export default function Home({ onNavigate, onOpenIntegrations }: ViewProps) { )); }, [domainGroups, filterQuery]); + const groupedPortfolio = useMemo(() => { + const map = new Map(); + for (const group of filteredGroups) { + const key = portfolioRootDomain(group); + const items = map.get(key) ?? []; + items.push(group); + map.set(key, items); + } + return Array.from(map.entries()) + .map(([rootDomain, items]) => ({ + rootDomain, + items: items.toSorted((a, b) => b.generatedAtMs - a.generatedAtMs), + })) + .toSorted((a, b) => (b.items[0]?.generatedAtMs ?? 0) - (a.items[0]?.generatedAtMs ?? 0)); + }, [filteredGroups]); + const emptyMessage = filterQuery ? vh.noSearchResults : vh.empty; @@ -192,20 +228,12 @@ export default function Home({ onNavigate, onOpenIntegrations }: ViewProps) {
+
+ +

{vh.title}

{vh.subtitle}

- {onOpenIntegrations ? ( - - ) : null} -
+
{strings.app.loading} -
- - - +
+ +
+ + + +
) : filteredGroups.length > 0 ? ( -
-
- {filteredGroups.map((group) => { +
+ {groupedPortfolio.map(({ rootDomain, items }) => { + const collapsed = collapsedGroups.has(rootDomain); + return ( +
+ + {!collapsed ? ( +
+ {items.map((group) => { const cardKey = portfolioCardKey(group); const confirmOpen = pendingDeleteKey === cardKey; const isDeleting = deletingKey === cardKey; return (
); })} -
+
+ ) : null} +
+ ); + })}
) : ( diff --git a/web/src/views/Landing.tsx b/web/src/views/Landing.tsx new file mode 100644 index 00000000..2c745496 --- /dev/null +++ b/web/src/views/Landing.tsx @@ -0,0 +1,281 @@ +'use client'; + +import Link from 'next/link'; +import { + AlertOctagon, + ArrowDown, + BarChart2, + ChevronRight, + Cpu, + FolderTree, + Gauge, + Key, + Link2, + MessageSquare, + Play, + Settings2, + TrendingUp, +} from 'lucide-react'; +import AppLogo from '@/components/AppLogo'; +import Button from '@/components/Button'; +import LandingGoogleSetup from '@/components/landing/LandingGoogleSetup'; +import LandingPathStrip from '@/components/landing/LandingPathStrip'; +import LandingSectionHeader from '@/components/landing/LandingSectionHeader'; +import LandingShell from '@/components/LandingShell'; +import { strings } from '@/lib/strings'; + +const vl = strings.views.landing; +const app = strings.app; + +const FEATURES = [ + { icon: FolderTree, title: vl.featureCrawlTitle, description: vl.featureCrawlDescription }, + { icon: AlertOctagon, title: vl.featureIssuesTitle, description: vl.featureIssuesDescription }, + { icon: Gauge, title: vl.featureOnPageTitle, description: vl.featureOnPageDescription }, + { icon: TrendingUp, title: vl.featureSearchTitle, description: vl.featureSearchDescription }, + { icon: Key, title: vl.featureKeywordsTitle, description: vl.featureKeywordsDescription }, + { icon: Link2, title: vl.featureLinksTitle, description: vl.featureLinksDescription }, + { icon: MessageSquare, title: vl.featureAiTitle, description: vl.featureAiDescription }, + { icon: Cpu, title: vl.featureSelfHostedTitle, description: vl.featureSelfHostedDescription }, +] as const; + +const STEPS = [ + { + step: 1, + icon: Play, + title: vl.step1Title, + description: vl.step1Description, + href: '/pipeline', + linkLabel: vl.ctaRunAudit, + }, + { + step: 2, + icon: Settings2, + title: vl.step2Title, + description: vl.step2Description, + href: '#google-setup', + linkLabel: vl.step2Link, + }, + { + step: 3, + icon: BarChart2, + title: vl.step3Title, + description: vl.step3Description, + href: '/home', + linkLabel: vl.step3Link, + }, +] as const; + +function CodeBlock({ label, command }: { label?: string; command: string }) { + return ( +
+ {label ? ( +

{label}

+ ) : null} + 
+        {command}
+
+
+ ); +} + +export default function LandingPage() { + const footer = ( +
+
+

{app.productName}

+

{app.productSubtitle}

+
+
+ + {vl.footerOpenApp} + + + {vl.footerRunAudit} + + + {vl.footerChat} + +
+
+ ); + + return ( + +
+
+
+
+
+
+ +
+
+ + {vl.heroBadge} + +
+ +
+

+ {vl.heroTitle} +

+

+ {vl.heroSubtitle} +

+
+ + + + + + +
+ + + {vl.scrollHint} + +
+ +
+

+ {vl.pathTitle} +

+
    + {STEPS.map(({ step, title, description, href, linkLabel }) => ( +
  1. + + + {step} + + + {title} + + {description} + + + {linkLabel} + + + + +
    2. + ))} +
+
+
+
+ + + +
+
+ +
+ {STEPS.map(({ step, icon: Icon, title, description, href, linkLabel }, index) => ( +
+ {index < STEPS.length - 1 ? ( + + ) : null} +
+ + {step} + + +
+

{title}

+

{description}

+ {href.startsWith('/') ? ( + + {linkLabel} + + + ) : ( + + {linkLabel} + + + )} +
+ ))} +
+
+
+ +
+ +
+ +
+ + +
+
+

{vl.quickStartDocsHint}

+
+ + + +
+ +
+ {FEATURES.map(({ icon: Icon, title, description }) => ( +
+ + + +

{title}

+

{description}

+
+ ))} +
+
+ + ); +} From 768bb88c9b4a3fec3816e7a32369173cd57421e5 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 7 Jun 2026 21:35:16 +0530 Subject: [PATCH 3/4] some test --- .github/workflows/ci.yml | 6 +- .gitignore | 1 + AGENT.md | 35 +- README.md | 12 - scripts/local-test.ps1 | 9 + scripts/local-test.sh | 71 ++- tests/test_audit_tools_dispatch_coverage.py | 479 ++++++++++++++++++++ tests/test_export_artifacts_coverage.py | 25 + tests/test_export_compare_coverage.py | 13 + tests/test_export_custom_coverage.py | 101 +++++ tests/test_export_tools_coverage.py | 160 +++++++ tests/test_schedule_runner.py | 6 +- 12 files changed, 894 insertions(+), 24 deletions(-) create mode 100644 tests/test_audit_tools_dispatch_coverage.py create mode 100644 tests/test_export_artifacts_coverage.py create mode 100644 tests/test_export_compare_coverage.py create mode 100644 tests/test_export_custom_coverage.py create mode 100644 tests/test_export_tools_coverage.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4901e7f4..0402ea0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,11 @@ jobs: run: | pytest tests/test_alert_checker.py tests/test_schedule_runner.py tests/test_export_audit.py \ tests/test_export_audit_coverage.py tests/test_audit_tools.py tests/test_audit_tools_expanded.py \ - tests/test_audit_tools_coverage.py \ + tests/test_audit_tools_coverage.py tests/test_audit_tools_dispatch_coverage.py \ + tests/test_export_custom_coverage.py tests/test_export_artifacts_coverage.py \ + tests/test_export_compare_coverage.py tests/test_export_tools_coverage.py \ + tests/test_image_tools.py tests/test_export_custom.py tests/test_export_artifacts.py \ + tests/test_export_compare.py \ tests/test_mcp_registry.py tests/test_mcp_resources.py \ --cov=website_profiling.tools --cov-config=.coveragerc.tools \ --cov-report=term-missing --cov-fail-under=95 -q -o addopts= diff --git a/.gitignore b/.gitignore index a1a3e6cd..9e22a9bd 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ input.txt # Runtime shadow written to DATA_DIR by the web UI (not the committed example) pipeline-config.txt *__pycache__* +.coverage diff --git a/AGENT.md b/AGENT.md index 55c5d8da..a93961df 100644 --- a/AGENT.md +++ b/AGENT.md @@ -12,7 +12,7 @@ - `web/app/` -- routes; `web/src/` -- React; pipeline: `PipelineRunnerFab`, `server/pipelineJobs.ts`, `server/pipelineConfig.ts`, `server/llmConfig.ts`, `server/db.ts` - `alembic/` -- schema migrations -**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, Next.js on host). See `scripts/local-run.sh`. **Local tests (CI parity):** `./local-test` (100% in-scope coverage gate); `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`. +**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, Next.js on host). See `scripts/local-run.sh`. **Local tests (CI parity):** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 95%); `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`. **JavaScript crawl (optional):** Config keys `crawl_render_mode` (`static` | `javascript` | `auto`) and `crawl_js_*` in pipeline config / `pipelineConfigSchema.ts`. JS/auto crawls can capture browser console errors and uncaught exceptions (`crawl_js_capture_console`, stored under `page_analysis.browser`). **Auto mode** uses static-first fetch, pre-parse SPA heuristics (`needs_js_render`), then post-parse low-outlink fallback (`needs_js_render_after_parse`) in `crawler.py`. **Preflight:** `GET /api/crawl/browser-status` (localhost) spawns Python `browser_status()`; Run audit settings/run validation calls it when render mode is `javascript` or `auto`. Browser deps: `requirements-browser.txt` (installed by `./local-run setup` and `./local-test`). Runtime needs Chromium on `PATH` or `CHROME_PATH` (Docker sets `CHROME_PATH=/usr/bin/chromium`). Integration tests: `@pytest.mark.browser` — excluded by default in `pytest.ini`; Docker CI runs `tests/test_crawl_fetchers.py` and `tests/test_crawler_browser_e2e.py -m browser`; locally `./local-test browser`. @@ -84,4 +84,35 @@ These recur when adding features. Verify explicitly — do not assume tests caug report_id = int(rid) if rid is not None else None ``` -**Checklist:** new report page uses `ReportShell` · no duplicate local imports in long functions · new `fetchone()` uses `_row_field` +4. **Python — local vs CI coverage gates (three jobs, not one)** + - CI runs **three separate** pytest coverage jobs (see `.github/workflows/ci.yml` and `scripts/local-test.sh`): + | Gate | Config | Source | Threshold | Test scope | + |------|--------|--------|-----------|------------| + | Core | `.coveragerc` | all packages **except** `tools/` and `reporting/` | 100% | `pytest tests/ -m "not browser"` | + | Reporting | `.coveragerc.reporting` | `website_profiling.reporting` | 100% | fixed test file list | + | Tools | `.coveragerc.tools` | `website_profiling.tools` | 95% | fixed test file list | + - **Symptom:** `./local-test` or core pytest passes at 100%, but CI fails on tools/reporting (e.g. 84% tools). + - **Causes:** (a) only ran core pytest, not reporting/tools gates; (b) added tests under `tests/test__coverage.py` but did not add the file to the tools gate list in **both** `scripts/local-test.sh`, `scripts/local-test.ps1`, and `.github/workflows/ci.yml`; (c) changed code under `website_profiling/tools/` without tests that hit those lines in the tools gate subset. + - **Do:** Run full `./local-test` before push. When adding tools coverage tests, name them `tests/test__coverage.py` (repo convention) and register the file in all three places above. Keep bash and PowerShell local-test scripts in sync. + - **Don't:** Assume `pytest tests/` alone matches CI. Don't rely on a single mega `test_tools_coverage_gaps.py` — split by module. + +5. **Python — `runpy.run_module` / `__main__` guard tests** + - Tests that execute a module as `__main__` via `runpy.run_module(..., run_name="__main__")` emit: + `RuntimeWarning: '' found in sys.modules after import of package ...` + when the same module was already imported at the top of the test file (or by another import). + - **Do:** Before `runpy.run_module`, remove the target from `sys.modules` so Python re-executes `__main__` cleanly. Name tests `test_module_main_guard` (see `tests/test_schedule_runner.py`). + - **Don't:** Call `runpy.run_module` on a module already imported in that test file without popping it first. + + ```python + import runpy + import sys + + sys.modules.pop("website_profiling.tools.schedule_runner", None) + runpy.run_module( + "website_profiling.tools.schedule_runner", + run_name="__main__", + alter_sys=False, + ) + ``` + +**Checklist:** new report page uses `ReportShell` · no duplicate local imports in long functions · new `fetchone()` uses `_row_field` · `./local-test` passes all three coverage gates · new tools coverage test file listed in CI + both local-test scripts · `runpy` main-guard tests pop `sys.modules` first diff --git a/README.md b/README.md index 3bdf5895..67270aad 100644 --- a/README.md +++ b/README.md @@ -18,18 +18,6 @@ docker compose up --build Open [http://localhost:3000/home](http://localhost:3000/home). -**Docker (published image)** - -The app requires PostgreSQL on the same Docker network. Do **not** run the image alone with `docker run` — the hostname `postgres` only resolves inside Compose. - -```bash -docker pull your-registry/website-profiling:tag -export WEB_IMAGE=your-registry/website-profiling:tag -docker compose -f docker-compose.pull.yml up -d -``` - -Open [http://localhost:3000/home](http://localhost:3000/home). - **Local dev** ```bash diff --git a/scripts/local-test.ps1 b/scripts/local-test.ps1 index a04218d9..a354c8fe 100644 --- a/scripts/local-test.ps1 +++ b/scripts/local-test.ps1 @@ -254,6 +254,15 @@ function Invoke-PytestTools { tests/test_audit_tools.py ` tests/test_audit_tools_expanded.py ` tests/test_audit_tools_coverage.py ` + tests/test_audit_tools_dispatch_coverage.py ` + tests/test_export_custom_coverage.py ` + tests/test_export_artifacts_coverage.py ` + tests/test_export_compare_coverage.py ` + tests/test_export_tools_coverage.py ` + tests/test_image_tools.py ` + tests/test_export_custom.py ` + tests/test_export_artifacts.py ` + tests/test_export_compare.py ` tests/test_mcp_registry.py ` tests/test_mcp_resources.py ` --cov=website_profiling.tools ` diff --git a/scripts/local-test.sh b/scripts/local-test.sh index 8d0172d4..960683fb 100755 --- a/scripts/local-test.sh +++ b/scripts/local-test.sh @@ -102,16 +102,71 @@ cmd_web_deps() { fi } -run_pytest() { +run_pytest_core() { if [[ "$PYTEST_NO_COV" -eq 1 ]]; then - log "Pytest (tests/ -q --no-cov)" - "$VENV/bin/pytest" tests/ -q --no-cov + log "Pytest (tests/ -q -m not browser --no-cov)" + "$VENV/bin/pytest" tests/ -q -m "not browser" --no-cov else - log "Pytest (tests/ -q, 100% in-scope coverage gate — same as CI)" - "$VENV/bin/pytest" tests/ -q + log "Pytest (tests/ -q -m not browser, core 100% coverage gate)" + "$VENV/bin/pytest" tests/ -q -m "not browser" fi } +run_pytest_reporting() { + [[ "$PYTEST_NO_COV" -eq 1 ]] && return 0 + log "Pytest (reporting coverage gate, 100%)" + "$VENV/bin/pytest" \ + tests/test_categories_roadmap.py \ + tests/test_report_categories_golden.py \ + tests/test_categories_coverage.py \ + tests/test_indexation_coverage.py \ + tests/test_crawl_segments.py \ + tests/test_terminology.py \ + tests/test_compare_payload.py \ + --cov=website_profiling.reporting \ + --cov-config=.coveragerc.reporting \ + --cov-report=term-missing \ + --cov-fail-under=100 \ + -q \ + -o addopts= +} + +run_pytest_tools() { + [[ "$PYTEST_NO_COV" -eq 1 ]] && return 0 + log "Pytest (tools coverage gate, 95%)" + "$VENV/bin/pytest" \ + tests/test_alert_checker.py \ + tests/test_schedule_runner.py \ + tests/test_export_audit.py \ + tests/test_export_audit_coverage.py \ + tests/test_audit_tools.py \ + tests/test_audit_tools_expanded.py \ + tests/test_audit_tools_coverage.py \ + tests/test_audit_tools_dispatch_coverage.py \ + tests/test_export_custom_coverage.py \ + tests/test_export_artifacts_coverage.py \ + tests/test_export_compare_coverage.py \ + tests/test_export_tools_coverage.py \ + tests/test_image_tools.py \ + tests/test_export_custom.py \ + tests/test_export_artifacts.py \ + tests/test_export_compare.py \ + tests/test_mcp_registry.py \ + tests/test_mcp_resources.py \ + --cov=website_profiling.tools \ + --cov-config=.coveragerc.tools \ + --cov-report=term-missing \ + --cov-fail-under=95 \ + -q \ + -o addopts= +} + +run_pytest() { + run_pytest_core + run_pytest_reporting + run_pytest_tools +} + run_browser_pytest() { if "$VENV/bin/python" -c "from website_profiling.crawl.fetchers import browser_status; import sys; sys.exit(0 if browser_status().get('ok') else 1)" 2>/dev/null; then log "Browser pytest (tests/test_crawl_fetchers.py tests/test_crawler_browser_e2e.py -m browser)" @@ -152,7 +207,7 @@ cmd_web() { cmd_all() { cmd_python cmd_web - ok "All local tests passed (CI python + web jobs)" + ok "All local tests passed (CI python + web jobs, including reporting/tools gates)" } cmd_quick() { @@ -179,8 +234,8 @@ cmd_help() { Local test runner — mirrors CI (python + web jobs) ./local-test Same as: all - ./local-test all Postgres + migrations + pytest + CLI + web checks - ./local-test python DB + pytest + browser pytest + python -m src --help + ./local-test all Postgres + migrations + pytest (core + reporting + tools) + CLI + web + ./local-test python DB + pytest (core + reporting + tools) + browser pytest + CLI smoke ./local-test browser Browser integration pytest only (skips if no Chromium) ./local-test web typecheck, lint, vitest (no Docker) ./local-test quick pytest + web without starting Docker (DB must be ready) diff --git a/tests/test_audit_tools_dispatch_coverage.py b/tests/test_audit_tools_dispatch_coverage.py new file mode 100644 index 00000000..175ada41 --- /dev/null +++ b/tests/test_audit_tools_dispatch_coverage.py @@ -0,0 +1,479 @@ +"""Line-coverage tests for audit_tools dispatch handlers.""" +from __future__ import annotations + +import json +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from website_profiling.tools.audit_tools import dispatch_tool +from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx + + +@pytest.fixture +def ctx() -> Ctx: + return Ctx(property_id=1, report_id=1) + + +@pytest.fixture +def conn() -> MagicMock: + return MagicMock() + + +def _payload() -> dict: + return { + "site_name": "Example", + "report_generated_at": "2026-06-07T12:00:00Z", + "categories": [{"id": "tech", "name": "Tech", "score": 80, "issues": []}], + "executive_summary": {"headline": "OK"}, + "content_urls": { + "missing_canonical": [{"url": "https://ex.com/no-canonical"}], + "canonical_mismatch": [{"url": "https://ex.com/m", "canonical_url": "https://ex.com/other"}], + "missing_alt": [{"url": "https://ex.com/alt"}], + }, + "social_coverage": {"og_image_missing": ["https://ex.com/no-og"]}, + "top_pages": [{"url": "https://ex.com/", "pagerank": 0.9, "inlinks": 3}], + "content_analytics": {"thin_pages": [{"url": "https://ex.com/thin", "word_count": 50}]}, + "content_duplicates": [ + { + "representative_url": "https://ex.com/dup", + "member_urls": ["https://ex.com/dup", "https://ex.com/dup2"], + }, + ], + "lighthouse_by_url": { + "https://ex.com/slow": { + "performance": 30, + "seo": 70, + "accessibility": 60, + "best-practices": 55, + "scores": {"performance": 30, "seo": 70}, + }, + "https://ex.com/ok": {"performance": 90, "seo": 95}, + }, + "lighthouse_diagnostics": [ + {"lighthouse_audit_id": "uses-optimized-images", "title": "Images", "url": "https://ex.com"}, + {"bad": True}, + ], + "crux_summary": {"available": True}, + "tech_stack_summary": {"technologies": [{"name": "React", "sample_urls": ["https://ex.com/app"]}]}, + } + + +def test_crawl_lists_paths(conn: MagicMock, ctx: Ctx) -> None: + df = pd.DataFrame([ + { + "url": "https://ex.com/a", + "status": "200", + "canonical_url": "", + "images_without_alt": 2, + "heading_sequence": "h1,h4", + "viewport_present": "false", + "redirect_chain_length": 5, + "og_image": "", + "tech_stack": '["Vue"]', + }, + { + "url": "https://ex.com/mismatch", + "status": "200", + "canonical_url": "https://ex.com/canonical", + "images_without_alt": 0, + "heading_sequence": "h1", + "viewport_present": "true", + "redirect_chain_length": 0, + "og_image": "https://ex.com/og.png", + "tech_stack": "[]", + }, + {"url": "https://ex.com/b", "status": "blocked_by_robots"}, + {"url": "https://ex.com/c", "status": "404"}, + ]) + payload = _payload() + payload.pop("content_urls") + with patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df): + assert dispatch_tool("list_pages_missing_canonical", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_canonical_mismatch", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_pages_with_missing_alt", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_robots_blocked_urls", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_pages_by_technology", {"technology_name": "Vue"}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_pages_by_technology", {}, context=ctx, conn=conn)["error"] + assert dispatch_tool("get_top_pages_by_pagerank", {}, context=ctx, conn=conn)["total"] == 1 + + with patch.object(Ctx, "load_payload", return_value={"content_urls": "bad"}): + assert dispatch_tool("list_pages_missing_canonical", {}, context=ctx, conn=conn)["missing"] + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame([{ + "url": "https://ex.com/h", "status": "200", "heading_sequence": "h1,h4", + }])): + assert dispatch_tool("list_pages_skipped_headings", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_canonical_mismatch", {}, context=ctx, conn=conn)["total"] == 0 + + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()): + assert dispatch_tool("list_pages_missing_viewport", {}, context=ctx, conn=conn)["note"] + + +def test_content_and_lighthouse_paths(conn: MagicMock, ctx: Ctx) -> None: + payload = _payload() + with patch.object(Ctx, "load_payload", return_value=payload): + thin = dispatch_tool("list_thin_content_pages", {}, context=ctx, conn=conn) + assert thin["total"] == 1 + by_url = dispatch_tool( + "get_duplicate_cluster", + {"url": "https://ex.com/dup2"}, + context=ctx, + conn=conn, + ) + assert by_url["cluster_index"] == 0 + assert dispatch_tool("get_duplicate_cluster", {"cluster_index": 99}, context=ctx, conn=conn)["error"] + assert dispatch_tool("get_duplicate_cluster", {"cluster_index": "x"}, context=ctx, conn=conn)["error"] + assert dispatch_tool("get_duplicate_cluster", {"url": "https://ex.com/none"}, context=ctx, conn=conn)["error"] + assert dispatch_tool("get_duplicate_cluster", {}, context=ctx, conn=conn)["error"] + assert dispatch_tool("get_crux_summary", {}, context=ctx, conn=conn)["data"]["available"] is True + assert dispatch_tool("get_lighthouse_for_url", {"url": "https://ex.com/slow"}, context=ctx, conn=conn)["lighthouse"] + assert dispatch_tool("get_lighthouse_for_url", {}, context=ctx, conn=conn)["error"] + assert dispatch_tool("list_slow_pages", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("get_issue_priority_breakdown", {}, context=ctx, conn=conn).get("items") is not None + + thin_count_payload = {"seo_health": {"thin_content": 3}} + with patch.object(Ctx, "load_payload", return_value=thin_count_payload): + note = dispatch_tool("list_thin_content_pages", {}, context=ctx, conn=conn) + assert note["total"] == 3 + assert note.get("note") + + lh_payload = { + "lighthouse_summary": {"human_summary": "text", "pages_audited": 1}, + "lighthouse_by_url": { + "https://ex.com/x": { + "category_scores": {"accessibility": 40}, + "median_metrics": {"best-practices": 45}, + }, + }, + } + with patch.object(Ctx, "load_payload", return_value=lh_payload), patch( + "website_profiling.tools.audit_tools.lighthouse.read_lighthouse_page_summaries", + return_value={"https://ex.com/x": {"performance": 40, "scores": {"performance": 40}}}, + ), patch( + "website_profiling.tools.audit_tools.lighthouse.read_lighthouse_summary", + return_value={"human_summary": "db"}, + ): + summary = dispatch_tool("get_lighthouse_summary", {}, context=ctx, conn=conn) + assert summary["pages_audited"] >= 1 + assert dispatch_tool("list_lighthouse_poor_accessibility_pages", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_lighthouse_poor_best_practices_pages", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_lighthouse_cwv_failures", {}, context=ctx, conn=conn)["total"] >= 0 + + +def test_compare_slices_and_llm(conn: MagicMock, ctx: Ctx) -> None: + payload = _payload() + with patch("website_profiling.tools.audit_tools.compare_helpers.read_report_payload", return_value=payload): + for name in ( + "compare_duplicate_deltas", + "compare_tech_deltas", + "compare_content_metrics", + "compare_google_metrics", + "compare_priority_counts", + ): + result = dispatch_tool(name, {"baseline_report_id": 1}, context=ctx, conn=conn) + assert "error" not in result, name + + with patch("website_profiling.tools.audit_tools.llm_tools.run_page_coach", return_value={"coach": "ok"}): + assert dispatch_tool("get_page_coach", {"url": "https://ex.com"}, context=ctx, conn=conn)["coach"] == "ok" + assert dispatch_tool("get_page_coach", {}, context=ctx, conn=conn)["error"] + assert dispatch_tool("generate_content_brief", {}, context=ctx, conn=conn)["error"] + assert dispatch_tool("expand_keywords", {}, context=ctx, conn=conn)["error"] + assert dispatch_tool("expand_keywords", {"seeds": []}, context=ctx, conn=conn)["error"] + + kw_rows = {"rows": [{"keyword": "widgets sale", "gsc_position": 5}]} + with patch.object(Ctx, "load_keywords", return_value=kw_rows): + brief = dispatch_tool("generate_content_brief", {"keyword": "widgets", "gaps": ["gap"]}, context=ctx, conn=conn) + assert brief["matched_rows"] == 1 + + with patch("website_profiling.tools.audit_tools.llm_tools.batch_expand", return_value={}), patch.object( + Ctx, "load_keywords", return_value=None, + ): + expanded = dispatch_tool( + "expand_keywords", + {"seeds": "a, b", "sources": ["web"]}, + context=ctx, + conn=conn, + ) + assert expanded["seed_count"] == 2 + + conn.execute = MagicMock(return_value=MagicMock(fetchone=MagicMock(return_value={ + "health_score": 70, + "generated_at": datetime.now(timezone.utc), + "report_id": 1, + "issue_counts": json.dumps({"High": 2}), + }))) + with patch("website_profiling.tools.audit_tools.llm_tools.list_properties_public", return_value=[{"id": 1, "name": "Ex"}]): + portfolio = dispatch_tool("get_portfolio_summary", {}, conn=conn) + assert portfolio["count"] == 1 + assert portfolio["properties"][0]["issue_counts"]["High"] == 2 + + +def test_ops_log_paths(conn: MagicMock, ctx: Ctx) -> None: + assert dispatch_tool("get_log_analysis_by_id", {}, context=ctx, conn=conn)["error"] + assert dispatch_tool("get_log_top_paths", {}, context=Ctx(property_id=None), conn=conn)["error"] + assert dispatch_tool("get_latest_log_analysis", {}, context=Ctx(property_id=None), conn=conn)["error"] + assert dispatch_tool("get_google_integration_status", {}, context=Ctx(property_id=None), conn=conn)["error"] + + conn.execute = MagicMock(return_value=MagicMock(fetchone=MagicMock(return_value=None))) + assert dispatch_tool("get_log_analysis_by_id", {"upload_id": 9}, context=ctx, conn=conn)["error"] + assert dispatch_tool("get_log_analysis_by_id", {"upload_id": "x"}, context=ctx, conn=conn)["error"] + + log_row = { + "upload_id": 1, + "filename": "access.log", + "line_count": 10, + "analysis": {"top_paths": [{"path": "/"}], "parsed_lines": 10, "googlebot_hits": 2}, + "uploaded_at": datetime.now(timezone.utc).isoformat(), + } + with patch("website_profiling.tools.audit_tools.ops._load_log_analysis", return_value=log_row): + assert dispatch_tool("get_log_analysis_by_id", {"upload_id": 1}, context=ctx, conn=conn)["upload_id"] == 1 + assert dispatch_tool("get_latest_log_analysis", {}, context=ctx, conn=conn)["filename"] == "access.log" + assert dispatch_tool("get_log_top_paths", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("list_log_only_paths", {}, context=ctx, conn=conn)["total"] == 0 + assert dispatch_tool("list_crawl_only_paths", {}, context=ctx, conn=conn)["total"] == 0 + assert dispatch_tool("get_log_googlebot_stats", {}, context=ctx, conn=conn)["googlebot_hits"] == 2 + + with patch("website_profiling.tools.audit_tools.ops._load_log_analysis", return_value=None): + assert dispatch_tool("get_log_top_paths", {}, context=ctx, conn=conn)["missing"] + + +def test_tech_lighthouse_charts_keywords(conn: MagicMock, ctx: Ctx) -> None: + from website_profiling.tools.audit_tools import tech as tech_mod + from website_profiling.tools.audit_tools import lighthouse as lh_mod + + assert tech_mod.list_pages_by_technology(conn, ctx, {})["error"] + with patch.object(Ctx, "load_payload", return_value={}): + assert tech_mod.list_pages_by_technology(conn, ctx, {"technology_name": "x"})["error"] + + summary_payload = { + "tech_stack_summary": { + "technologies": [ + "bad", + {"name": "Next.js", "sample_urls": ["https://ex.com/n"]}, + ], + }, + } + df = pd.DataFrame([ + {"url": "https://ex.com/p", "status": "200", "tech_stack": '["next.js"]'}, + ]) + with patch.object(Ctx, "load_payload", return_value=summary_payload), patch.object(Ctx, "load_crawl_df", return_value=df): + from_summary = tech_mod.list_pages_by_technology(conn, ctx, {"technology_name": "Next.js"}) + assert from_summary["total"] == 1 + summary_payload["tech_stack_summary"] = {"technologies": []} + from_crawl = tech_mod.list_pages_by_technology(conn, ctx, {"technology_name": "next.js"}) + assert from_crawl["total"] == 1 + + lh_data = { + "lighthouse_summary": {"human_summary": "ok"}, + "lighthouse_diagnostics": [], + "lighthouse_by_url": { + "https://ex.com/a": {"performance": 20, "scores": {"performance": 20}}, + "https://ex.com/b": {"seo": 50, "category_scores": {"seo": 50}}, + }, + "lighthouse_human_summary": "inline", + } + with patch.object(Ctx, "load_payload", return_value=lh_data), patch( + "website_profiling.tools.audit_tools.lighthouse.read_lighthouse_page_summaries", + return_value={"https://ex.com/a": {"performance": 20}}, + ), patch( + "website_profiling.tools.audit_tools.lighthouse.read_lighthouse_summary", + return_value={"human_summary": "db"}, + ): + out = lh_mod.get_lighthouse_summary(conn, ctx, {}) + assert out["poor_performance_pages"] + assert lh_mod.get_lighthouse_for_url(conn, ctx, {"url": ""})["error"] + assert lh_mod.get_lighthouse_for_url(conn, ctx, {"url": "https://ex.com/b"})["lighthouse"] + assert lh_mod.list_slow_pages(conn, ctx, {})["total"] >= 1 + assert lh_mod.list_lighthouse_poor_seo_pages(conn, ctx, {})["total"] >= 1 + + score_payload = { + "lighthouse_by_url": { + "https://ex.com/c": { + "scores": {"accessibility": 30}, + "median_metrics": {"best-practices": 40}, + }, + }, + } + with patch.object(Ctx, "load_payload", return_value=score_payload): + assert lh_mod.list_lighthouse_poor_accessibility_pages(conn, ctx, {})["total"] >= 1 + assert lh_mod.list_lighthouse_poor_best_practices_pages(conn, ctx, {})["total"] >= 1 + assert lh_mod.list_lighthouse_cwv_failures(conn, ctx, {})["total"] >= 0 + + with patch("website_profiling.tools.audit_tools.report.get_report_summary", return_value={ + "issue_counts": {"Critical": 2, "High": 1, "bad": "x"}, + "total_issues": 3, + "health_score": 70, + }): + breakdown = dispatch_tool("get_issue_priority_breakdown", {}, context=ctx, conn=conn) + assert breakdown["items"] + + kw_payload = { + "rows": [ + "bad", + {"keyword": "x", "recommended_action": "fix", "gsc_position": 3, "gsc_impressions": 100}, + ], + "striking_distance": [{"keyword": "y"}], + "cannibalisation": [{"keyword": "z"}], + "query_page_misalignment": [{"keyword": "q"}], + "serp_overlay_count": 1, + } + with patch.object(Ctx, "load_keywords", return_value=kw_payload), patch( + "website_profiling.tools.audit_tools.keywords.read_keyword_history", + return_value=[{"keyword": "x", "position": 4}], + ): + assert dispatch_tool("get_striking_distance_keywords", {}, context=ctx, conn=conn)["keywords"] + assert dispatch_tool("search_keywords", {"query": "x"}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("get_keyword_summary", {"limit": "bad"}, context=ctx, conn=conn)["total_keywords"] >= 1 + assert dispatch_tool("get_keyword_cannibalisation", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("get_query_page_misalignment", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("get_keyword_history", {"keyword": "x"}, context=ctx, conn=conn)["count"] == 1 + with patch.object(Ctx, "load_keywords", return_value=None): + assert dispatch_tool("get_keyword_summary", {}, context=ctx, conn=conn)["error"] + assert dispatch_tool("search_keywords", {"query": "x"}, context=ctx, conn=conn)["error"] + + img_payload = _payload() + img_payload["links"] = [ + { + "url": "https://ex.com/", + "og_image": "https://cdn.ex.com/og.png", + "twitter_image": "https://cdn.ex.com/tw.png", + "page_analysis": {"image_urls": ["https://cdn.ex.com/hero.jpg", "https://cdn.ex.com/hero.jpg"]}, + }, + "bad", + ] + img_payload["image_inventory"] = [ + {"url": "https://cdn.ex.com/big.png", "size_bytes": 500_000, "content_type": "image/png", "source_pages": [], "kinds": []}, + ] + img_payload["image_inventory_summary"] = {"unoptimized_min_kb": 200} + with patch.object(Ctx, "load_payload", return_value=img_payload): + twitter = dispatch_tool("list_site_image_urls", {"kind": "twitter"}, context=ctx, conn=conn) + assert twitter["total"] == 1 + largest = dispatch_tool("list_largest_images", {"min_size_kb": "bad"}, context=ctx, conn=conn) + assert largest["inventory_available"] is True + assert largest["min_size_kb"] == 200 + + bad_tech_df = pd.DataFrame([{"url": "https://ex.com/t", "status": "200", "tech_stack": "not-json"}]) + with patch.object(Ctx, "load_payload", return_value={"tech_stack_summary": {"technologies": []}}), patch.object( + Ctx, "load_crawl_df", return_value=bad_tech_df, + ): + assert tech_mod.list_pages_by_technology(conn, ctx, {"technology_name": "vue"})["total"] == 0 + + +def test_security_google_lighthouse_and_portfolio(conn: MagicMock, ctx: Ctx, tmp_path, monkeypatch) -> None: + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + sec_payload = { + "security_findings": [ + "bad", + {"severity": "High", "finding_type": "hsts", "message": "m"}, + {"severity": "Low", "finding_type": "csp", "message": "n"}, + ], + } + with patch.object(Ctx, "load_payload", return_value=sec_payload): + assert dispatch_tool("get_security_findings", {"severity": "high"}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("get_security_findings_summary", {}, context=ctx, conn=conn)["type_count"] == 2 + assert dispatch_tool("list_security_findings_by_type", {}, context=ctx, conn=conn)["error"] + with patch.object(Ctx, "load_payload", return_value={"security_findings": "bad"}): + assert dispatch_tool("get_security_findings", {}, context=ctx, conn=conn)["total"] == 0 + assert dispatch_tool("get_security_findings_summary", {}, context=ctx, conn=conn)["total_findings"] == 0 + assert dispatch_tool("list_security_findings_by_type", {"finding_type": "x"}, context=ctx, conn=conn)["total"] == 0 + + google_data = { + "gsc": {"top_queries": [{"query": "q"}], "top_pages": [{"page": "/"}]}, + "ga4": {"summary": {"sessions": 1}, "top_pages": [{"path": "/home", "sessions": 5}]}, + "fetched_at": "2026-01-01", + } + assert dispatch_tool("search_keywords", {"query": "x"}, context=Ctx(property_id=None), conn=conn)["error"] + with patch.object(Ctx, "load_google", return_value=None): + assert dispatch_tool("get_gsc_top_pages", {}, context=ctx, conn=conn)["error"] + assert dispatch_tool("get_gsc_page_query_slice", {"url": "https://ex.com"}, context=ctx, conn=conn)["error"] + with patch.object(Ctx, "load_google", return_value=google_data), patch( + "website_profiling.tools.audit_tools.google.slice_from_google_row", + return_value={"gsc": {"clicks": 1}, "ga4": {"sessions": 2}}, + ): + assert dispatch_tool("get_gsc_top_pages", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("get_ga4_summary", {}, context=ctx, conn=conn)["top_pages"] + assert dispatch_tool("get_gsc_page_query_slice", {"url": "https://ex.com/home"}, context=ctx, conn=conn)["gsc_ga4"] + assert dispatch_tool("get_ga4_page_metrics", {"path": "https://ex.com/home"}, context=ctx, conn=conn)["metrics"]["sessions"] == 5 + assert dispatch_tool("get_ga4_page_metrics", {"path": "/missing"}, context=ctx, conn=conn)["metrics"]["sessions"] == 2 + + lh_payload = { + "lighthouse_by_url": { + "bad": "skip", + "https://ex.com/cwv": { + "median_metrics": {"lcp_ms": 5000, "cls": 0.5, "tbt_ms": 500}, + "category_scores": {"accessibility_score": 30}, + "scores": {"best_practices": 40}, + }, + "https://ex.com/score": { + "scores": {"performance": "n/a"}, + "category_scores": {"seo": 70}, + "median_metrics": {"cumulative_layout_shift": 0.1}, + }, + }, + } + with patch.object(Ctx, "load_payload", return_value=lh_payload): + assert dispatch_tool("list_lighthouse_cwv_failures", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_lighthouse_poor_accessibility_pages", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("list_lighthouse_poor_best_practices_pages", {}, context=ctx, conn=conn)["total"] >= 1 + assert dispatch_tool("get_lighthouse_diagnostics", {}, context=ctx, conn=conn)["total"] == 0 + assert dispatch_tool("get_crux_summary", {}, context=ctx, conn=conn)["missing"] + + kw_rows = {"rows": [{"keyword": "a", "gsc_position": 5, "gsc_impressions": 10}]} + with patch.object(Ctx, "load_keywords", return_value=kw_rows): + assert dispatch_tool("list_keywords_by_position", {"min_position": "x"}, context=ctx, conn=conn)["error"] + assert dispatch_tool("list_keywords_by_impressions", {"min_impressions": "x"}, context=ctx, conn=conn)["error"] + assert dispatch_tool("list_keywords_by_position", {"min_position": 1, "max_position": 10}, context=ctx, conn=conn)["total"] == 1 + + with patch("website_profiling.tools.audit_tools.llm_tools.list_properties_public", return_value=["bad", {"id": None}]): + assert dispatch_tool("get_portfolio_summary", {}, conn=conn)["count"] == 0 + + conn.execute = MagicMock(return_value=MagicMock(fetchone=MagicMock(return_value={ + "health_score": 50, + "generated_at": datetime.now(timezone.utc), + "report_id": 3, + "issue_counts": "not-json", + }))) + with patch("website_profiling.tools.audit_tools.llm_tools.list_properties_public", return_value=[{"id": 3, "name": "c"}]): + portfolio = dispatch_tool("get_portfolio_summary", {}, conn=conn) + assert portfolio["count"] == 1 + assert portfolio["properties"][0]["issue_counts"] == {} + + attention_payload = { + "content_urls": { + "missing_alt": [{"url": "https://ex.com/page"}], + "missing_lazy": [{"url": "https://ex.com/page"}], + "missing_dimensions": [{"url": "https://ex.com/page"}], + }, + } + with patch.object(Ctx, "load_payload", return_value=attention_payload): + no_inv = dispatch_tool("list_images_needing_attention", {}, context=ctx, conn=conn) + assert no_inv["inventory_available"] is False + assert no_inv["total"] >= 1 + + attention_payload["image_inventory"] = [{ + "url": "https://cdn.ex.com/huge.jpg", + "size_bytes": 400_000, + "content_type": "image/jpeg", + "source_pages": ["https://ex.com/page"], + "error": "timeout", + }] + attention_payload["image_inventory_summary"] = {"unoptimized_min_kb": 200} + with patch.object(Ctx, "load_payload", return_value=attention_payload): + with_inv = dispatch_tool("list_images_needing_attention", {}, context=ctx, conn=conn) + assert with_inv["inventory_available"] is True + assert with_inv["items"][0].get("reasons") + + lh_summary_payload = { + "lighthouse_summary": "bad", + "lighthouse_by_url": {"skip": "bad"}, + } + with patch.object(Ctx, "load_payload", return_value=lh_summary_payload), patch( + "website_profiling.tools.audit_tools.lighthouse.read_lighthouse_page_summaries", + return_value={"https://ex.com/a": "bad"}, + ), patch( + "website_profiling.tools.audit_tools.lighthouse.read_lighthouse_summary", + return_value=None, + ): + assert dispatch_tool("get_lighthouse_summary", {}, context=ctx, conn=conn)["pages_audited"] == 1 diff --git a/tests/test_export_artifacts_coverage.py b/tests/test_export_artifacts_coverage.py new file mode 100644 index 00000000..db7aad0c --- /dev/null +++ b/tests/test_export_artifacts_coverage.py @@ -0,0 +1,25 @@ +"""Line-coverage tests for export_artifacts edge paths.""" +from __future__ import annotations + +import pytest + +from website_profiling.tools import export_artifacts + + + + +def test_export_artifacts_edge_cases(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + assert export_artifacts.read_artifact_meta("not-a-uuid") is None + assert export_artifacts.read_report_spec("not-a-uuid") is None + env = export_artifacts.save_artifact(b"x", filename="b.bin", mime_type="application/octet-stream", meta={"k": 1}) + meta_path = tmp_path / "exports" / f"{env['artifact_id']}.meta.json" + with open(meta_path, "w", encoding="utf-8") as f: + f.write("{not json") + assert export_artifacts.sweep_expired_artifacts() >= 0 + assert export_artifacts.rows_from_tool_result({"error": "x"}) == [] + assert export_artifacts.rows_from_tool_result({"pages": ["a", {"url": "b"}]})[0]["value"] == "a" + assert export_artifacts.dicts_to_csv([]) == "" + assert export_artifacts.dicts_to_csv([{}]) == "" + csv_filtered = export_artifacts.dicts_to_csv([{"a": 1, "b": 2}], columns=[" ", "a"]) + assert "a" in csv_filtered and "1" in csv_filtered diff --git a/tests/test_export_compare_coverage.py b/tests/test_export_compare_coverage.py new file mode 100644 index 00000000..fdc9ad55 --- /dev/null +++ b/tests/test_export_compare_coverage.py @@ -0,0 +1,13 @@ +"""Line-coverage tests for export_compare helpers.""" +from __future__ import annotations + +from website_profiling.tools.export_compare import export_compare_issues_csv + + + + +def test_export_compare_skips_bad_categories() -> None: + current = {"categories": ["bad", {"name": "T", "issues": ["bad", {"url": "u", "message": "m"}]}]} + baseline = {"categories": []} + csv_text = export_compare_issues_csv(current, baseline) + assert "removed" in csv_text diff --git a/tests/test_export_custom_coverage.py b/tests/test_export_custom_coverage.py new file mode 100644 index 00000000..ed30f6f7 --- /dev/null +++ b/tests/test_export_custom_coverage.py @@ -0,0 +1,101 @@ +"""Line-coverage tests for export_custom helpers.""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx +from website_profiling.tools.export_custom import ( + _section_html_tool_result, + _table_from_rows, + render_custom_report_html, + render_custom_report_pdf, + resolve_section_results, + validate_sections, +) + + +@pytest.fixture +def ctx() -> Ctx: + return Ctx(property_id=1, report_id=1) + + +@pytest.fixture +def conn() -> MagicMock: + return MagicMock() + + +def _payload() -> dict: + return { + "site_name": "Example", + "report_generated_at": "2026-06-07T12:00:00Z", + "categories": [{"id": "tech", "name": "Tech", "score": 80, "issues": []}], + "executive_summary": {"headline": "OK"}, + } + + + + +def test_export_custom_helpers() -> None: + assert "No data" in _table_from_rows([]) + assert "No columns" in _table_from_rows([{}]) + big = _table_from_rows([{"url": f"https://ex.com/{i}", "n": i} for i in range(60)], max_rows=50) + assert "Showing 50 of 60" in big + + err_html = _section_html_tool_result("H", {"error": "boom"}) + assert "boom" in err_html + rows_html = _section_html_tool_result("H", {"pages": [{"url": "u"}]}) + assert "u" in rows_html + items_html = _section_html_tool_result("H", {"items": [{"k": "v"}]}) + assert "v" in items_html + preview_html = _section_html_tool_result("H", {"meta": "x"}) + assert "json-preview" in preview_html + + sections, err = validate_sections([{"type": "notes", "markdown": "hi"}] * 13) + assert err and "max" in err + _, err2 = validate_sections([{"type": "tool"}]) + assert err2 and "tool_name" in err2 + _, err3 = validate_sections([{"type": "notes"}]) + assert err3 and "markdown" in err3 + _, err4 = validate_sections("bad") + assert err4 + + payload = _payload() + html_doc = render_custom_report_html( + title="T", + payload=payload, + sections=[ + {"type": "executive_summary"}, + {"type": "category_scores"}, + {"type": "notes", "heading": "N", "markdown": "line"}, + {"type": "tool", "heading": "Broken", "tool_name": "list_broken_links"}, + ], + section_results=[None, None, None, {"pages": [{"url": "https://ex.com/x"}]}], + ) + assert "Executive summary" in html_doc + assert "Category scores" in html_doc + assert "line" in html_doc + + try: + pdf = render_custom_report_pdf(html_doc, "T") + assert pdf[:4] == b"%PDF" + except RuntimeError as exc: + pytest.skip(str(exc)) + + +def test_export_custom_resolve_sections(conn: MagicMock, ctx: Ctx) -> None: + sections = [ + {"type": "executive_summary"}, + {"type": "tool", "tool_name": "list_broken_links", "tool_args": {}}, + ] + with patch.object(Ctx, "load_payload", return_value=_payload()): + results = resolve_section_results( + conn, + ctx, + _payload(), + sections, + lambda name, args, **kw: {"pages": [{"url": "https://ex.com"}]}, + ) + assert results[0] is None + assert results[1]["pages"] diff --git a/tests/test_export_tools_coverage.py b/tests/test_export_tools_coverage.py new file mode 100644 index 00000000..f16fe31d --- /dev/null +++ b/tests/test_export_tools_coverage.py @@ -0,0 +1,160 @@ +"""Line-coverage tests for audit_tools.export_tools dispatch paths.""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from website_profiling.tools.audit_tools import dispatch_tool +from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx + + +@pytest.fixture +def ctx() -> Ctx: + return Ctx(property_id=1, report_id=1) + + +@pytest.fixture +def conn() -> MagicMock: + return MagicMock() + + +def _payload() -> dict: + return { + "site_name": "Example", + "report_generated_at": "2026-06-07T12:00:00Z", + "categories": [{"id": "tech", "name": "Tech", "score": 80, "issues": []}], + "executive_summary": {"headline": "OK"}, + } + + +def test_export_tools_formats(conn: MagicMock, ctx: Ctx, tmp_path, monkeypatch) -> None: + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + payload = _payload() + with patch.object(Ctx, "load_payload", return_value=payload): + assert dispatch_tool("export_audit_report", {"format": "bad"}, context=ctx, conn=conn)["error"] + assert dispatch_tool("export_list_as_csv", {}, context=ctx, conn=conn)["error"] + assert dispatch_tool("export_list_as_csv", {"tool_name": "nope"}, context=ctx, conn=conn)["error"] + assert dispatch_tool("compose_custom_report", {"sections": []}, context=ctx, conn=conn)["error"] + assert dispatch_tool("export_custom_report", {"format": "bad"}, context=ctx, conn=conn)["error"] + assert dispatch_tool("export_custom_report", {"report_spec_id": "missing"}, context=ctx, conn=conn)["error"] + + with patch.object(Ctx, "load_payload", return_value=payload), patch( + "website_profiling.tools.audit_tools.export_tools.export_audit_html", + return_value="", + ): + out = dispatch_tool("export_audit_report", {"format": "html"}, context=ctx, conn=conn) + assert out.get("artifact_id") + + with patch.object(Ctx, "load_payload", return_value=payload), patch( + "website_profiling.tools.audit_tools.export_tools.export_audit_json", + return_value="{}", + ): + out = dispatch_tool("export_audit_report", {"format": "json"}, context=ctx, conn=conn) + assert out.get("format") == "json" + + with patch.object(Ctx, "load_payload", return_value=payload), patch( + "website_profiling.tools.audit_tools.export_tools.export_audit_pdf", + side_effect=FileNotFoundError, + ): + assert dispatch_tool("export_audit_report", {"format": "pdf"}, context=ctx, conn=conn)["error"] + + with patch.object(Ctx, "load_payload", return_value=payload), patch( + "website_profiling.tools.audit_tools.export_tools._dispatch", + return_value={"meta": "only"}, + ): + assert dispatch_tool( + "export_list_as_csv", + {"tool_name": "list_broken_links"}, + context=ctx, + conn=conn, + )["error"] + + with patch.object(Ctx, "load_payload", return_value=payload), patch( + "website_profiling.tools.audit_tools.export_tools.load_compare_pair", + return_value=(None, None, None, None, {"error": "bad"}), + ): + assert dispatch_tool("export_compare_csv", {"baseline_report_id": 1}, context=ctx, conn=conn)["error"] + + with patch.object(Ctx, "load_payload", return_value=payload): + bad_tool = dispatch_tool( + "compose_custom_report", + { + "title": "T", + "sections": [{"type": "tool", "tool_name": "export_audit_report", "tool_args": {}}], + }, + context=ctx, + conn=conn, + ) + assert "not allowed" in bad_tool["error"] + + spec = dispatch_tool( + "compose_custom_report", + {"title": "T", "sections": [{"type": "executive_summary"}]}, + context=ctx, + conn=conn, + ) + with patch( + "website_profiling.tools.audit_tools.export_tools.resolve_section_results", + return_value=[{"pages": [{"url": "https://ex.com"}]}], + ): + html_out = dispatch_tool( + "export_custom_report", + { + "title": "Direct", + "format": "html", + "sections": [ + {"type": "tool", "tool_name": "list_broken_links", "tool_args": {}}, + ], + }, + context=ctx, + conn=conn, + ) + assert html_out.get("artifact_id") + pdf_out = dispatch_tool( + "export_custom_report", + {"report_spec_id": spec["report_spec_id"], "format": "pdf"}, + context=ctx, + conn=conn, + ) + if pdf_out.get("error"): + pytest.skip(pdf_out["error"]) + assert pdf_out.get("format") == "pdf" + +def test_export_custom_report_pdf_error(conn: MagicMock, ctx: Ctx, tmp_path, monkeypatch) -> None: + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + with patch.object(Ctx, "load_payload", return_value=_payload()), patch( + "website_profiling.tools.audit_tools.export_tools.render_custom_report_pdf", + side_effect=RuntimeError("no pdf"), + ): + assert dispatch_tool( + "export_custom_report", + {"title": "T", "format": "pdf", "sections": [{"type": "executive_summary"}]}, + context=ctx, + conn=conn, + )["error"] == "no pdf" + + +def test_export_audit_report_paths(conn: MagicMock, ctx: Ctx, tmp_path, monkeypatch) -> None: + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + with patch.object(Ctx, "load_payload", return_value=_payload()), patch( + "website_profiling.tools.audit_tools.export_tools.export_audit_pdf", + return_value=b"%PDF", + ): + pdf_out = dispatch_tool("export_audit_report", {"format": "pdf"}, context=ctx, conn=conn) + assert pdf_out.get("format") == "pdf" + + with patch.object(Ctx, "load_payload", return_value=_payload()), patch( + "website_profiling.tools.audit_tools.export_tools.export_audit_csv", + side_effect=RuntimeError("export failed"), + ): + assert "export failed" in dispatch_tool("export_audit_report", {"format": "csv"}, context=ctx, conn=conn)["error"] + + with patch.object(Ctx, "load_payload", return_value=_payload()): + assert dispatch_tool("compose_custom_report", {"sections": [{"type": "notes", "markdown": "x"}]}, context=ctx, conn=conn)["error"] + assert dispatch_tool( + "export_custom_report", + {"sections": [{"type": "executive_summary"}]}, + context=ctx, + conn=conn, + )["error"] diff --git a/tests/test_schedule_runner.py b/tests/test_schedule_runner.py index 9aa9f1ab..ee4b211c 100644 --- a/tests/test_schedule_runner.py +++ b/tests/test_schedule_runner.py @@ -115,8 +115,9 @@ def test_run_gsc_links_staleness_alerts_delegates() -> None: assert len(run_gsc_links_staleness_alerts()) == 1 -def test_name_main_guard(capsys, monkeypatch) -> None: +def test_module_main_guard(capsys, monkeypatch) -> None: import runpy + import sys monkeypatch.setenv("DATABASE_URL", "postgres://u:p@127.0.0.1:5432/test") conn = MagicMock() @@ -124,6 +125,9 @@ def test_name_main_guard(capsys, monkeypatch) -> None: cur.fetchall.return_value = [] conn.execute.return_value = cur + # run_module executes __main__ in a fresh import; drop any prior import from this file. + sys.modules.pop("website_profiling.tools.schedule_runner", None) + with patch("website_profiling.db.storage.db_session") as mock_session: mock_session.return_value.__enter__.return_value = conn with patch( From 1e7b90391f3e29b6fd2afb51ad6463f9076736da Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 7 Jun 2026 21:36:51 +0530 Subject: [PATCH 4/4] Stop tracking local coverage data file --- .coverage | Bin 69632 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .coverage diff --git a/.coverage b/.coverage deleted file mode 100644 index 77631336c2ee31ad9dbe32cca9dde7047cb08966..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69632 zcmeHQ3v?URnI1_a$&yAhe#G%BiIs=#IB~2vX&W9(O$b?_zy>y?Y(j}Vl19=vl19wP zcHEK(QMx?#>|slH;Q;M%PnRyI^tD^IX(C1-X*6{8WM3%dk6s%^Ni!j^4=0-ZJ~E4uy>BFU-P=({lEX-|K9)Jxtgt4Um>R@Uqn$8V%oQ!sbwq{=0cy3VHg+u z*TH{zbHW$4yaUow_5EyLT}Gr|4W{q#7PgO>IDM|H3Pl)_HLglMI=>9h9s?+irwVlotyv>u;mXc$r;pS#mV zZB-@VPbzZLly1@$0{G|&L6rDboqiJ*d*@}))1Ok1=S9i9>;cRMav)t^` zi&4p1CE=V?gvypYgBvZMN=TwMq?R4YA99lAJ1SnXTt`Z0K%F&CN7Kf(v#C?MF3?x{ zQj>ME&ZfpjOSV`euwTOe_Bl%6Bb6#JuOax;v!4>vG2iKV$A=-~U3N#)`Hg4GIL<<# z4}UstskQ1^zRjkyp8%(i|7~-!jE_{Ze7Q8bW>}1aH%W=I3RM<-aIppO;S0keTr38a zp|oCp@vdGxaobbLZ!v+&ewll?s$0Jts`pijrpeH(;$}-buYAw9q!WeRBl|bo4*DsUAcb2mqbELhD+@gZib7g zg73$b6XO(}2;AuFf=a1D6`hx6InmNt?6Jz}_ojdooN7W07pprqN>z#@G3m7AimlPb zVp>Ye3CV{q(KDkLtPxT3g;WVnmfBK1W58H!ey)niTncB(KD4M!cbDxCl^ty2Kp+0Z zQF0CL%4ORv&ZgC?E!m+`4K6yTfCfE*gjm+Kv}>JHvgVzv6tXSgrJ;*dUAQpeRUG2o zA_De^MY9Ks)RnI)b&%ZMUdW|I$TRRqH_8BIfHFWCpbSt3C zd<@tuR*MtI|5o7zMmP>1bfXMV1}FoR0m=YnfHFWCpbSt3C<=?6dsUc|-obX>O1!0k-bd{>W z0uNYY>DSbdf24RYkTQ;nNijaE$(lc|L<1>Ng_L1+Nu-lBPB`d~DoPX<5Q(W&Kug1- zGr$?>0G!RH=##?h+yG$(X@rm}j>P@?r_R;Da#Ik%Mv^AMVnfhjnJEZ4J#O%e|P1Y?o8dT*lAjwMr z=~@$Yi-;kpIHLeHv>2ci6QP2ziVA!|AS{c~q*CTmUSzU{C0Nif*#=PJd!eC}!o?)g z3V1|a>%1xu6gBLS23r7UyNP-Q0&-GL2Lga4HUrirX0Y99P9yK~2!CFAMj3jY@qDm=6O&b`J z02o^cW5nZ$KuCm!)qsic07KMyfhuu18cW9%_&E^+XTS?MMEh%i11qngRf8Wgi9;b1 zIY1!lvH(H%#D)?(JOFVnVVk62NjYRc=!UdOmnrz20PrG4{s&wp7(sWoMz_3%Gy-0+ zAJ9Z-nB-4mUvj+@5c&z*Ab^+M>pt;%hY87Jc8Wp8PCF#Ok#z1W?e+jmnt#KkV?*F- zW&;GG)yzPME5(`|wgLoECvHGgH5J!c0Ofp=rYI&3v3YHm6+SJ(9XL)Gggf|y%w3is zKkB^0xuWKB=WC81T7K-_`i!d4tU`+#Q&Wlo84@VKBf4-gE&I5 zTd#lW@&5{Q5US$;<>nw%#sAA>HoJ|e9p(i;M*P3@1~x01iZnm|_nEDm5&yRjve|3R z)U6W#FBxXDikVPnj{g^%tzjkpU!<^E@jcP782`5scX{)wKq>xj-Nk0No2ge-r7-*z8uKxi>Vw>}0e3q-{`%|D9$e zUy1)60XDmlc;Emn3aE{^&Wu{riIspXN3oZe-;A5Rl=phd4ijNg+ItY$?xYM z<{#wK{2=e+IiB_YqxW;(5$^@wm0q4Z#l6Zs$Nil94)*}}MecSk$$g6J;CRogo}YR4 zc^>fG?AhhH*0aTPiKpMw3NUn|3{VCr1C#;E0A+wOKpCJ6yq_3wt!FK+1vPj74YDj> z2VXDkK5=Mz0zH_UnMOGT+lz^DGD#WK%{qxot={OE?W3J=IL<-p3vZm9ncj=`&Ti;O zh-GJHb2EioZk&B|>fm(O$s{P+kMj4Rr}7B>Jkwv>0kwjVAVN7S-wubDEFF}N=cmwAhOb)!`zw48F-+>j(cGbq z*T%=W#Wf1SI{UX@wT=LM~qwHg-tB?Wv+MCCP3H=e^ITz0{U+5sqy^H=z=N0##?3__|dn?hjYUSP2#uHp9E9oCebUiD3h@{8ZmscwSED} z&c@)WO#hn`{RQ^7BjeUf8*r9t7re2dkUomWp11;`u9N6!-i02$XFodf_z21s&`dkZ z++6r(AfMlxMkg^=Ul%ir&|8I!YZm=(>?AsnVQcCk_4>ZyM})pQ_!eHZ&+$*W8MGX( zpV*tv9Q;_mm`ikjekzChA4X{O1Z48~!#RXDxNG5-h5q5^cIBKN0lqi)4nOxw201)D z-dVRV3!GRvFYL6g-S^1sq1@hfo`bz5YlaU`=D&#wdFNse?Eh8IzR_p$$4`wH4j;V# z(I@whB}a}S`PuPP7xcQ}^v=~Me)*kWZ7K8@jwg@bIx`l1an}=_ z)Z>$T#wYWKr}kJl7Y1t@KC)+O{6yYS1AF{JtJeu~5xZ{&^?P4H&h-x1jdgrwb#(R! zLch-=zvYDj`o^13Pe#xyZ$CFNJv09H2s$(|hWZQWzyu1+&P?!jxM@k(@Lz*bm%lXG z-p9gz*mh#o#1y)BIyW(P{zx!~4!wQ2ur~vCeC&CYnLx*;erY>7d0_JG0?;Tl+2EG7 z4Z~0NqnQE%Q~ZABGnofx3mK;sPSjfe-fFk7E*Edb^Z)DlFEK)1cuROu_+McGzhCh3 zUlJTV;(yJbQXTUaKv z2{GX}{OdxF{|^6v@DIXc!gqyl3HJ(j2@eXl2w&#E{(f@xG%u6^$^d16GC&!i3{VCr z1C#;E0A+wO@V;RH2hWxoJ%+Y)uhaL|_vvEgT3yupb>ZmMMeQ10)b;4XvsxFzDqZm1 zy6~>lMQ4{TxK3TTJ9Ob%p$qtRdkkw|ri2+l@Ga3r`(j-zS)_}_ZMtY{ z)y1L~U9>jqqNPa}D;DaaxltF53v{udK^J}Xx>!}G3xBOHdIepq<8`sttBW<9E_yt= zSnbwDlS>y1YjnXnbk38K4Z9zyO~Ar}4iD#L^@w1C#;E0A+wOKpCJ6PzERilmW^BWq>m9 zK4Ji`|EKZ)`^X?PE0h7s0A+wOKpCJ6PzERilmW^BWq>k388CqXx3!a5%-qGmFaLeK z=HK{(%w3isKkB^0xx)GybISTT?>StM_j`vwiV}@W zflwmcn;Pvg??sm>F)9H<2XTbrYrXYPovRftfe8?O8EEl!wJBB7q_oyc{*~*7DYQ@^ zE+=L@@h8D_Hz`9fDwDyS+W;0pp_S&SwXEiH#MfL(01h9y{l zTBNX9@jcN{N@3O7$h*9`#tVuXF8HnFE7@F}su$X~5P!>W(fL(DG?RSGFVxzo->Ap> zE2)d^M3Z?aRk|$P!Dcs*EpkdSB1ggMjV2_WNGM52dO?iMZY7#)qfa0wpb-tER3(T@ zZUe*E$O6g9v=mjvw5%jGf0ZHXiJKc<0uO4WBsD#1ya&FHVc7eK5(g-%lv32Rfl(V_ zvs*}G#N&xTNQB16fDwp0FHj{eM`P)jGNe^-c;dY&z=_HD&FXSUj6)&w5_egEQ0!1) z6kLuuhOvHyv`tcqg0t@lvDwSsLE7XSgL03n%6df~x{iE5sEjots!4F&<$h$Jc}TmY z(Gf)rmyyv&^|M9LvB9wwK zHX+K%Kma;f>vyu*e&PVo(;3zWp1XKbAneLk$?GkBUHt`dp%g7NeA+V>%jAjxx?1djM5) zM?=gQ*D1}wiK&z^e$+`k90X0~jk=goLrT+1<&hmXagT`Ah>4Z)JeL`6B!Gv!^|4%M zfX!|sUHPg4T03!q=FU~7Mwgp|;Kz}BLQFxUw2XM0QJmB;=JT{Pj7Wz7X(@jGzl-@i zBmApy0smWmH{a}i#Jht#$=%JZ^E~6xJaz7`xi4}ZcI|fYHD9gygmcn)JN!c8e#bTT zSM4|2>)88PzwIZst=6N~Tdf_IA6Z0Z8j^eWTcnF+vR9KF?J&kv;!4&0f25N%4pgKw z_?>|cz}akyKBf8pffb|?ils&W)VpB=%S}Om5vk(*e`<7KnJEZWbt<+L5Vn!D!`bRo z$_Gf2nMf%y5R-;fm{cgvS;yK<*3FpzmzMz2wI=FTng1VJ3{Z-RP-mY1mlv6=VP*b5 z*#=PJd!b=*{y)+Rctl<6yed$d{~v4tob4v+Rn^>!&46_Yd8{gka1%gWNfe?y15-8s zA6{r0N|i3#8v$hlSzSu=|JxUkPrB4+j5a{hTS?~H=o8GH|BuxJ5Ya~BY{7cC4v>fz zD(HTk7U>3S0p&81!~x2@m|YS8V+&!7)93#qJirijUZ6_V401WZ8t?)R(f%6X%!znK z4iJdCEI`mbu@UU<@BqZQgl$q9;Rw1RZPI0GUX(lFGQkLQqTGQR(g^cM7O!^#LO)>} zRO0CC9VR4S8SUF?hvYYs&V9fb79C(o^KZEH()_<@0|cVg%s?p5|A(yrLDY#G5LIpV zbrwK5pQI_O636jB=V@nzG2ue~8GewrdhhYB=Js=+^t|f12~NjNyJqg%#8rg@LY$JUyL zH1F|v*$+r$CnU=J&3`<8u-7zGo%VRVyv8)OqVtx-MtT5>=)js$D&{PN9$al2eWxwG kjjaMKb4vQWrMK620}|0u9_%2D2jBuLA$b%0k1-DOf5zLvMF0Q*