diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..14e8321 --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,29 @@ +{ + "name": "research-figure-studio", + "version": "0.2.0", + "description": "Generate paper-grounded scientific figures as editable PowerPoint compositions.", + "author": { + "name": "ResearchFigureStudio contributors", + "url": "https://github.com/yiweiqin/ResearchFigureStudio" + }, + "homepage": "https://github.com/yiweiqin/ResearchFigureStudio", + "repository": "https://github.com/yiweiqin/ResearchFigureStudio", + "license": "MIT", + "keywords": ["research", "scientific-figures", "powerpoint", "pptx", "codex"], + "skills": "./skills/", + "interface": { + "displayName": "Research Figure Studio", + "shortDescription": "Paper to editable PowerPoint figures", + "longDescription": "Build scientifically grounded framework figures whose exact labels and relations remain editable in PowerPoint.", + "developerName": "ResearchFigureStudio contributors", + "category": "Productivity", + "capabilities": ["Read", "Write"], + "websiteURL": "https://github.com/yiweiqin/ResearchFigureStudio", + "defaultPrompt": [ + "Turn this paper into an editable PPT framework figure.", + "Rebuild this scientific figure as editable PowerPoint.", + "Check whether this figure matches the paper semantics." + ], + "brandColor": "#2F6F8F" + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8c5ccd..9e78731 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,16 +20,16 @@ jobs: - name: Compile package and bundled scripts run: | python -m compileall -q rfs - python -m py_compile codex-skills/research-figure-making/scripts/validate_framework_outputs.py - python -m py_compile codex-skills/research-figure-making/scripts/estimate_asset_fill.py + python -m py_compile skills/research-figure-studio/scripts/validate_framework_outputs.py + python -m py_compile skills/research-figure-studio/scripts/estimate_asset_fill.py - name: Run unit tests run: python -m unittest discover -s tests -q - name: Check bundled skill metadata shell: python run: | from pathlib import Path - skill = Path('codex-skills/research-figure-making/SKILL.md') + skill = Path('skills/research-figure-studio/SKILL.md') text = skill.read_text(encoding='utf-8') assert text.startswith('---'), 'SKILL.md must start with YAML frontmatter' - assert 'name: research-figure-making' in text + assert 'name: research-figure-studio' in text assert 'description:' in text diff --git a/.gitignore b/.gitignore index 3250ca3..0045bb2 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,11 @@ research_figure_studio.egg-info/ venv/ env/ tmp/translation_env/ +tmp/codex-presentations/ +tmp/pdfs/*/ +tmp/pdfs/ +tmp/paper-to-editable-smoke*/ +benchmarks/**/inputs/ *.key *.pem diff --git a/README.md b/README.md index cad8544..6bfacd0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Summary -ResearchFigureStudio is a PPTX-first research figure generation pipeline for paper-grounded, reference-guided scientific framework figures. It turns a paper plus a user-provided visual reference image into an editable PowerPoint composition assembled from many slot-level image assets, not one flattened full-diagram bitmap. +ResearchFigureStudio is a paper-to-editable-PPT scientific figure engine and Codex plugin. Paper contracts provide exact labels and scientific relations; generated or user-provided images provide layout and visual style; deterministic code compiles the result into editable PowerPoint objects. The current workflow is optimized for AI/ML/NLP system figures: @@ -77,14 +77,82 @@ python -m pip install -e ".[ocr]" Windows with PowerPoint installed gives the best PPTX/PDF/PNG export path. The Python package itself can still generate and validate most intermediate artifacts without external image APIs when using `--asset-mode placeholder`. -## Paper To Image Without PPTX +## Paper To Editable PPTX + +The main product route is: + +```powershell +rfs paper-to-editable ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_editable" ` + --positive-reference "C:\path\optional_style_reference.png" ` + --json +``` + +This writes paper-review and image-generation artifacts under `paper_to_image/`, then writes the semantic contract, editable figure program, PPTX, preview, and QA reports under `editable/`. If no image candidate passes the production gates, the workflow stops before creating an official editable deliverable. + +For offline engineering validation only, add `--image-asset-mode placeholder --rebuild-asset-mode placeholder --allow-engineering-preview`. This does not create a production-approved result. + +## Fast Paper Contract And Optional Editable PPTX + +For the fast “paper to evidence-grounded framework prompt” path, without image generation: + +```powershell +rfs fast-framework-prompt ` + --paper "C:\path\paper.pdf" ` + --out "output\fast_result" ` + --deadline 180 ` + --json +``` + +This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. When a VLM already supplies a rich graph, new deterministic entities must occur in an overview caption or connect directly to a VLM-declared entity through non-reference evidence. Credential-free heuristic mode seeds its graph from overview captions and expands only along evidence-supported connected method chains; if no overview seed exists, it selects the largest connected method component. This prevents Related Work examples and unrelated experimental branches from becoming visible framework nodes. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. + +Add `--editable-ppt` to compile the same evidence-backed semantic contract directly into `editable_composition.pptx` without image generation. Every visible module is a native PowerPoint shape with editable text, every relation is a native connector, and `figure_program.json`, `semantic_ppt_report.json`, and `composition_quality_report.json` remain available for auditing. This direct route is intended for fast structural drafts; generated icons or richer visual assets can be added later without changing the scientific graph. + +Add `--verify-ppt-roundtrip` when the editable draft must be checked end to end. It implies `--editable-ppt`, exports the deck through PowerPoint, and writes `ppt_roundtrip_report.json` with editable label/node/connector coverage, flattened-picture area, render status, and canvas-ratio drift. The default fast path leaves this off because launching PowerPoint usually adds roughly 10-20 seconds. + +Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Native extraction preserves font size and bold metadata so unnumbered or split-number headings such as `Model Architecture` and `Encoder and Decoder Stacks` still create correct section boundaries. Text quality uses Unicode letters and numbers, and section detection includes common English, Spanish, French, German, Portuguese, Chinese, Japanese, and Korean headings plus localized Figure/Table captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records section, typographic-heading, caption, column, rotation, OCR render DPI, and spacing-repair metrics so assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently at 84 DPI and filters short vertical margin artifacts. The base installation includes `wordninja` for conservative English spacing recovery, while requiring multiple English anchors so non-English Latin words are not split accidentally. Set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, spacing-repair support, and model readiness. + +Run a repeatable fast suite with provider, cache, recall, and timing aggregation: + +```powershell +rfs benchmark fast-suite --root benchmarks --out output\benchmarks\fast-suite --planner-mode heuristic --json +``` + +Run the generated multi-layout PDF extraction stress suite independently of paper semantics: + +```powershell +rfs benchmark pdf-suite --out output\benchmarks\pdf-extraction --ocr-engine auto --json +``` + +It generates 10 deterministic parser cases covering columns, typography, rotation, repeated margins, CJK and Spanish text, hyphenation, formulas, and structured tables. A real OCR engine expands the suite to 16 cases with mixed, skewed, degraded, formula/table, and Spanish scans. Each case writes its fixture, rendered preview, document model, extraction report, assertions, and timing so reading-order or OCR regressions are reproducible without committing third-party PDFs. Use `paper-to-image` when the required endpoint is a generated raster framework -figure rather than an editable PowerPoint file. The production route performs a +figure, with an optional native editable PowerPoint structure layer. The production route performs a universal evidence-grounded paper review, loads a domain extension, converts positive references into content-free architecture templates, selects a template, renders `layout_blueprint.png`, and uses Image2 edit to create and review three -candidates. It never invokes the PPTX compiler. +candidates. For evidence-grounded contracts with 2-16 visible nodes, a generic +semantic compiler now writes ranked node boxes, distinct input/output ports, +global obstacle-aware orthogonal paths, branch lanes, and outer feedback loops +into `layout_blueprint.json.semantic_plan`. Image2 receives a separate +`visual_substrate_blueprint.png` with no text or arrows. Because Image2 can move +or resize those cards, the compiler detects every generated card, requires an +exact one-to-one match with semantic nodes, and recomposes its header-free visual +content into `candidate_XX_substrate.png` at the exact semantic boxes without +cropping. It then adds exact paper labels and directed connectors, producing the +reviewed `candidate_XX.png` plus auditable geometry and overlay reports. Self-Refine feedback and SAM-style dense +multi-frame contracts retain stricter specialized layouts. It never invokes the +PPTX compiler unless `--editable-overlay-ppt` is requested; that option places +the normalized substrate below native editable node labels and connectors. + +The contract normalizer preserves semantic roles instead of checking labels alone. Explicit provenance clauses such as `CFP and OCT from MEH-MIDAS and public datasets` compile to `data source -> modality -> method`; multi-source phrases collapse into one evidence-backed source group; explicit internal/external evaluation outputs are restored; generic labels such as `foundation model` are replaced by an explicitly named paper model when the evidence provides the alias; and named techniques such as `masked autoencoder` become required innovations. Paper-contract caching is independent of canvas ratio and display-language preferences, so changing 16:9 to 3:2 cannot select a different scientific contract. + +Production Image-2 review rejects a neat but empty text-only flowchart. The critic checks entity-role mismatches, focused provenance topology, mechanism visualization, visual information density, and publication polish. A deterministic blueprint-enrichment gate requires substantive visual content beyond gradients and shadows; configure its default `0.08` changed-pixel threshold with `RFS_MIN_BLUEPRINT_ENRICHMENT_RATIO`. Generation prompts use a compact scientific contract so evidence, geometry, roles, and visual requirements remain explicit without repeating the full planning documents. + +Branch contracts now promote evidence-backed classification, regression, mask, prediction, and similar parallel leaf heads to one consistent output role, remove ancestor shortcuts that bypass the shared branch point, and merge duplicate innovation labels into the existing node instead of drawing a second node. Feedback blueprints anchor nested feedback connectors at the actual inner artifact rather than the outer container. Visual enrichment is measured inside the blueprint's active figure region while retaining the same `0.08` threshold, so large white margins do not reject a genuinely illustrated figure and sparse box-only diagrams still fail. + +Dense multi-frame overview normalization tolerates descriptive paper wording such as `heavyweight image encoder` and `lightweight mask decoder`. Those phrases still map to the canonical Figure 1 nodes `Image Encoder` and `Mask Decoder`, allowing the overview compiler to remove lower-level implementation details instead of misclassifying the whole contract as a generic branch graph. Offline engineering validation: @@ -125,10 +193,32 @@ rfs paper-to-image ` The main outputs are `paper_review.json`, `review_coverage_report.json`, `domain_profile.json`, `template_profiles/`, `selected_template.json`, -`layout_blueprint.png`, `figure_specification.json`, `image_prompt.txt`, -`image2_request_manifest.json`, four critic reports, the `candidates/` +`layout_blueprint.png`, `layout_blueprint.json`, `visual_substrate_blueprint.png`, +`visual_substrate_blueprint.json`, `overlay_spec.json`, `overlays/`, +`figure_specification.json`, `image_prompt.txt`, +`image2_request_manifest.json`, the general and focused-topology critic reports, the `candidates/` directory, and production-only `selected_image.png`. +To repair a reviewed failed candidate without paying for a new initial candidate +batch, reuse it as the starting image: + +```powershell +rfs paper-to-image ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_image_repair" ` + --asset-mode image2 ` + --review-mode vlm ` + --repair-source "output\paper_to_image\candidates\candidate_01.png" ` + --repair-rounds 1 ` + --image-retries 0 ` + --json +``` + +The reused source is rechecked against the current paper contract and Prompt. +Only a failing source triggers one localized Image2 edit. This is useful after +fixing a contract or Prompt bug because it preserves correct regions and avoids +regenerating the initial image from scratch. + Reference-conditioned production generation requires an Image2 edit endpoint. It defaults to `/images/edits` and may be overridden with `RFS_IMAGE_EDIT_URL`. No API key value is written to output artifacts or logs. @@ -137,6 +227,43 @@ a newly rotated key through environment variables before running production. See [docs/paper-to-image.md](docs/paper-to-image.md) for the review schema, template contract, production gates, and failure behavior. +Automatic selection includes dedicated `dense-multiframe`, `multimodal`, +`feedback`, and `branch` templates. Dense task/model/data-engine overviews use +`dense-multiframe`; multiple modality inputs converging into one shared +representation use `multimodal`; +Compact generation → feedback → refinement loops use `feedback`; shared-trunk +systems with parallel prediction heads use `branch`; true search-tree systems +use `arbor`; simple sequential systems use `linear`. Feedback, branch, +multimodal, and dense candidates also run a focused connector judge that +verifies visible arrow endpoints and rejects shortcuts that bypass required +modules. The judge understands explicit containment: an evidence-supported +repeatable shared component inside a labeled operation container may implement +that container's outgoing relation without being treated as an invented edge. + +The general production critic and focused connector critic run concurrently. +Each uses a 90-second default timeout with no automatic full-call retry; override +them with `RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and +`RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT`. Use `--repair-source` to re-review or +locally repair an existing candidate without regenerating correct regions. +Image-2 generation uses a 120-second default request timeout configured by +`RFS_IMAGE2_TIMEOUT`. A network timeout is not blindly retried; retryable HTTP +or provider failures receive at most the configured `--image-retries` attempts, +whose default is one. + +Use `--candidates 3` for a minimum stability audit. Independent candidates are +generated and reviewed concurrently, and `stability_report.json` records the +production pass rate, mean score, worst-case score, standard deviation, and +failure-mode counts. Benchmark scoring consumes this report automatically +instead of treating one selected best image as evidence of stability. + +Every visible scientific label is checked against the normalized contract +whitelist. Paper-supported but out-of-scope details still fail the overview if +they were intentionally omitted from the figure contract. For interrupted +stability runs, rerun the same output directory with `--resume-candidates`; +existing images are re-reviewed and only missing candidates are generated. +Three-seed stability runs may automatically replace one provider-failed seed, +without regenerating successful candidates. + ## Offline Smoke Test Use placeholder assets to validate the local pipeline without calling any API: @@ -161,6 +288,7 @@ rfs make-framework ` --text-extractor-mode ocr ` --ocr-engine paddle ` --ocr-lang en_ch ` + --editable-overlay-ppt ` --json rfs validate --out "output\demo_placeholder" --json @@ -168,6 +296,38 @@ rfs validate --out "output\demo_placeholder" --json `output/` is intentionally ignored by Git. +## Reuse the Fast Paper Contract + +When `fast-framework-prompt` or `paper-to-image` has already produced a validated +`figure_specification.json`, pass that run directory directly to +`make-framework`. This bypasses the legacy paper planner: panel-local node +instances, exact paper labels, evidence IDs, and panel-local relations become +the scientific authority for slots, editable text, and PPT connectors. + +```powershell +rfs make-framework ` + --paper "C:\path\paper.pdf" ` + --reference "output\paper_to_image\run\selected_image.png" ` + --paper-contract-dir "output\paper_to_image\run" ` + --out "output\editable\contract_driven" ` + --slot-count 25 ` + --slot-source reference-primary ` + --asset-mode placeholder ` + --candidates-per-slot 1 ` + --asset-workers 3 ` + --locator-mode heuristic ` + --control-localizer-mode off ` + --prompt-plan-mode heuristic ` + --text-extractor-mode heuristic ` + --ocr-engine off ` + --no-export ` + --json +``` + +For the complete paper-to-image-to-editable route, use +`paper-to-editable --editable-route framework`. The default `rebuild` route is +kept for compatibility with older image-reconstruction runs. + ## Real VLM + Image Generation Set API credentials only through environment variables. Do not write keys into source files. @@ -224,6 +384,7 @@ rfs rebuild-editable-pro ` --reference "C:\path\figure.png" ` --out "output\editable_rebuild_pro" ` --asset-mode api ` + --asset-policy smart-api ` --repair-rounds 2 ` --export-preview ``` @@ -235,14 +396,21 @@ write `professional_gap_report.json` against a specialized-script output. Use `--repair-mode vlm` only when you want the VLM to apply constrained DSL patches after preview comparison. -Lower-cost mode, using VLM structure planning but keeping reference crops as -slot assets: +In `smart-api` policy, reference crops are only used as image-generation context. +They are not inserted as final PPT assets. Text-like slots are filtered into +editable text, duplicate complex icons reuse one generated asset, and API +failures fall back to placeholders instead of reference crops. + +Lower-cost structure check. With `smart-api`, this does not keep reference crops +as final assets; it writes placeholders for visual slots while still producing +the DSL, overlays, and reports: ```powershell rfs rebuild-editable ` --reference "C:\path\figure.png" ` --out "output\editable_rebuild_crop" ` --asset-mode crop ` + --asset-policy smart-api ` --layout-mode hybrid ` --control-mode hybrid ``` @@ -368,7 +536,7 @@ Optional arrow-beautification pass: rfs make-framework ` --paper "C:\path\paper.pdf" ` --reference "C:\path\reference.png" ` - --out D:\ResearchFigureStudio\output\aesthetic_experiment ` + --out .\output\aesthetic_experiment ` --slot-count 40 ` --slot-source reference-primary ` --control-localizer-mode hybrid ` @@ -458,7 +626,7 @@ Run validation: ```powershell rfs validate --out "output\paper_reference_image2" --json -python codex-skills\research-figure-making\scripts\validate_framework_outputs.py "output\paper_reference_image2" +python skills\research-figure-studio\scripts\validate_framework_outputs.py "output\paper_reference_image2" ``` ## Codex Skill @@ -466,15 +634,15 @@ python codex-skills\research-figure-making\scripts\validate_framework_outputs.py This repository includes the Codex skill under: ```text -codex-skills/research-figure-making +skills/research-figure-studio ``` To install it locally into Codex: ```powershell -$dst = Join-Path $env:USERPROFILE ".codex\skills\research-figure-making" +$dst = Join-Path $env:USERPROFILE ".codex\skills\research-figure-studio" if (Test-Path $dst) { Remove-Item -Recurse -Force $dst } -Copy-Item -Recurse "codex-skills\research-figure-making" $dst +Copy-Item -Recurse "skills\research-figure-studio" $dst ``` The skill documents the full research-figure workflow and includes the standalone framework-output validator. @@ -484,7 +652,7 @@ The skill documents the full research-figure workflow and includes the standalon ```powershell python -m compileall -q rfs python -m unittest discover -s tests -q -python -m py_compile codex-skills\research-figure-making\scripts\validate_framework_outputs.py +python -m py_compile skills\research-figure-studio\scripts\validate_framework_outputs.py ``` ## Repository Hygiene diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..9433360 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,70 @@ +# ResearchFigureStudio Benchmarks + +ResearchFigureStudio uses two independent product benchmark suites plus a generated PDF extraction stress suite, so failures can be attributed to paper parsing, reference-image generation, or editable reconstruction. + +## Suites + +### `paper-to-image` + +Measures scientific faithfulness, terminology, relation correctness, information coverage and density, clarity, aesthetics, reference compliance, hallucinations, and stability across repeated generations. + +Scientific errors are hard failures and cannot be offset by aesthetics. + +### `image-to-ppt` + +Measures rendered visual fidelity, object and relation reconstruction, text alignment, editable PowerPoint structure, anti-cheating rules, visual blockers, and eventually edit-mutation behavior. + +A full-slide copy of the reference image is a hard failure even if pixel similarity is perfect. + +### Generated PDF extraction stress suite + +`benchmark pdf-suite` creates deterministic native two-column, unnumbered-bold-section, rotated-page, and mixed native/scanned PDFs at runtime. It validates section boundaries, cross-column reading order, displayed coordinates, caption recovery, local OCR scheduling, English OCR spacing recovery, and elapsed time. `--ocr-engine auto` adds a real installed-OCR probe after the deterministic adapter-backed tier. + +## Commands + +```powershell +rfs benchmark list --root benchmarks --json +rfs benchmark validate --case benchmarks/paper-to-image/cases/001_linear_pipeline --json +rfs benchmark fetch --case benchmarks/paper-to-image/cases/101_vit_linear --json +rfs benchmark fast --case benchmarks/paper-to-image/cases/101_vit_linear --out output/benchmarks/vit_fast --planner-mode heuristic --json +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/fast_suite --planner-mode heuristic --json +rfs benchmark pdf-suite --out output/benchmarks/pdf_extraction --ocr-engine auto --json +rfs benchmark run --case benchmarks/paper-to-image/cases/001_linear_pipeline --out output/benchmarks/p2i_001 --json +rfs benchmark score --case benchmarks/image-to-ppt/cases/001_three_stage_layout --run output/rebuild_case --json +``` + +## Case policy + +Each case contains `case.json` plus suite-specific human-authored ground truth. Synthetic cases may be committed. Real papers and figures should only be committed when redistribution is permitted; otherwise use local paths or a private benchmark data repository. + +Generated runs and reports belong under `output/benchmarks/` and are not source fixtures. + +Real-paper cases commit `source.json` and human-authored Ground Truth, while `benchmark fetch` downloads the PDF into an ignored local `inputs/` directory. This keeps the public repository reproducible without redistributing publisher files. + +The committed unseen-generalization set spans vision, multimodal learning, and NLP: ViT, Mask R-CNN, Self-Refine, ImageBind, SAM, DETR, CLIP, NeRF, Transformer, BERT, and retrieval-augmented generation. The NLP cases specifically exercise short overview captions, method-text recovery, embedding summation, pre-training/fine-tuning separation, and retrieval-conditioned generation. Local image-only scan regressions remain under ignored `tmp/pdfs/` and `output/pdf/`; do not commit publisher PDFs or generated OCR artifacts. + +Any locally available PDF benchmark can also be tested as an image-only degradation without committing a publisher scan: + +```powershell +rfs benchmark fast-suite ` + --root benchmarks ` + --out output/benchmarks/scanned ` + --case-id 106_detr_set_prediction ` + --case-id 110_bert_pretrain_finetune ` + --rasterize-dpi 144 ` + --ocr-engine rapidocr ` + --planner-mode heuristic ` + --deadline 180 ` + --json +``` + +The generated image-only PDFs stay under the benchmark output directory. Score them with the same human-authored entity, relation, and forbidden-content contracts as the native PDFs. + +`benchmark fast-suite` writes `fast_suite_report.json` with per-case results and aggregate entity/relation recall, forbidden content, cache hit rates, provider success/retry counts, failure categories, and stage timings. + +## Evaluation tiers + +- Offline contract tier: placeholder assets, deterministic validation, CI-safe. +- Fast planning tier: entity recall, relation recall, forbidden content, evidence grounding, deadline, and cache behavior without image generation. +- Production quality tier: real VLM/image models, frozen judge, repeated seeds, and human calibration. +- Human audit tier: blinded pairwise ratings for aesthetics, clarity, and information density. diff --git a/benchmarks/image-to-ppt/annotation_template.json b/benchmarks/image-to-ppt/annotation_template.json new file mode 100644 index 0000000..7edcc82 --- /dev/null +++ b/benchmarks/image-to-ppt/annotation_template.json @@ -0,0 +1,33 @@ +{ + "summary": "Independent human annotation template for image-to-PPT reconstruction.", + "case_id": "", + "canvas": {"width_px": 0, "height_px": 0}, + "objects": [ + { + "id": "object_01", + "type": "panel|card|asset|text|legend|formula", + "bbox_percent": {"x": 0.0, "y": 0.0, "w": 0.0, "h": 0.0}, + "text": "", + "parent_id": null, + "z_index": 0, + "must_be_editable": true + } + ], + "relations": [ + { + "id": "relation_01", + "source": "object_01", + "target": "object_02", + "type": "data_flow", + "path_percent": [[0.0, 0.0], [0.0, 0.0]], + "must_be_editable": true + } + ], + "groups": [], + "mutation_tests": [ + {"operation": "replace_text", "target_id": "", "value": "Edited label"}, + {"operation": "move_object", "target_id": "", "delta_percent": {"x": 0.05, "y": 0.0}}, + {"operation": "change_fill", "target_id": "", "value": "#FFAA00"}, + {"operation": "delete_relation", "target_id": ""} + ] +} diff --git a/benchmarks/image-to-ppt/cases/001_three_stage_layout/case.json b/benchmarks/image-to-ppt/cases/001_three_stage_layout/case.json new file mode 100644 index 0000000..0763927 --- /dev/null +++ b/benchmarks/image-to-ppt/cases/001_three_stage_layout/case.json @@ -0,0 +1,26 @@ +{ + "summary": "Synthetic three-stage image-to-editable-PPT reconstruction case.", + "case_id": "001_three_stage_layout", + "suite": "image-to-ppt", + "tier": "offline-contract", + "reference_image": "reference.ppm", + "expected_objects": "expected_objects.json", + "preview": "rebuild_preview.png", + "pptx": "editable_composition.pptx", + "run_config": { + "asset_mode": "placeholder", + "asset_policy": "smart-api", + "text_mode": "off", + "layout_mode": "heuristic", + "control_mode": "heuristic", + "design_plan_mode": "heuristic", + "ocr_engine": "off" + }, + "thresholds": { + "object_coverage": 0.8, + "relation_coverage": 1.0, + "editability_score": 0.66, + "full_slide_image_count": 0, + "blocking_visual_issue_count": 0 + } +} diff --git a/benchmarks/image-to-ppt/cases/001_three_stage_layout/expected_objects.json b/benchmarks/image-to-ppt/cases/001_three_stage_layout/expected_objects.json new file mode 100644 index 0000000..91a070f --- /dev/null +++ b/benchmarks/image-to-ppt/cases/001_three_stage_layout/expected_objects.json @@ -0,0 +1,14 @@ +{ + "summary": "Human-authored object and relation annotations for a three-stage layout.", + "objects": [ + {"id": "input", "type": "card", "bbox_percent": {"x": 0.05, "y": 0.25, "w": 0.22, "h": 0.45}}, + {"id": "method", "type": "card", "bbox_percent": {"x": 0.39, "y": 0.20, "w": 0.22, "h": 0.55}}, + {"id": "output", "type": "card", "bbox_percent": {"x": 0.73, "y": 0.25, "w": 0.22, "h": 0.45}} + ], + "relations": [ + {"source": "input", "target": "method", "type": "data_flow"}, + {"source": "method", "target": "output", "type": "data_flow"} + ], + "required_editable_types": ["card", "text", "connector"], + "forbid_full_slide_reference_image": true +} diff --git a/benchmarks/image-to-ppt/cases/001_three_stage_layout/reference.ppm b/benchmarks/image-to-ppt/cases/001_three_stage_layout/reference.ppm new file mode 100644 index 0000000..de7242c --- /dev/null +++ b/benchmarks/image-to-ppt/cases/001_three_stage_layout/reference.ppm @@ -0,0 +1,6 @@ +P3 +6 3 +255 +250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 +80 150 210 80 150 210 250 250 250 70 180 160 70 180 160 220 150 80 +80 150 210 80 150 210 250 250 250 70 180 160 70 180 160 220 150 80 diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/case.json b/benchmarks/paper-to-image/cases/001_linear_pipeline/case.json new file mode 100644 index 0000000..0e31eb3 --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/case.json @@ -0,0 +1,26 @@ +{ + "summary": "Synthetic three-stage pipeline for scientific and visual benchmark protocol validation.", + "case_id": "001_linear_pipeline", + "suite": "paper-to-image", + "tier": "offline-contract", + "paper": "paper.md", + "expected_semantics": "expected_semantics.json", + "preferences": "preferences.json", + "positive_references": [], + "negative_references": [], + "run_config": { + "planner_mode": "heuristic", + "image_asset_mode": "placeholder", + "image_candidates": 2, + "review_mode": "heuristic", + "aspect_ratio": "16:9", + "ocr_engine": "off" + }, + "thresholds": { + "entity_recall": 1.0, + "relation_recall": 1.0, + "exact_label_rate": 1.0, + "hallucination_count": 0, + "forbidden_content_count": 0 + } +} diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/expected_semantics.json b/benchmarks/paper-to-image/cases/001_linear_pipeline/expected_semantics.json new file mode 100644 index 0000000..05d6dcc --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/expected_semantics.json @@ -0,0 +1,17 @@ +{ + "summary": "Human-authored semantic ground truth for the ExampleFlow figure.", + "entities": [ + {"id": "raw_document", "label": "Raw Document", "role": "input", "required": true}, + {"id": "document_encoding", "label": "Document Encoding", "role": "module", "required": true}, + {"id": "evidence_aggregation", "label": "Evidence Aggregation", "role": "module", "required": true}, + {"id": "final_prediction", "label": "Final Prediction", "role": "output", "required": true} + ], + "relations": [ + {"source": "raw_document", "target": "document_encoding", "type": "data_flow", "required": true}, + {"source": "document_encoding", "target": "evidence_aggregation", "type": "data_flow", "required": true}, + {"source": "evidence_aggregation", "target": "final_prediction", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Agent", "Memory", "Retriever", "Knowledge Base"], + "required_innovation": "Evidence Aggregation", + "expected_reading_order": ["raw_document", "document_encoding", "evidence_aggregation", "final_prediction"] +} diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/paper.md b/benchmarks/paper-to-image/cases/001_linear_pipeline/paper.md new file mode 100644 index 0000000..352fc23 --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/paper.md @@ -0,0 +1,11 @@ +# ExampleFlow: A Three-Stage Evidence Processing Framework + +## Abstract + +ExampleFlow transforms a Raw Document into a Final Prediction through Document Encoding and Evidence Aggregation. + +## Method + +The system takes a Raw Document as input. Document Encoding converts the Raw Document into contextual representations. Evidence Aggregation consumes the contextual representations and produces a Final Prediction. + +The processing order is Raw Document, Document Encoding, Evidence Aggregation, and Final Prediction. diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/preferences.json b/benchmarks/paper-to-image/cases/001_linear_pipeline/preferences.json new file mode 100644 index 0000000..594d0a5 --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/preferences.json @@ -0,0 +1,9 @@ +{ + "summary": "Visual preferences separated from scientific ground truth.", + "aspect_ratio": "16:9", + "language": "English", + "preferred_flow": "left_to_right", + "style_description": "clean academic framework figure", + "preferred_palette": ["blue", "teal", "warm orange accent"], + "avoid": ["commercial poster", "cyberpunk", "generic dashboard"] +} diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/case.json b/benchmarks/paper-to-image/cases/101_vit_linear/case.json new file mode 100644 index 0000000..e12472a --- /dev/null +++ b/benchmarks/paper-to-image/cases/101_vit_linear/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ICLR 2021 Vision Transformer case for a clean linear architecture figure.", + "case_id": "101_vit_linear", + "suite": "paper-to-image", + "tier": "real-paper-offline-contract", + "topology": "linear_pipeline", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json b/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json new file mode 100644 index 0000000..3b65938 --- /dev/null +++ b/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json @@ -0,0 +1,25 @@ +{ + "summary": "Human-authored semantic contract for the Vision Transformer architecture.", + "entities": [ + {"id": "input_image", "label": "Input Image", "aliases": ["2D Image", "Image"], "role": "input", "required": true}, + {"id": "image_patches", "label": "Image Patches", "aliases": ["Patches"], "role": "module", "required": true}, + {"id": "linear_projection", "label": "Linear Projection of Flattened Patches", "aliases": ["Linear Projection"], "role": "module", "required": true}, + {"id": "position_embedding", "label": "Position Embedding", "role": "module", "required": true}, + {"id": "class_token", "label": "Class Token", "aliases": ["Class Embedding", "[class] Embedding"], "role": "module", "required": true}, + {"id": "transformer_encoder", "label": "Transformer Encoder", "role": "module", "required": true}, + {"id": "mlp_head", "label": "MLP Head", "role": "module", "required": true}, + {"id": "class_prediction", "label": "Class Prediction", "aliases": ["Image Classification Predictions", "Classification Predictions", "Class Label"], "role": "output", "required": true} + ], + "relations": [ + {"source": "input_image", "target": "image_patches", "type": "data_flow", "required": true}, + {"source": "image_patches", "target": "linear_projection", "type": "data_flow", "required": true}, + {"source": "linear_projection", "target": "transformer_encoder", "type": "data_flow", "required": true}, + {"source": "position_embedding", "target": "transformer_encoder", "type": "conditioning", "required": true}, + {"source": "class_token", "target": "transformer_encoder", "type": "conditioning", "required": true}, + {"source": "transformer_encoder", "target": "mlp_head", "type": "data_flow", "required": true}, + {"source": "mlp_head", "target": "class_prediction", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["CNN Backbone", "Recurrent Network", "Decoder", "Object Detection Head"], + "expected_reading_order": ["input_image", "image_patches", "linear_projection", "transformer_encoder", "mlp_head", "class_prediction"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 0, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/source.json b/benchmarks/paper-to-image/cases/101_vit_linear/source.json new file mode 100644 index 0000000..0b126fc --- /dev/null +++ b/benchmarks/paper-to-image/cases/101_vit_linear/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale", + "venue": "ICLR", + "year": 2021, + "arxiv_id": "2010.11929", + "official_url": "https://openreview.net/forum?id=YicbFdNTTy", + "paper_urls": ["https://arxiv.org/pdf/2010.11929", "https://export.arxiv.org/pdf/2010.11929"], + "target_figure": "Figure 1", + "verified_caption": "Model overview. The image is split into fixed-size patches, linearly embedded, combined with position embeddings and processed by a Transformer encoder.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json new file mode 100644 index 0000000..67ba6cb --- /dev/null +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json @@ -0,0 +1,14 @@ +{ + "summary": "ICCV 2017 Mask R-CNN case for shared-backbone multi-head branching.", + "case_id": "102_mask_rcnn_branch", + "suite": "paper-to-image", + "tier": "real-paper-offline-contract", + "topology": "branching_architecture", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json new file mode 100644 index 0000000..390a603 --- /dev/null +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json @@ -0,0 +1,24 @@ +{ + "summary": "Human-authored semantic contract for Mask R-CNN.", + "entities": [ + {"id": "input_image", "label": "Input Image", "aliases": ["Image"], "role": "input", "required": true}, + {"id": "backbone", "label": "Backbone", "aliases": ["Backbone Architecture"], "role": "module", "required": true}, + {"id": "region_proposals", "label": "Region Proposals", "aliases": ["Region Proposal Network (RPN)", "RPN", "Proposals"], "role": "module", "required": true}, + {"id": "roi_align", "label": "RoIAlign", "role": "module", "required": true}, + {"id": "classification_head", "label": "Classification", "aliases": ["Class Label", "Classification Head"], "role": "output", "required": true}, + {"id": "box_head", "label": "Bounding-box Regression", "aliases": ["Bounding Box", "Box Branch", "Box Regression"], "role": "output", "required": true}, + {"id": "mask_head", "label": "Mask Branch", "role": "output", "required": true} + ], + "relations": [ + {"source": "input_image", "target": "backbone", "type": "data_flow", "required": true}, + {"source": "backbone", "target": "region_proposals", "type": "data_flow", "required": true}, + {"source": "region_proposals", "target": "roi_align", "type": "data_flow", "required": true}, + {"source": "backbone", "target": "roi_align", "type": "feature_flow", "required": true}, + {"source": "roi_align", "target": "classification_head", "type": "branch", "required": true}, + {"source": "roi_align", "target": "box_head", "type": "branch", "required": true}, + {"source": "roi_align", "target": "mask_head", "type": "branch", "required": true} + ], + "forbidden_labels": ["Semantic Segmentation Only", "Single Output Head", "Transformer Decoder"], + "expected_reading_order": ["input_image", "backbone", "region_proposals", "roi_align"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 3, "convergence_count": 1, "feedback_loop_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/source.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/source.json new file mode 100644 index 0000000..06453b2 --- /dev/null +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/source.json @@ -0,0 +1,8 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "Mask R-CNN", "venue": "ICCV", "year": 2017, "arxiv_id": "1703.06870", + "official_url": "https://openaccess.thecvf.com/content_ICCV_2017/html/He_Mask_R-CNN_ICCV_2017_paper.html", + "paper_urls": ["https://arxiv.org/pdf/1703.06870", "https://export.arxiv.org/pdf/1703.06870"], + "target_figure": "Figure 1", "verified_caption": "The Mask R-CNN framework for instance segmentation.", + "license_note": "Verify redistribution rights; keep downloaded inputs local." +} diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json new file mode 100644 index 0000000..e75cef8 --- /dev/null +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json @@ -0,0 +1,8 @@ +{ + "summary": "NeurIPS 2023 Self-Refine case for an explicit iterative feedback loop.", + "case_id": "103_self_refine_feedback", "suite": "paper-to-image", "tier": "real-paper-offline-contract", "topology": "feedback_loop", + "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json new file mode 100644 index 0000000..104b0bd --- /dev/null +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json @@ -0,0 +1,25 @@ +{ + "summary": "Human-authored semantic contract for Self-Refine.", + "entities": [ + {"id": "task_input", "label": "Input", "aliases": ["Task Prompt", "Task Instruction"], "role": "input", "required": true}, + {"id": "initial_generator", "label": "Generate", "aliases": ["INIT", "Generator"], "role": "module", "required": true}, + {"id": "initial_output", "label": "Initial Output", "role": "module", "required": true}, + {"id": "feedback_module", "label": "Feedback", "aliases": ["FEEDBACK", "Feedback Provider"], "role": "module", "required": true}, + {"id": "feedback_text", "label": "Feedback", "aliases": ["Self-Feedback", "Self-Feedback (NL)"], "role": "control", "required": true}, + {"id": "refinement_module", "label": "Refine", "aliases": ["ITERATE / REFINE", "Refiner"], "role": "module", "required": true}, + {"id": "refined_output", "label": "Refined Output", "role": "output", "required": true} + ], + "relations": [ + {"source": "task_input", "target": "initial_generator", "type": "data_flow", "required": true}, + {"source": "initial_generator", "target": "initial_output", "type": "data_flow", "required": true}, + {"source": "initial_output", "target": "feedback_module", "type": "evaluation", "required": true}, + {"source": "feedback_module", "target": "feedback_text", "type": "feedback", "required": true}, + {"source": "initial_output", "target": "refinement_module", "type": "revision_input", "required": true}, + {"source": "feedback_text", "target": "refinement_module", "type": "feedback", "required": true}, + {"source": "refinement_module", "target": "refined_output", "type": "data_flow", "required": true}, + {"source": "refined_output", "target": "feedback_module", "type": "feedback_loop", "required": true} + ], + "forbidden_labels": ["Gradient Update", "Human Annotator Required", "External Reward Model", "Supervised Fine-tuning"], + "expected_reading_order": ["task_input", "initial_generator", "initial_output", "feedback_module", "refinement_module", "refined_output"], + "topology_requirements": {"primary_flow": "cyclic", "feedback_loop_count": 1, "iteration_must_be_visible": true} +} diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/source.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/source.json new file mode 100644 index 0000000..b0c34da --- /dev/null +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/source.json @@ -0,0 +1,7 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "Self-Refine: Iterative Refinement with Self-Feedback", "venue": "NeurIPS", "year": 2023, "arxiv_id": "2303.17651", + "official_url": "https://arxiv.org/abs/2303.17651", "paper_urls": ["https://arxiv.org/pdf/2303.17651", "https://export.arxiv.org/pdf/2303.17651"], + "target_figure": "Figure 1", "verified_caption": "Iterative generation, feedback and refinement without additional supervised training.", + "license_note": "Use the arXiv license shown on the source page; downloaded inputs remain local." +} diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json new file mode 100644 index 0000000..aa2f8cd --- /dev/null +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json @@ -0,0 +1,8 @@ +{ + "summary": "CVPR 2023 ImageBind case for six-modality shared-embedding alignment.", + "case_id": "104_imagebind_multimodal", "suite": "paper-to-image", "tier": "real-paper-offline-contract", "topology": "multimodal_system", + "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json new file mode 100644 index 0000000..49bbfa9 --- /dev/null +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json @@ -0,0 +1,27 @@ +{ + "summary": "Human-authored semantic contract for ImageBind.", + "entities": [ + {"id": "image", "label": "Image", "role": "input", "required": true}, + {"id": "text", "label": "Text", "role": "input", "required": true}, + {"id": "audio", "label": "Audio", "role": "input", "required": true}, + {"id": "depth", "label": "Depth", "role": "input", "required": true}, + {"id": "thermal", "label": "Thermal", "role": "input", "required": true}, + {"id": "imu", "label": "IMU", "role": "input", "required": true}, + {"id": "modality_encoders", "label": "Modality Encoders", "aliases": ["Transformer-based Encoders for Each Modality", "Encoders for Each Modality"], "role": "module", "required": true}, + {"id": "joint_embedding", "label": "Joint Embedding Space", "aliases": ["Single Shared Joint Embedding Space", "Single Shared Representation Space", "Joint Embedding"], "role": "module", "required": true}, + {"id": "emergent_alignment", "label": "Emergent Cross-modal Alignment", "aliases": ["Emergent Alignment", "Binding Agent", "Binding Property"], "role": "output", "required": true} + ], + "relations": [ + {"source": "image", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "text", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "audio", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "depth", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "thermal", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "imu", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "modality_encoders", "target": "joint_embedding", "type": "alignment", "required": true}, + {"source": "joint_embedding", "target": "emergent_alignment", "type": "enables", "required": true} + ], + "forbidden_labels": ["All Modality Pairs Are Directly Paired", "Single Shared Raw Encoder", "Late Fusion Classifier"], + "expected_reading_order": ["image", "text", "audio", "depth", "thermal", "imu", "modality_encoders", "joint_embedding", "emergent_alignment"], + "topology_requirements": {"primary_flow": "many_to_one_to_many", "input_modality_count": 6, "shared_space_required": true} +} diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/source.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/source.json new file mode 100644 index 0000000..726abe3 --- /dev/null +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/source.json @@ -0,0 +1,8 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "ImageBind: One Embedding Space To Bind Them All", "venue": "CVPR", "year": 2023, "arxiv_id": "2305.05665", + "official_url": "https://openaccess.thecvf.com/content/CVPR2023/html/Girdhar_ImageBind_One_Embedding_Space_To_Bind_Them_All_CVPR_2023_paper.html", + "paper_urls": ["https://arxiv.org/pdf/2305.05665", "https://export.arxiv.org/pdf/2305.05665"], + "target_figure": "Figure 2", "verified_caption": "Different naturally aligned modality pairs are linked through image data into a common embedding space.", + "license_note": "Verify redistribution rights; keep downloaded inputs local." +} diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json new file mode 100644 index 0000000..21ca329 --- /dev/null +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json @@ -0,0 +1,8 @@ +{ + "summary": "ICCV 2023 Segment Anything case for a dense three-part task-model-data-engine overview.", + "case_id": "105_sam_dense_multiframe", "suite": "paper-to-image", "tier": "real-paper-offline-contract", "topology": "dense_multi_panel", + "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json new file mode 100644 index 0000000..86f1af4 --- /dev/null +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json @@ -0,0 +1,32 @@ +{ + "summary": "Human-authored semantic contract for the Segment Anything overview.", + "entities": [ + {"id": "promptable_task", "label": "Promptable Segmentation Task", "role": "panel", "required": true}, + {"id": "image", "label": "Image", "role": "input", "required": true}, + {"id": "prompt", "label": "Prompt", "aliases": ["Sparse Prompts", "Dense Prompts", "Segmentation Prompt"], "role": "input", "required": true}, + {"id": "valid_mask", "label": "Valid Segmentation Mask", "aliases": ["Valid Masks", "Valid Mask"], "role": "output", "required": true}, + {"id": "sam_model", "label": "Segment Anything Model", "aliases": ["Segmentation Model (SAM)", "SAM"], "role": "panel", "required": true}, + {"id": "image_encoder", "label": "Image Encoder", "role": "module", "required": true}, + {"id": "prompt_encoder", "label": "Prompt Encoder", "role": "module", "required": true}, + {"id": "mask_decoder", "label": "Mask Decoder", "role": "module", "required": true}, + {"id": "data_engine", "label": "Data Engine", "role": "panel", "required": true}, + {"id": "assisted_manual", "label": "Assisted-manual", "role": "module", "required": true}, + {"id": "semi_automatic", "label": "Semi-automatic", "role": "module", "required": true}, + {"id": "fully_automatic", "label": "Fully Automatic", "role": "module", "required": true}, + {"id": "sa1b", "label": "SA-1B", "role": "output", "required": true} + ], + "relations": [ + {"source": "image", "target": "image_encoder", "type": "data_flow", "required": true}, + {"source": "prompt", "target": "prompt_encoder", "type": "data_flow", "required": true}, + {"source": "image_encoder", "target": "mask_decoder", "type": "feature_flow", "required": true}, + {"source": "prompt_encoder", "target": "mask_decoder", "type": "conditioning", "required": true}, + {"source": "mask_decoder", "target": "valid_mask", "type": "data_flow", "required": true}, + {"source": "assisted_manual", "target": "semi_automatic", "type": "stage_transition", "required": true}, + {"source": "semi_automatic", "target": "fully_automatic", "type": "stage_transition", "required": true}, + {"source": "fully_automatic", "target": "sa1b", "type": "data_generation", "required": true}, + {"source": "sam_model", "target": "data_engine", "type": "annotation_support", "required": true} + ], + "forbidden_labels": ["Text-to-Image Generation", "Object Detector Only", "Single-stage Manual Annotation", "Language Model", "Class Descriptions", "Contrastive Learning", "Text Embeddings", "Zero-Shot Classifier"], + "expected_panels": ["promptable_task", "sam_model", "data_engine"], + "topology_requirements": {"panel_count": 3, "dense_multi_panel": true, "feedback_between_model_and_data_engine": true} +} diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/source.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/source.json new file mode 100644 index 0000000..1065723 --- /dev/null +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/source.json @@ -0,0 +1,8 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "Segment Anything", "venue": "ICCV", "year": 2023, "arxiv_id": "2304.02643", + "official_url": "https://openaccess.thecvf.com/content/ICCV2023/html/Kirillov_Segment_Anything_ICCV_2023_paper.html", + "paper_urls": ["https://arxiv.org/pdf/2304.02643", "https://export.arxiv.org/pdf/2304.02643"], + "target_figure": "Figure 1", "verified_caption": "Three interconnected components: promptable segmentation task, Segment Anything Model and data engine.", + "license_note": "Verify redistribution rights; keep downloaded inputs local." +} diff --git a/benchmarks/paper-to-image/cases/106_detr_set_prediction/case.json b/benchmarks/paper-to-image/cases/106_detr_set_prediction/case.json new file mode 100644 index 0000000..94860f8 --- /dev/null +++ b/benchmarks/paper-to-image/cases/106_detr_set_prediction/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ECCV 2020 DETR case for learned queries, encoder-decoder flow, branched outputs, and a training-only matching objective.", + "case_id": "106_detr_set_prediction", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/106_detr_set_prediction/expected_semantics.json b/benchmarks/paper-to-image/cases/106_detr_set_prediction/expected_semantics.json new file mode 100644 index 0000000..56d332d --- /dev/null +++ b/benchmarks/paper-to-image/cases/106_detr_set_prediction/expected_semantics.json @@ -0,0 +1,28 @@ +{ + "summary": "Human-authored semantic contract for DETR's end-to-end set prediction architecture.", + "entities": [ + {"id": "input_image", "label": "Input Image", "aliases": ["Image"], "role": "input", "required": true}, + {"id": "cnn_backbone", "label": "CNN Backbone", "aliases": ["Convolutional Backbone", "Backbone"], "role": "module", "required": true}, + {"id": "transformer_encoder", "label": "Transformer Encoder", "aliases": ["Encoder"], "role": "module", "required": true}, + {"id": "object_queries", "label": "Object Queries", "aliases": ["Learned Object Queries"], "role": "input", "required": true}, + {"id": "transformer_decoder", "label": "Transformer Decoder", "aliases": ["Decoder"], "role": "module", "required": true}, + {"id": "prediction_ffn", "label": "Prediction Feed-Forward Network", "aliases": ["Prediction FFN", "Feed Forward Network", "FFN"], "role": "module", "required": true}, + {"id": "class_predictions", "label": "Class Predictions", "aliases": ["Class Labels", "Classes"], "role": "output", "required": true}, + {"id": "box_predictions", "label": "Bounding Box Predictions", "aliases": ["Bounding Boxes", "Box Predictions"], "role": "output", "required": true}, + {"id": "bipartite_matching", "label": "Bipartite Matching", "aliases": ["Hungarian Matching", "Set Prediction Loss"], "role": "training", "required": true} + ], + "relations": [ + {"source": "input_image", "target": "cnn_backbone", "type": "data_flow", "required": true}, + {"source": "cnn_backbone", "target": "transformer_encoder", "type": "data_flow", "required": true}, + {"source": "transformer_encoder", "target": "transformer_decoder", "type": "data_flow", "required": true}, + {"source": "object_queries", "target": "transformer_decoder", "type": "conditioning", "required": true}, + {"source": "transformer_decoder", "target": "prediction_ffn", "type": "data_flow", "required": true}, + {"source": "prediction_ffn", "target": "class_predictions", "type": "branch", "required": true}, + {"source": "prediction_ffn", "target": "box_predictions", "type": "branch", "required": true}, + {"source": "class_predictions", "target": "bipartite_matching", "type": "training_objective", "required": true}, + {"source": "box_predictions", "target": "bipartite_matching", "type": "training_objective", "required": true} + ], + "forbidden_labels": ["Region Proposal Network", "RoIAlign", "Non-Maximum Suppression", "Anchor Boxes", "Mask Branch"], + "expected_reading_order": ["input_image", "cnn_backbone", "transformer_encoder", "object_queries", "transformer_decoder", "prediction_ffn", "class_predictions", "box_predictions"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/106_detr_set_prediction/source.json b/benchmarks/paper-to-image/cases/106_detr_set_prediction/source.json new file mode 100644 index 0000000..e800719 --- /dev/null +++ b/benchmarks/paper-to-image/cases/106_detr_set_prediction/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "End-to-End Object Detection with Transformers", + "venue": "ECCV", + "year": 2020, + "arxiv_id": "2005.12872", + "official_url": "https://www.ecva.net/papers/eccv_2020/papers_ECCV/html/832_ECCV_2020_paper.php", + "paper_urls": ["https://arxiv.org/pdf/2005.12872", "https://export.arxiv.org/pdf/2005.12872"], + "target_figure": "Figure 2", + "verified_caption": "DETR uses a conventional CNN backbone, a transformer encoder-decoder, a fixed set of learned object queries, and parallel class and box predictions; bipartite matching supplies the set prediction training objective.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/107_clip_contrastive/case.json b/benchmarks/paper-to-image/cases/107_clip_contrastive/case.json new file mode 100644 index 0000000..30a56df --- /dev/null +++ b/benchmarks/paper-to-image/cases/107_clip_contrastive/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ICML 2021 CLIP case requiring explicit separation of contrastive training and zero-shot inference.", + "case_id": "107_clip_contrastive", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "multimodal", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/107_clip_contrastive/expected_semantics.json b/benchmarks/paper-to-image/cases/107_clip_contrastive/expected_semantics.json new file mode 100644 index 0000000..d1f43c8 --- /dev/null +++ b/benchmarks/paper-to-image/cases/107_clip_contrastive/expected_semantics.json @@ -0,0 +1,28 @@ +{ + "summary": "Human-authored semantic contract for CLIP contrastive pre-training and zero-shot inference.", + "entities": [ + {"id": "training_images", "label": "Images", "aliases": ["Image Batch", "Training Images"], "role": "input", "required": true}, + {"id": "training_text", "label": "Text", "aliases": ["Captions", "Text Batch", "Training Text"], "role": "input", "required": true}, + {"id": "image_encoder", "label": "Image Encoder", "role": "module", "required": true}, + {"id": "text_encoder", "label": "Text Encoder", "role": "module", "required": true}, + {"id": "image_embeddings", "label": "Image Embeddings", "aliases": ["Image Features"], "role": "module", "required": true}, + {"id": "text_embeddings", "label": "Text Embeddings", "aliases": ["Text Features"], "role": "module", "required": true}, + {"id": "contrastive_objective", "label": "Contrastive Learning", "aliases": ["Contrastive Pre-training", "Contrastive Objective", "Image-Text Contrastive Loss", "Correct Pairings"], "role": "training", "required": true}, + {"id": "class_descriptions", "label": "Class Descriptions", "aliases": ["Class Names", "Text Prompts"], "role": "input", "required": true}, + {"id": "zero_shot_classifier", "label": "Zero-Shot Classifier", "aliases": ["Zero-shot Prediction", "Zero-shot Linear Classifier"], "role": "output", "required": true} + ], + "relations": [ + {"source": "training_images", "target": "image_encoder", "type": "encoding", "required": true}, + {"source": "training_text", "target": "text_encoder", "type": "encoding", "required": true}, + {"source": "image_encoder", "target": "image_embeddings", "type": "data_flow", "required": true}, + {"source": "text_encoder", "target": "text_embeddings", "type": "data_flow", "required": true}, + {"source": "image_embeddings", "target": "contrastive_objective", "type": "alignment", "required": true}, + {"source": "text_embeddings", "target": "contrastive_objective", "type": "alignment", "required": true}, + {"source": "class_descriptions", "target": "text_encoder", "type": "encoding", "required": true}, + {"source": "text_embeddings", "target": "zero_shot_classifier", "type": "classification", "required": true}, + {"source": "image_embeddings", "target": "zero_shot_classifier", "type": "classification", "required": true} + ], + "forbidden_labels": ["Bounding Box Regression", "Mask Decoder", "Autoregressive Decoder", "Image Generator", "Region Proposals"], + "expected_reading_order": ["training_images", "image_encoder", "image_embeddings", "contrastive_objective", "training_text", "text_encoder", "text_embeddings", "zero_shot_classifier"], + "topology_requirements": {"primary_flow": "multimodal_alignment", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": true} +} diff --git a/benchmarks/paper-to-image/cases/107_clip_contrastive/source.json b/benchmarks/paper-to-image/cases/107_clip_contrastive/source.json new file mode 100644 index 0000000..10de429 --- /dev/null +++ b/benchmarks/paper-to-image/cases/107_clip_contrastive/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "Learning Transferable Visual Models From Natural Language Supervision", + "venue": "ICML", + "year": 2021, + "arxiv_id": "2103.00020", + "official_url": "https://proceedings.mlr.press/v139/radford21a.html", + "paper_urls": ["https://arxiv.org/pdf/2103.00020", "https://export.arxiv.org/pdf/2103.00020"], + "target_figure": "Figure 1", + "verified_caption": "CLIP jointly trains an image encoder and a text encoder to predict correct image-text pairings, then uses text embeddings of class descriptions as a zero-shot classifier at inference time.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/case.json b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/case.json new file mode 100644 index 0000000..e344436 --- /dev/null +++ b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ECCV 2020 NeRF case for coordinate conditioning, branched field outputs, and differentiable rendering.", + "case_id": "108_nerf_volume_rendering", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/expected_semantics.json b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/expected_semantics.json new file mode 100644 index 0000000..c7cb0ca --- /dev/null +++ b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/expected_semantics.json @@ -0,0 +1,28 @@ +{ + "summary": "Human-authored semantic contract for NeRF's neural scene representation and differentiable rendering pipeline.", + "entities": [ + {"id": "camera_ray", "label": "Camera Ray", "aliases": ["Camera Rays", "Ray"], "role": "input", "required": true}, + {"id": "sampled_points", "label": "Sampled 3D Points", "aliases": ["5D Coordinates", "Spatial Locations"], "role": "module", "required": true}, + {"id": "viewing_direction", "label": "Viewing Direction", "aliases": ["View Direction"], "role": "input", "required": true}, + {"id": "positional_encoding", "label": "Positional Encoding", "aliases": ["Fourier Features"], "role": "module", "required": true}, + {"id": "nerf_mlp", "label": "Neural Radiance Field", "aliases": ["NeRF", "MLP", "Neural Network"], "role": "module", "required": true}, + {"id": "volume_density", "label": "Volume Density", "aliases": ["Density", "Sigma"], "role": "module", "required": true}, + {"id": "color", "label": "Color", "aliases": ["RGB Color", "Radiance"], "role": "module", "required": true}, + {"id": "volume_rendering", "label": "Volume Rendering", "aliases": ["Differentiable Volume Rendering"], "role": "module", "required": true}, + {"id": "rendered_image", "label": "Rendered Image", "aliases": ["Synthesized Image", "Synthesized View", "Novel View"], "role": "output", "required": true} + ], + "relations": [ + {"source": "camera_ray", "target": "sampled_points", "type": "sampling", "required": true}, + {"source": "sampled_points", "target": "positional_encoding", "type": "encoding", "required": true}, + {"source": "positional_encoding", "target": "nerf_mlp", "type": "data_flow", "required": true}, + {"source": "viewing_direction", "target": "nerf_mlp", "type": "conditioning", "required": true}, + {"source": "nerf_mlp", "target": "volume_density", "type": "prediction", "required": true}, + {"source": "nerf_mlp", "target": "color", "type": "prediction", "required": true}, + {"source": "volume_density", "target": "volume_rendering", "type": "rendering_input", "required": true}, + {"source": "color", "target": "volume_rendering", "type": "rendering_input", "required": true}, + {"source": "volume_rendering", "target": "rendered_image", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Voxel Grid", "Mesh Reconstruction", "Convolutional Decoder", "GAN Discriminator", "Depth Sensor"], + "expected_reading_order": ["camera_ray", "sampled_points", "positional_encoding", "nerf_mlp", "volume_density", "color", "volume_rendering", "rendered_image"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/source.json b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/source.json new file mode 100644 index 0000000..e3cb3ab --- /dev/null +++ b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "NeRF: Representing Scenes as Neural Radiance Fields for View Synthesis", + "venue": "ECCV", + "year": 2020, + "arxiv_id": "2003.08934", + "official_url": "https://www.ecva.net/papers/eccv_2020/papers_ECCV/html/1473_ECCV_2020_paper.php", + "paper_urls": ["https://arxiv.org/pdf/2003.08934", "https://export.arxiv.org/pdf/2003.08934"], + "target_figure": "Figure 2", + "verified_caption": "NeRF samples 5D coordinates along camera rays, maps position and viewing direction through a neural radiance field to color and volume density, and applies differentiable volume rendering to synthesize an image.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/case.json b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/case.json new file mode 100644 index 0000000..22cdb5e --- /dev/null +++ b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/case.json @@ -0,0 +1,15 @@ +{ + "summary": "NeurIPS 2017 Transformer case for dual embedding paths, encoder-decoder conditioning, and output projection.", + "case_id": "109_transformer_encoder_decoder", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/expected_semantics.json b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/expected_semantics.json new file mode 100644 index 0000000..2d079df --- /dev/null +++ b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/expected_semantics.json @@ -0,0 +1,30 @@ +{ + "summary": "Human-authored semantic contract for the Transformer encoder-decoder architecture.", + "entities": [ + {"id": "source_tokens", "label": "Inputs", "aliases": ["Input Sequence", "Source Tokens"], "role": "input", "required": true}, + {"id": "input_embedding", "label": "Input Embedding", "aliases": ["Input Embeddings"], "role": "module", "required": true}, + {"id": "position_encoding", "label": "Positional Encoding", "aliases": ["Position Encoding"], "role": "module", "required": true}, + {"id": "encoder_stack", "label": "Encoder Stack", "aliases": ["Encoder", "N× Encoder"], "role": "module", "required": true}, + {"id": "target_tokens", "label": "Outputs (shifted right)", "aliases": ["Target Tokens", "Target Sequence"], "role": "input", "required": true}, + {"id": "output_embedding", "label": "Output Embedding", "aliases": ["Output Embeddings"], "role": "module", "required": true}, + {"id": "decoder_stack", "label": "Decoder Stack", "aliases": ["Decoder", "N× Decoder"], "role": "module", "required": true}, + {"id": "linear", "label": "Linear", "aliases": ["Linear Layer"], "role": "module", "required": true}, + {"id": "softmax", "label": "Softmax", "role": "module", "required": true}, + {"id": "output_probabilities", "label": "Output Probabilities", "aliases": ["Next-token Probabilities", "Outputs"], "role": "output", "required": true} + ], + "relations": [ + {"source": "source_tokens", "target": "input_embedding", "type": "embedding", "required": true}, + {"source": "input_embedding", "target": "encoder_stack", "type": "data_flow", "required": true}, + {"source": "position_encoding", "target": "encoder_stack", "type": "conditioning", "required": true}, + {"source": "target_tokens", "target": "output_embedding", "type": "embedding", "required": true}, + {"source": "output_embedding", "target": "decoder_stack", "type": "data_flow", "required": true}, + {"source": "position_encoding", "target": "decoder_stack", "type": "conditioning", "required": true}, + {"source": "encoder_stack", "target": "decoder_stack", "type": "cross_attention", "required": true}, + {"source": "decoder_stack", "target": "linear", "type": "data_flow", "required": true}, + {"source": "linear", "target": "softmax", "type": "data_flow", "required": true}, + {"source": "softmax", "target": "output_probabilities", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Convolutional Backbone", "Recurrent Layer", "LSTM", "Region Proposals"], + "expected_reading_order": ["source_tokens", "input_embedding", "encoder_stack", "target_tokens", "output_embedding", "decoder_stack", "linear", "softmax", "output_probabilities"], + "topology_requirements": {"primary_flow": "encoder_decoder", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/source.json b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/source.json new file mode 100644 index 0000000..be0df1d --- /dev/null +++ b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "Attention Is All You Need", + "venue": "NeurIPS", + "year": 2017, + "arxiv_id": "1706.03762", + "official_url": "https://papers.nips.cc/paper/7181-attention-is-all-you-need", + "paper_urls": ["https://arxiv.org/pdf/1706.03762", "https://export.arxiv.org/pdf/1706.03762"], + "target_figure": "Figure 1", + "verified_caption": "The Transformer model architecture contains encoder and decoder stacks, embeddings with positional encoding, attention and feed-forward sublayers, and a linear-softmax output path.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/case.json b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/case.json new file mode 100644 index 0000000..63d8280 --- /dev/null +++ b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/case.json @@ -0,0 +1,15 @@ +{ + "summary": "NAACL 2019 BERT case for embedding summation, pre-training objectives, and downstream fine-tuning.", + "case_id": "110_bert_pretrain_finetune", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "dense_multiframe", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/expected_semantics.json b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/expected_semantics.json new file mode 100644 index 0000000..527cf05 --- /dev/null +++ b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/expected_semantics.json @@ -0,0 +1,29 @@ +{ + "summary": "Human-authored semantic contract for BERT pre-training, input representation, and fine-tuning.", + "entities": [ + {"id": "input_tokens", "label": "Input Tokens", "aliases": ["Token Sequence", "Input Sequence"], "role": "input", "required": true}, + {"id": "token_embeddings", "label": "Token Embeddings", "role": "module", "required": true}, + {"id": "segment_embeddings", "label": "Segment Embeddings", "aliases": ["Segmentation Embeddings"], "role": "module", "required": true}, + {"id": "position_embeddings", "label": "Position Embeddings", "role": "module", "required": true}, + {"id": "input_representation", "label": "Input Representation", "aliases": ["BERT Input Representation"], "role": "module", "required": true}, + {"id": "bert_encoder", "label": "BERT", "aliases": ["BERT Encoder", "Bidirectional Transformer Encoder"], "role": "module", "required": true}, + {"id": "masked_lm", "label": "Masked LM", "aliases": ["Masked Language Model", "Masked Language Modeling"], "role": "training", "required": true}, + {"id": "next_sentence", "label": "Next Sentence Prediction", "aliases": ["NSP"], "role": "training", "required": true}, + {"id": "fine_tuning", "label": "Fine-tuning", "aliases": ["Fine-tune"], "role": "module", "required": true}, + {"id": "downstream_tasks", "label": "Downstream Tasks", "aliases": ["Task-specific Outputs", "Fine-tuning Tasks"], "role": "output", "required": true} + ], + "relations": [ + {"source": "input_tokens", "target": "token_embeddings", "type": "embedding", "required": true}, + {"source": "token_embeddings", "target": "input_representation", "type": "sum", "required": true}, + {"source": "segment_embeddings", "target": "input_representation", "type": "sum", "required": true}, + {"source": "position_embeddings", "target": "input_representation", "type": "sum", "required": true}, + {"source": "input_representation", "target": "bert_encoder", "type": "data_flow", "required": true}, + {"source": "bert_encoder", "target": "masked_lm", "type": "training_objective", "required": true}, + {"source": "bert_encoder", "target": "next_sentence", "type": "training_objective", "required": true}, + {"source": "bert_encoder", "target": "fine_tuning", "type": "initialization", "required": true}, + {"source": "fine_tuning", "target": "downstream_tasks", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Autoregressive Decoder", "Convolutional Backbone", "Retrieval Index", "Image Encoder"], + "expected_reading_order": ["input_tokens", "token_embeddings", "segment_embeddings", "position_embeddings", "input_representation", "bert_encoder", "fine_tuning", "downstream_tasks"], + "topology_requirements": {"primary_flow": "pretrain_then_finetune", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": true} +} diff --git a/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/source.json b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/source.json new file mode 100644 index 0000000..f200c15 --- /dev/null +++ b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding", + "venue": "NAACL", + "year": 2019, + "arxiv_id": "1810.04805", + "official_url": "https://aclanthology.org/N19-1423/", + "paper_urls": ["https://arxiv.org/pdf/1810.04805", "https://export.arxiv.org/pdf/1810.04805"], + "target_figure": "Figures 1 and 2", + "verified_caption": "BERT uses the same bidirectional Transformer encoder for pre-training and fine-tuning; its input representation sums token, segment, and position embeddings.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/case.json b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/case.json new file mode 100644 index 0000000..89c7667 --- /dev/null +++ b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/case.json @@ -0,0 +1,15 @@ +{ + "summary": "NeurIPS 2020 RAG case for parametric/non-parametric composition and retrieval-conditioned generation.", + "case_id": "111_rag_retrieval_generation", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json new file mode 100644 index 0000000..c54db48 --- /dev/null +++ b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json @@ -0,0 +1,24 @@ +{ + "summary": "Human-authored semantic contract for retrieval-augmented generation.", + "entities": [ + {"id": "input_query", "label": "Input Query", "aliases": ["Query", "Input x"], "role": "input", "required": true}, + {"id": "query_encoder", "label": "Query Encoder", "role": "module", "required": true}, + {"id": "document_index", "label": "Document Index", "aliases": ["Non-parametric Memory", "Wikipedia Index"], "role": "module", "required": true}, + {"id": "retriever", "label": "Retriever", "aliases": ["Neural Retriever"], "role": "module", "required": true}, + {"id": "top_k_documents", "label": "Top-K Documents", "aliases": ["Retrieved Documents", "Retrieved Passages"], "role": "module", "required": true}, + {"id": "generator", "label": "Generator", "aliases": ["Seq2seq Generator", "Pre-trained Generator"], "role": "module", "required": true}, + {"id": "output_sequence", "label": "Output Sequence", "aliases": ["Generated Output", "Output y"], "role": "output", "required": true} + ], + "relations": [ + {"source": "input_query", "target": "query_encoder", "type": "encoding", "required": true}, + {"source": "query_encoder", "target": "retriever", "type": "retrieval_query", "required": true}, + {"source": "document_index", "target": "retriever", "type": "memory_lookup", "required": true}, + {"source": "retriever", "target": "top_k_documents", "type": "retrieval", "required": true}, + {"source": "input_query", "target": "generator", "type": "generation_input", "required": true}, + {"source": "top_k_documents", "target": "generator", "type": "conditioning", "required": true}, + {"source": "generator", "target": "output_sequence", "type": "generation", "required": true} + ], + "forbidden_labels": ["Image Encoder", "Object Queries", "Mask Decoder", "Convolutional Backbone", "Class Prediction", "Class Predictions", "Classification Head"], + "expected_reading_order": ["input_query", "query_encoder", "document_index", "retriever", "top_k_documents", "generator", "output_sequence"], + "topology_requirements": {"primary_flow": "retrieval_then_generation", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/source.json b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/source.json new file mode 100644 index 0000000..e3435f1 --- /dev/null +++ b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks", + "venue": "NeurIPS", + "year": 2020, + "arxiv_id": "2005.11401", + "official_url": "https://papers.nips.cc/paper/2020/hash/6b493230205f780e1bc26945df7481e5-Abstract.html", + "paper_urls": ["https://arxiv.org/pdf/2005.11401", "https://export.arxiv.org/pdf/2005.11401"], + "target_figure": "Figure 1", + "verified_caption": "RAG combines a query encoder and non-parametric document index with a pre-trained seq2seq generator; retrieved top-K documents condition output generation.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/human_rating_template.json b/benchmarks/paper-to-image/human_rating_template.json new file mode 100644 index 0000000..301efbc --- /dev/null +++ b/benchmarks/paper-to-image/human_rating_template.json @@ -0,0 +1,30 @@ +{ + "summary": "Blinded human rating record for one generated scientific figure.", + "case_id": "", + "candidate_id": "", + "reviewer_id": "anonymous_reviewer_01", + "reviewer_background": "paper_author|domain_expert|visual_design_expert|general_researcher", + "scores_1_to_5": { + "scientific_faithfulness": 0, + "information_completeness": 0, + "information_density_without_clutter": 0, + "reading_order_clarity": 0, + "arrow_and_grouping_clarity": 0, + "text_readability_at_target_size": 0, + "aesthetic_quality": 0, + "academic_professionalism": 0 + }, + "quick_comprehension": { + "input_identified": false, + "output_identified": false, + "core_innovation_identified": false, + "main_flow_direction_identified": false + }, + "hard_errors": [], + "pairwise_preference": { + "other_candidate_id": "", + "preferred": "this|other|tie", + "reason": "" + }, + "notes": "" +} diff --git a/benchmarks/paper-to-image/preferences/top_conference.json b/benchmarks/paper-to-image/preferences/top_conference.json new file mode 100644 index 0000000..bbdf8c7 --- /dev/null +++ b/benchmarks/paper-to-image/preferences/top_conference.json @@ -0,0 +1,9 @@ +{ + "summary": "Shared visual preferences for computer-science top-conference benchmark cases.", + "aspect_ratio": "16:9", + "language": "English", + "style_description": "clean top-conference method figure, compact academic layout, restrained color, clear arrows and grouping", + "preferred_palette": ["cool blue", "teal", "muted orange accent", "light neutral background"], + "visual_density": "high but readable", + "avoid": ["commercial poster", "photorealistic decoration", "cyberpunk", "generic robot imagery", "fake equations", "fake charts", "long paragraphs"] +} diff --git a/benchmarks/schema/case.schema.json b/benchmarks/schema/case.schema.json new file mode 100644 index 0000000..076f0ae --- /dev/null +++ b/benchmarks/schema/case.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ResearchFigureStudio benchmark case", + "type": "object", + "required": ["summary", "case_id", "suite", "thresholds", "run_config"], + "properties": { + "summary": {"type": "string", "minLength": 1}, + "case_id": {"type": "string", "minLength": 1}, + "suite": {"enum": ["paper-to-image", "image-to-ppt"]}, + "thresholds": {"type": "object"}, + "run_config": {"type": "object"} + }, + "additionalProperties": true +} diff --git a/benchmarks/schema/expected_objects.schema.json b/benchmarks/schema/expected_objects.schema.json new file mode 100644 index 0000000..f0119ea --- /dev/null +++ b/benchmarks/schema/expected_objects.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Image-to-PPT object ground truth", + "type": "object", + "required": ["summary", "objects", "relations"], + "properties": { + "summary": {"type": "string"}, + "objects": {"type": "array", "items": {"type": "object", "required": ["id", "type", "bbox_percent"]}}, + "relations": {"type": "array", "items": {"type": "object", "required": ["source", "target", "type"]}} + } +} diff --git a/benchmarks/schema/expected_semantics.schema.json b/benchmarks/schema/expected_semantics.schema.json new file mode 100644 index 0000000..9d98d43 --- /dev/null +++ b/benchmarks/schema/expected_semantics.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Paper-to-image semantic ground truth", + "type": "object", + "required": ["summary", "entities", "relations", "forbidden_labels"], + "properties": { + "summary": {"type": "string"}, + "entities": {"type": "array", "items": {"type": "object", "required": ["id", "label", "role"]}}, + "relations": {"type": "array", "items": {"type": "object", "required": ["source", "target", "type"]}}, + "forbidden_labels": {"type": "array", "items": {"type": "string"}} + } +} diff --git a/benchmarks/schema/paper_image_review.schema.json b/benchmarks/schema/paper_image_review.schema.json new file mode 100644 index 0000000..e42afd9 --- /dev/null +++ b/benchmarks/schema/paper_image_review.schema.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Frozen judge or human-calibrated paper-image review", + "type": "object", + "required": ["scientific", "paper_alignment", "topology", "ocr", "information", "clarity", "aesthetic", "stability"], + "properties": { + "scientific": { + "type": "object", + "required": ["score", "missing_modules", "missing_relations", "reversed_relations", "invented_items"] + }, + "paper_alignment": { + "type": "object", + "required": ["score", "passed", "priorities", "missing_priorities", "evidence_conflicts", "unsupported_visual_claims"] + }, + "topology": { + "type": "object", + "required": ["score", "passed", "missing_relations", "reversed_relations", "invented_relations"] + }, + "ocr": { + "type": "object", + "required": ["score", "missing_labels", "misspelled_labels", "forbidden_labels_found"] + }, + "information": { + "type": "object", + "required": ["coverage", "density", "redundancy", "unsupported_information", "score"] + }, + "clarity": { + "type": "object", + "required": ["reading_order", "arrow_ambiguity", "grouping", "target_size_readability", "cognitive_load", "score"] + }, + "aesthetic": { + "type": "object", + "required": ["hierarchy", "balance", "whitespace", "color", "icon_consistency", "professionalism", "score"] + }, + "stability": { + "type": "object", + "required": ["seed_count", "mean_score", "worst_case_score", "standard_deviation", "production_pass_rate"] + } + } +} diff --git a/codex-skills/research-figure-making/SKILL.md b/codex-skills/research-figure-making/SKILL.md deleted file mode 100644 index fe1efa6..0000000 --- a/codex-skills/research-figure-making/SKILL.md +++ /dev/null @@ -1,456 +0,0 @@ ---- -name: research-figure-making -description: >- - Use this skill when the user provides a research paper, manuscript, method - section, experiment section, LaTeX/Word/PDF/Markdown draft, result table, CSV, - figure sketch, or old paper figure and asks to create, redesign, or improve - publication-quality scientific figures, including paper-driven framework - figures, system overview diagrams, model architecture figures, pipeline - diagrams, data plots, main-result figures, multi-panel figures, or PPT/Lark - editable figures. Prioritize AI/ML/NLP research papers, paper-grounded visual - extraction, AI-generated visual blocks for framework figures, reproducible - Python plots for data figures, and PPTX-first editable outputs with optional - Lark/SVG exports. Also use for - Chinese-language requests that mean drawing figures from a paper, making paper - framework diagrams, system diagrams, data figures, main paper figures, PPT - figures, Lark whiteboard figures, method figures, result figures, or - top-tier-publication figures. ---- - -# Research Figure Making - -## Summary - -Create publication-oriented research figures from the user's actual paper content. -Do not use generic architecture templates until the paper has been read and the -paper-specific concepts, variables, modules, and claims have been extracted. -For image-rich framework figures, default to many slot-level image2 blocks, -PPTX-first editable composition, front Summary sections, no semantic cropping, -and hard asset-fill validation. - -## Trigger - -Use this skill when the user asks for figures based on: - -- a paper draft, method section, experiment section, abstract, related work, or notes -- LaTeX, Word, PDF, Markdown, plain text, tables, CSV/Excel, logs, or existing figures -- requests such as "draw Figure 1 from my paper", "make a system overview", - "create a framework diagram", "turn this paper into PPT figures", "improve my - data plot", or "redesign this figure for a top-tier paper" -- Chinese-language requests that mean drawing framework figures from a paper, - making system diagrams from method sections, turning experimental results into - data figures, creating main paper figures, PPT figures, or top-tier paper figures - -## Mandatory Framework Protocol - -For system architecture, framework, pipeline, model overview, or Figure 1 style -requests where the user wants an image-rich figure, this protocol is mandatory: - -## Front Summary Rule - -Every text artifact must start with a `Summary` section before details. This -applies to the figure brief, slot inventory, style sheet, figure program, -prompts, asset quality report, asset visual review, layout plan, visual critic, -alignment review, critic report, and final response. For Markdown, the first -non-empty heading must be `# Summary` or `## Summary`. For JSON, use a top-level -non-empty `summary` field. - -## Local RFS Route - -When `D:\ResearchFigureStudio` and the `rfs` CLI are available, use them as the -default implementation for image-rich framework figures: - -```powershell -rfs make-framework --paper --reference --out --slot-count 40 --slot-source reference-primary --complexity-profile reference-dense --candidates-per-slot 4 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 8 --asset-mode image2 --asset-workers 6 --asset-retries 3 --asset-review-mode heuristic --critic-mode heuristic -``` - -Use `--asset-mode image2` for real image generation through the Yunwu -OpenAI-compatible Images API. The logical `image-2` target maps to Yunwu's listed -`gpt-image-2` model unless `RFS_IMAGE_MODEL` or `IMAGE_MODEL` overrides it. Keep -`--asset-mode gemini` as a fallback and use `--asset-mode placeholder` only for -engineering validation. Use `--locator-mode vlm` when a -VLM should borrow the LiveFigure-style positioning idea by returning -`layout_plan.json` coordinates. The VLM must not write arbitrary PPT code or -generate a single full diagram. For API-cost control, run a small real pass first -with `--slot-count 25 --candidates-per-slot 1 --asset-workers 3`, then a full -pass after the contract validates. Yunwu image2/Gemini slot generation may run -concurrently through RFS workers; do not fall back to manual one-by-one image2 -generation unless the API path is unavailable. -Prompt planning is not a cost-saving heuristic stage by default: use -`--prompt-plan-mode vlm` so the model inspects the reference image and each -local slot before producing `slot_prompt_plan.json`. Use heuristic prompt -planning only for offline engineering validation. Use `--prompt-plan-workers` -to parallelize per-slot VLM prompt planning; use `--asset-workers` separately to -parallelize Yunwu image2/Gemini image generation. -Use `--control-localizer-mode hybrid` by default for arrow and connector -localization. This AutoFigure-inspired stage writes -`reference_control_candidates.json`, `slot_overlay.png`, and -`reference_control_overlay.png`; VLM binding may assign source-target semantics -from those overlays, while the fallback heuristic keeps the workflow offline. -The VLM may patch arrow IDs, anchors, and normalized paths only; it must not -write PPT code, rasterize arrows, or redraw the full figure. -Use `--arrow-style-mode reference` by default. This stage may soften PPT -connector line caps, vary stroke widths, assign bundle/lane metadata, and score -crossing/bend/overlap quality, but the reference image remains the hard -constraint. It must not replace reference-derived flow logic with a generic -graph router. Orthogonal obstacle-aware routing is allowed only for missing -paths or arrows explicitly marked `route_policy=fallback_reroute_allowed`; it -must never rewrite reference-locked `path_percent` values. -Use `--arrow-style-mode aesthetic` only for an explicit experimental -beautification pass. In that mode, the router may apply curve connectors and -halo underlays by default. Bundle lane offsets require explicit route opt-in -and must stay inside `reference_tunnel_percent`; the router must record -`reference_original_path_percent`, `reference_path_delta_max`, and -`reference_tunnel_preserved`, and it must not change source-target logic. -Use `rfs presentations-qa --out ` only as an optional QA pass when -the Presentations plugin is available. Presentations may import/render the PPTX, -extract layout JSON, and report connector/font/rendering drift, but it must not -mutate the PPTX, rewrite arrows, regenerate layout, or replace RFS as the -authoritative compiler for reference-locked connector-heavy figures. - -Before execution, read these references in this order: - -1. `references/paper_to_figure.md` -2. `references/visual_framework_figures.md` -3. `references/reference_image_alignment.md` when a reference image is provided -4. `references/stylist_stage.md` before writing image2 prompts -5. `references/figure_program.md` before composition or scripting -6. `references/image_block_prompting.md` before writing image2 prompts -7. `references/critic_stage.md` before final delivery -8. `references/journal_quality_checklist.md` before final delivery - -Execution checklist: - -1. Read the paper and write a paper-grounded figure brief. -2. If the user provides a reference image, parameterize it before generation: - identify slots, canvas ratios, safe areas, and fit policies. - When the user says the reference image should guide the figure, treat its - slot positions and flow logic as layout source-of-truth. The paper calibrates - terminology and scientific labels; it must not cause a generic paper-derived - grid to replace the reference image's spatial logic. - In reference-primary mode, the reference image is the highest authority for - layout, visual object choice, style, framework colors, arrow logic, and - visual rhythm. The paper only assists terminology and scientific mapping. - Write `reference_geometry.json` with precise panel/slot geometry before any - prompt generation. Every panel and slot must include `bbox_percent`, - `center_percent`, `width_percent`, `height_percent`, - `aspect_ratio_decimal`, `aspect_ratio_w_h`, `target_pixels`, - `target_pixels_exact`, and `generation_min_pixels` with at least three - decimals where numeric precision matters. `target_pixels` must equal - `target_pixels_exact`; use `generation_min_pixels` only for the image - generator's minimum resolution safeguard. - Also write `reference_control_candidates.json`, - `slot_overlay.png`, `reference_control_overlay.png`, and - `reference_controls.json` for measured arrows, connector lines, dashed - loops, transition arrows, framework shapes, and text regions. Control - candidates are analogous to a boxlib/overlay layer: they identify possible - arrows and connectors from the reference image before semantic binding. - Bound controls must record geometry, `source_id`, `target_id`, - `source_anchor`, `target_anchor`, multi-point `path_percent`, color token, - and `render_policy: ppt_shape_not_image_asset`. - Then write `arrow_style_profile.json`, `selected_arrow_routes.json`, and - `arrow_quality_report.json`. These files must state - `reference_image_hard_constraint`, preserve locked reference paths, and only - synthesize missing or explicitly fallback-allowed paths. If fallback routing - is used, record `routing_algorithm`, `route_generation_status`, candidate - count, and obstacle/crossing metrics. - If `--arrow-style-mode aesthetic` is used, also record original reference - paths, tunnel width, path delta, halo settings, connector type, and whether - each adjusted route stayed inside the reference tunnel. -3. Create a slot inventory before using image2. Default to 25-50 image slots for - a normal paper system figure. This count means non-arrow image slots only; - arrows, connector lines, dashed loops, panel frames, and titles must not be - counted as image2 assets. -4. Write `reference_style_profile.json` and `style_sheet.md` before writing - image2 prompts. They must define reference-derived color tokens, palette, - line weight, shadows, viewpoint, icon complexity, background, visual density, - font-layer rules, and image2 text policy. -5. Write `layout_plan.json` and `figure_program.json` before composition. - `layout_plan.json` records normalized reference-guided coordinates; it may be - produced by heuristic rules or VLM positioning, but never by freeform PPT - code. `figure_program.json` must record `canvas`, - `panels`, `slots`, `assets`, `labels`, `arrows`, `groups`, and - `export_targets`; slots must bind paper concepts, reference bboxes, target - ratios, safe areas, fit policies, and asset IDs. `export_targets` must - include the main editable PPTX target, usually `editable_composition.pptx`. - In reference-primary mode, `bbox_percent` values extracted from the reference - image must be preserved into `layout_plan.json`, `figure_program.json`, and - PPT placement. A VLM locator may refine arrows or identify missing slots, but - it must not freely redraw the macro framework or overwrite reference-derived - slot positions. Do not convert exact reference ratios into coarse presets - like `1:1`, `4:3`, `3:4`, `16:9`, or `9:16`; use precise decimal ratios such - as `0.538:1.000`. - Text size, position, color, and hierarchy are reference-primary too. Do not - create a default `publication_scale.json`, do not enforce a fixed - `paper_double_column` target width, and do not fail a figure only because a - reference-matched label would be small after manuscript scaling. Instead, - write `reference_text_geometry.json`, `text_program.json`, and - `text_alignment_report.json`. These files must preserve the reference - image's text bbox, center, relative font height, color, role, and hierarchy. - The paper may adapt terminology, but the PPT text layer must stay editable - and must not override the reference layout with a generic typography rule. -6. Write `slot_visual_spec.json`, `reference_slot_prompt_brief.json`, and - `slot_prompt_plan.json` before image generation. `slot_visual_spec.json` - records each slot's reference crop objects, foreground subject, secondary - objects, micro details, background fill elements, scientific mechanism - detail, required visual complexity, and forbidden simplifications. Except - explicit legend/badge slots, normal image slots must be planned as dense mini - scientific scenes/cards with 2-5 layered objects, not simple standalone - icons. The brief records what each reference slot does and what paper concept - it carries. `slot_prompt_plan.json` must be generated by a VLM by default and - include per-slot `slot_function`, local reference style, concrete - `visual_metaphor`, `must_show`, `avoid_showing`, visual-complexity fields, - and `image_prompt_core`. Every image slot must have - `reference_slot_crops/.png`, and the prompt planner must use the - full reference image, the local slot crop, the global style profile, local - color token ids, precise geometry, and the paper concept. Arrows and - connector controls never receive image prompts. For normal 25-50 slot - figures, run VLM prompt planning in parallel unless rate limits force a lower - worker count. -7. Generate image assets per slot, preferably through RFS Gemini/Yunwu - concurrent workers when configured. Manual image2 generation is only a - fallback. Do not generate one full architecture image as the final figure. -8. Create `asset_quality_report.json`, `asset_complexity_report.json`, - `composition_quality_report.json`, `asset_visual_review.json`, selected asset - contact sheet, and candidate contact sheet. The quality report must record - `content_fill_percent`, `empty_margin_percent`, `edge_cutoff_status`, - `ratio_status`, selected candidate, and `action` for every image block. The - complexity report must record `detail_score`, `object_count_estimate`, - `simple_icon_risk`, `reference_crop_match`, `style_match`, and - `selected_reason`; unresolved `too_simple`, `generic_icon`, - `reference_crop_ignored`, `single_object_on_blank_background`, or - `style_drift` must block delivery. - `composition_quality_report.json` must prove that PPT insertion used - frameless image slots, no extra white tile, no caption inside the image slot, - and at least 95% image-slot area fill. -9. Compose the final figure in `editable_composition.pptx` by default. Use PPT - editable layers for containers, arrows, connector lines, dashed loops, labels, - formulas, legends, and panel IDs. Image slots are direct frameless assets; - arrows are PPT shapes/connectors, not generated PNGs. -10. Write an alignment review noting source grounding, slot count, rejected assets, - and reference-image mismatches. -11. Write `visual_critic_iter_0.json` and `critic_report.md` or - `critic_report.json` before final delivery. - It must check paper faithfulness, slot count, text controllability, whether - the output used a single full diagram, vector-only fallback, semantic - cropping, incomplete image blocks, asset fill quality, asset complexity, - reference-crop match, style match, and PPT editability. - Major failures must be fixed or the delivery must stop. -12. Run `scripts/validate_framework_outputs.py ` before final - delivery when a local output directory exists. -13. Optionally run `rfs presentations-qa --out ` after validation. - Treat `autoRouteConnectorPx failed` or similar plugin connector fallback - warnings as QA evidence. Do not let the Presentations plugin rebuild the - figure or override RFS connector geometry. - -Hard workflow order: - -`input archive -> paper brief -> reference slot/control analysis -> reference_geometry.json -> -reference_control_candidates.json -> slot_overlay.png/reference_control_overlay.png -> -reference_controls.json -> arrow_style_profile.json/selected_arrow_routes.json/arrow_quality_report.json -> -reference_style_profile.json/style sheet -> -layout_plan.json -> figure_program.json -> reference_text_geometry.json/text_program.json/text_alignment_report.json -> slot_visual_spec.json -> -reference_slot_prompt_brief.json -> slot_prompt_plan.json -> image2/Gemini slot prompts -> generated -assets/asset_quality_report/asset_complexity_report/composition_quality_report/asset_visual_review/contact sheets -> -editable_composition.pptx -> PDF/PNG export -> visual critic -> critic report -> -final validation/export -> optional presentations QA report` - -Image block fill rules: - -- Target useful visual content fill: 90%-97% of the canvas. -- Minimum passing content fill: 85% unless the slot is explicitly marked as a - sparse symbol with a documented reason. -- Maximum pure empty margin in any direction: 10%. -- Safe area means the key subject is not cut off; it does not mean leaving a - large blank border. Background texture, secondary details, card surfaces, and - non-critical supporting marks should extend close to the canvas edge. -- If a block has too much whitespace, regenerate or redesign the prompt. Padding - is only for ratio matching, not for fixing a tiny subject on a blank canvas. - -Forbidden shortcuts: - -- Do not satisfy an image-rich framework request with vector-only shapes unless - the user explicitly asks for a vector-only diagram. -- Do not silently replace image2 blocks with PPT shapes, SVG, or Lark shapes - because that is faster or easier. -- Do not deliver only an SVG source file for an editable research figure unless - the user explicitly asks for SVG-only or Illustrator/Inkscape-first work. -- Do not generate a single full diagram image and then screenshot or crop it as - the final result. -- Do not use a browser or canvas screenshot as the primary artifact unless it is - a rendered composition assembled from generated slot assets. -- Do not use the Presentations plugin as the primary compiler for image-rich - research figures; it is QA-only unless the user explicitly asks for a - separate presentation deck workflow. -- If image2/image generation is unavailable, say so and stop or ask for a - fallback; do not silently switch to a vector-only workflow. - -Required output files for image-rich framework figures: - -- `slot_inventory.json` or `slot_inventory.md` -- `reference_geometry.json` -- `reference_control_candidates.json` -- `slot_overlay.png` -- `reference_control_overlay.png` -- `reference_controls.json` -- `arrow_style_profile.json` -- `selected_arrow_routes.json` -- `arrow_quality_report.json` -- `reference_style_profile.json` -- `style_sheet.md` or `style_sheet.json` -- `input_manifest.json` -- `layout_plan.json` -- `figure_program.json` -- `reference_text_geometry.json` -- `text_program.json` -- `text_alignment_report.json` -- `slot_visual_spec.json` -- `reference_slot_prompt_brief.json` -- `slot_prompt_plan.json` -- `reference_slot_crops/.png` for every non-arrow image slot -- `prompts.md` -- at least 25 generated image assets for normal system figures -- `asset_candidate_contact_sheet.png` -- `asset_quality_report.json` -- `asset_complexity_report.json` -- `composition_quality_report.json` -- `asset_visual_review.json` -- `asset_contact_sheet.png` -- `editable_composition.pptx` -- final figure export -- `visual_critic_iter_0.json` -- `alignment_review.md` -- `critic_report.md` or `critic_report.json` - -Optional Presentations-plugin QA outputs: - -- `presentations_plugin_qa_report.json` -- `presentations_plugin_qa_report.md` -- `presentations_plugin_qa_workspace/` - -Validation must fail if the output uses only a single generated full-diagram -image, lacks slot assets, lacks prompts, lacks a contact sheet, or records -semantic cropping as the fitting strategy. Validation must also fail when the -style sheet, layout plan, figure program, reference text geometry, text program, -text alignment report, slot visual spec, reference slot -prompt brief, slot prompt plan, reference geometry, asset quality report, asset -complexity report, composition quality report, asset visual review, visual -critic report, critic report, or editable PPTX source is missing. -Validation must fail when any unresolved image block has `content_fill_percent` -below its minimum or `empty_margin_percent` above its maximum. -Validation must fail when a slot uses a coarse preset ratio, when PPT insertion -adds an extra white tile, or when an inserted image fills less than 95% of its -slot area. -Validation must fail when PPT text is not bound to -`reference_text_geometry.json`, when text center/bbox/font-ratio drift from the -reference exceeds the recorded alignment tolerance, or when critical text is -not editable in PPTX. Validation must not fail only because the text would be -small under a default paper-double-column scaling assumption. -Validation must fail when a normal non-legend slot lacks `secondary_objects` or -`micro_details`, or when a selected asset has unresolved `too_simple`, -`generic_icon`, `reference_crop_ignored`, `single_object_on_blank_background`, -or `style_drift`. -Validation must fail when arrow, dashed loop, transition, or connector elements -appear in `slots`/`assets`; they must appear in `reference_controls.json` and -`figure_program.json` as editable PPT controls with source/target logic and -style tokens. Validation must also fail when `reference_control_candidates.json` -or the slot/control overlay images are missing, when a bound control lacks -`source_id`, `target_id`, `source_anchor`, `target_anchor`, or at least two -`path_percent` points, or when composition reports that a control was not -rendered as an editable PPT connector. Validation must fail when -`arrow_style_profile.json`, `selected_arrow_routes.json`, or -`arrow_quality_report.json` are missing, or when an arrow style stage overrides -a locked reference path without a documented reason. Validation must also fail -when fallback obstacle routing is applied to a reference-locked path, or when -route artifacts omit `routing_algorithm` / `route_generation_status`. -For aesthetic mode, validation or critic review must fail when -`reference_tunnel_preserved` is false, when source-target binding changes, or -when a curve/bundle/halo style is baked into raster images instead of PPT -editable connector shapes. -Validation must also fail when image slots lack local reference crops, local color token ids, or -`reference_style_profile.json` grounding. - -## Core Workflow - -1. Read the source material first. Extract the research problem, method story, - inputs/outputs, named components, training/inference flow, datasets, baselines, - metrics, and main claims before planning any figure. -2. Create a candidate figure inventory. Include only figures that carry paper - information: method overview, system pipeline, model architecture, data - construction, main results, ablation, case study, error analysis, or appendix - figures. -3. Route each figure: - - **Framework / architecture / pipeline**: use the visual-block workflow in - `references/visual_framework_figures.md`. - If the user provides a full reference image first, - also use `references/reference_image_alignment.md`. - Before image2 prompting, write a style sheet using - `references/stylist_stage.md` and a structured program using - `references/figure_program.md`. - Then create `reference_slot_prompt_brief.json` and VLM-generated - `slot_prompt_plan.json`; for image2 visual blocks, use - `references/image_block_prompting.md` to - generate slot-level prompts with target aspect ratios and low-blank-space - requirements. - Before delivery, run the critic stage in `references/critic_stage.md`. - - **Data / statistical / result plots**: use `references/data_figures.md`. - - **Mixed paper figure or multi-panel layout**: combine both routes, then use - PPTX-first composition for the editable working file. -4. Keep scientific structure editable. Use generated images for visual blocks, - but add labels, formulas, arrows, panel marks, grouping boxes, legends, and - captions with editable PPT tools by default. -5. For reference-image-driven framework figures, parameterize the reference image - first: identify visual slots, estimate each slot's relative bounding box and - aspect ratio, then generate many small blocks for those slots instead of a few - large macro-panel images. -6. Run the journal-quality check before final delivery. See - `references/journal_quality_checklist.md`. - -## Required Grounding Rules - -- Every framework module name must come from the user's paper or be explicitly - introduced as an inferred simplification. -- Never insert stock modules such as encoder, retriever, memory, or decoder unless - the paper actually uses that concept or the user approves the abstraction. -- AI-generated visual blocks may illustrate scientific objects, data flow, - model components, reasoning stages, or experimental settings, but must not be - trusted for exact text, formulas, numbers, axes, or citations. -- For data figures, Python remains the source of truth for computation and plots. - PPT editing is for layout, annotation, panel assembly, and presentation polish, - not for changing plotted values. -- Keep source artifacts: raw input, extracted figure brief, generated prompts, - full reference image when provided, original image blocks, enhanced blocks, - slot inventory, layout plan, style sheet, figure program, asset quality - report, asset visual review, visual critic report, critic report, alignment - review notes, editable composition, and final exports. - -## Output Policy - -Default outputs: - -- editable working file: `editable_composition.pptx` by default -- publication export: PDF plus 600 DPI PNG/TIFF by default -- optional SVG/PDF vector export only when the venue, user, or downstream - Illustrator/Inkscape workflow explicitly needs it -- reproducibility files: Python plotting script and cleaned data for data plots -- prompt record: visual-block prompts and selected/rejected image notes -- for image-rich framework figures: slot inventory, image2 prompts, generated - assets, asset quality report, asset visual review, contact sheets, style - sheet, layout plan, figure program, editable composition, visual critic, - critic report, final export, and alignment review that pass - `rfs validate` or `scripts/validate_framework_outputs.py` when outputs are - local - -Use `references/paper_to_figure.md` for paper analysis and figure planning. Use -`references/enhancement_pipeline.md` when generated images need upscaling, -sharpening, background removal, or artifact cleanup. Use -`references/reference_image_alignment.md` only when the user supplies a complete -reference framework image as the visual blueprint for the final editable -composition. Do not create a full reference image yourself unless the user -explicitly asks for one. Use `references/stylist_stage.md` to lock style before -prompting. Use `references/figure_program.md` to define the structured -intermediate layout before scripting or composition. Use -`references/image_block_prompting.md` whenever image2 prompts are needed for -framework sub-blocks. Use `references/critic_stage.md` to block final delivery -when the workflow violates the required image-rich, editable, no-crop rules. - diff --git a/codex-skills/research-figure-making/agents/openai.yaml b/codex-skills/research-figure-making/agents/openai.yaml deleted file mode 100644 index e4b5c2e..0000000 --- a/codex-skills/research-figure-making/agents/openai.yaml +++ /dev/null @@ -1,7 +0,0 @@ -interface: - display_name: "Research Figure Making" - short_description: "Create paper-grounded scientific figures and editable framework diagrams." - default_prompt: "Use $research-figure-making to turn my paper draft into publication-quality framework and data figures." - -policy: - allow_implicit_invocation: true diff --git a/docs/architecture.md b/docs/architecture.md index bf376f3..0ab1925 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,69 +1,101 @@ -# Summary +# Summary -ResearchFigureStudio is organized as a small deterministic pipeline, not a patched copy of LiveFigure. The borrowed ideas are reference-image-based positioning and AutoFigure-style structured control localization: the system writes overlay/candidate JSON for visual elements, then renders editable PPT objects from deterministic programs. +ResearchFigureStudio has one product goal: turn a paper into a scientifically faithful, editable PowerPoint figure. Paper-derived contracts own semantics; generated or supplied images own layout and visual style; deterministic code owns PPTX composition and validation. -## Architecture +## Product architecture + +```mermaid +flowchart LR + P["Paper"] --> G["PaperGroundTruth"] + R["Optional visual references"] --> I["Reference image generation"] + G --> I + I --> V["Approved visual reference"] + V --> A["Layout, cards, slots, style analysis"] + G --> B["Semantic binding"] + A --> B + B --> C["Editable figure program"] + C --> PPT["Editable PPTX"] + PPT --> Q["Deterministic and optional VLM QA"] +``` + +The semantic path never passes through OCR alone. OCR may help recover geometry from an image, but exact labels, entity IDs, relations, and evidence links are restored from `paper_review.json` and `figure_specification.json` before compilation. + +## Repository map ```text -Input Loader - -> Paper Analyzer - -> Reference Analyzer - -> Control Localizer - -> Arrow Stylist/Router - -> Stylist - -> Layout Locator - -> Figure Program Builder - -> Prompt Planner - -> Asset Generator - -> Asset Reviewer - -> PPT Compiler - -> Exporter - -> Visual Critic - -> Validator +.codex-plugin/ Codex plugin manifest +skills/research-figure-studio Thin agent instructions; no machine paths +rfs/ Installable, agent-independent Python engine + contracts/ Paper semantic contracts and stable schemas + providers/ VLM and external model integrations + analysis/ Paper, layout, and text analysis entry points + planning/ Paper and rebuild planning entry points + generation/ Reference and slot-asset generation entry points + workflows/ Product-level workflow entry points + composition/ PPTX compilation and fallback preview rendering + evaluation/ Deterministic rebuild quality checks + paper_to_image/ Paper review, grounding, planning, image candidates + coevolution/ Experimental creator/judge research + paper_to_editable.py Product workflow orchestrator + semantic_contract.py Paper truth → editable object bindings + editable_rebuild.py Reference image → editable figure program/PPTX + ppt_compiler.py Deterministic PowerPoint renderer + rebuild_visual_critic.py Deterministic rebuild QA +tests/ Unit, integration, and future end-to-end tests +benchmarks/ Real product acceptance cases and metrics +experiments/ Research prototypes and historical scripts +docs/ Architecture, workflows, decisions, roadmaps +scripts/ Reusable installation/maintenance scripts only ``` -## Modules - -- `rfs/input_archive.py`: copies paper and reference into `out/inputs/` and writes `input_manifest.json`. -- `rfs/input_loader.py`: extracts text from source documents. -- `rfs/paper_analyzer.py`: creates paper-grounded figure brief. -- `rfs/reference_analyzer.py`: creates 25-50 non-arrow slot inventory from the reference image geometry and paper figure goal; it also detects arrow/control candidates, writes overlays, and emits `reference_control_candidates.json` / `reference_controls.json`. -- `rfs/arrow_router.py`: assigns reference-preserving arrow aesthetics, bundle metadata, line softness, and route QA reports without changing reference-image flow logic. It also provides reference-constrained orthogonal fallback routing for missing or explicitly fallback-allowed connectors only. In aesthetic mode it records original paths and applies reference-tunnel bundle offsets only for explicitly opted-in routes. -- `rfs/stylist.py`: writes the style sheet before image prompting. -- `rfs/layout_locator.py`: creates `layout_plan.json`; heuristic mode is local, VLM mode returns JSON coordinates only. -- `rfs/program_builder.py`: creates `figure_program.json`, the single source for PPT compilation, preserving control source/target anchors and normalized routes from `reference_controls.json`. -- `rfs/prompt_planner.py`: creates `reference_slot_prompt_brief.json` and `slot_prompt_plan.json`; default VLM mode inspects the full reference image plus local slot crops and writes `image_prompt_core` for each slot. Per-slot calls can run in parallel via `--prompt-plan-workers`. -- `rfs/asset_generator.py`: generates 1-5 candidates per slot, selects the best block, and writes contact sheets and QA metrics. -- `rfs/asset_reviewer.py`: performs heuristic or VLM visual review of selected assets. -- `rfs/ppt_compiler.py`: renders editable PPTX from `figure_program.json` with contain-fit images and editable labels/arrows/groups; multi-segment arrows and dashed loops are rendered as PPT connector shapes, not image assets. -- `rfs/exporter.py`: exports PDF/PNG where local tooling supports it. -- `rfs/visual_critic.py`: compares reference and final render; VLM mode can propose JSON coordinate corrections and arrow-only patches without rewriting the whole figure. -- `rfs/validator.py`: blocks delivery when required artifacts or hard constraints are missing. - -## Non-Goals - -- Do not import LiveFigure as a dependency. -- Do not let a VLM write arbitrary PowerPoint code. -- Do not generate a single full architecture image and crop it into pieces. -- Do not make SVG the main editing source. -- Do not bake critical scientific text, formulas, arrows, or labels into image blocks. -- Do not count arrows, connector lines, dashed loops, or transition symbols as generated image slots. - -## Data Contracts - -- `slot_inventory.json`: paper concept plus candidate slot metadata. -- `reference_control_candidates.json`: AutoFigure-inspired boxlib-like candidate list for arrows, connectors, loops, and branch routes. -- `slot_overlay.png`: visual overlay of detected slot IDs for human/VLM binding. -- `reference_control_overlay.png`: visual overlay of control candidate IDs such as `AR01`. -- `reference_controls.json`: bound editable PPT controls with `source_id`, `target_id`, anchors, path, style token, and render policy. -- `arrow_style_profile.json`: reference-first arrow style rules, fallback routing policy, and routing algorithm metadata for main flow, branch, convergence, feedback loops, and module flow. -- `selected_arrow_routes.json`: selected route/style assignments, bundle IDs, lane indices, reference-lock status, route generation status, and routing algorithm. -- `arrow_quality_report.json`: crossing, bend, obstacle-overlap, and aesthetic-score diagnostics for PPT connectors. -- `composition_quality_report.json`: records PPT arrow connector type, halo usage, route style, and editability alongside slot composition checks. -- `layout_plan.json`: normalized positions, panels, arrows, and z order. -- `figure_program.json`: final composition program consumed by the PPT compiler. -- `reference_slot_prompt_brief.json`: per-slot paper concept, local reference role, geometry, and function briefing sent before prompt planning. -- `slot_prompt_plan.json`: VLM-generated per-slot prompt plan, including `image_prompt_core`, `must_show`, and `avoid_showing`. -- `asset_quality_report.json`: fill/margin/ratio/cutoff/candidate-selection metrics. -- `asset_visual_review.json`: semantic and visual review of selected image blocks. -- `visual_critic_iter_*.json`: reference alignment and optional coordinate correction proposals. +The current package remains under `rfs/` to avoid a disruptive all-at-once move. A future `src/rfs/` migration should be a packaging-only change after the paper-to-editable contracts and benchmarks stabilize. + +## Authority model + +| Concern | Source of truth | +|---|---| +| Scientific entities and exact labels | `paper_review.json`, `figure_specification.json` | +| Scientific relations and evidence | Paper semantic contract | +| Entity roles, provenance, and evaluation boundaries | Normalized evidence-backed contract completion | +| Paper-to-image draft node and connector geometry | `layout_blueprint.json.semantic_plan` | +| Direct paper-to-PPT node and connector geometry | `figure_program.json` compiled from the same semantic plan | +| Layout, visual rhythm, palette, object style | Approved generated/user reference image | +| Editable object geometry | `figure_program.json` | +| PowerPoint rendering | `ppt_compiler.py` | +| Production eligibility | Candidate, semantic, visual, and PPTX validation reports | + +The scientific contract cache is keyed by paper hash, planner model, compiler +version, and scientific domain profile. Canvas ratio and display-language +preferences are intentionally excluded because they are rendering concerns and +must not produce competing scientific interpretations of the same paper. + +Topology normalization also owns role-consistent branch heads and shortcut +removal, while deterministic template rendering owns exact nested connector +anchors. Candidate review measures raster enrichment inside the detected active +blueprint region and records both active-region and whole-canvas ratios. + +## Stable workflow boundaries + +- `fast-framework-prompt`: produces the cached semantic contract and prompt; `--editable-ppt` additionally emits a fast native-shape PPTX without image generation. +- `paper-to-image`: produces reviewed raw, geometry-normalized substrate, and composed visual candidates plus paper-grounding artifacts; `--editable-overlay-ppt` also creates a PPTX whose labels and connectors are native editable objects over the normalized visual substrate. +- `rebuild-editable`: reconstructs an image; it can optionally accept a paper semantic contract. +- `paper-to-editable`: runs both stages and requires a production-approved image unless engineering preview use is explicitly enabled. +- Plugin skill: selects and invokes workflows; it does not contain the engine or hardcoded repository paths. +- Benchmark layer: independently scores paper-to-image scientific/visual quality and image-to-PPT fidelity/editability. + +## Production rules + +- Failed candidates never become `selected_image.png`. +- Whole-image Image2 candidates must pass exact card/node cardinality, unique geometry matching, deterministic re-slotting, restored connector corridors, and a no-semantic-crop gate before VLM approval can count. +- Engineering previews are opt-in and never production-approved. +- Critical text and relations remain editable PPT objects. +- Image/OCR guesses cannot override paper-grounded terminology or relation endpoints. +- Deterministic completion is graph-scoped: rich VLM contracts only gain overview-local or directly connected evidence-backed entities, while heuristic contracts grow from overview seeds or the largest connected method chain. Late appendix-only diagram labels, ungrounded innovations, and input shortcuts that bypass an explicit intermediate node are removed before planning. +- `semantic_blueprint.py` compiles 2-16 visible contract entities into normalized graph geometry. It ranks DAG layers, places source-only conditioning nodes immediately before their targets, assigns distinct fan-in/fan-out ports, routes skip-layer edges around intermediate nodes, and sends feedback loops through an outer lane. +- Benchmark entity matching uses both label fidelity and relation consistency so relation labels and variable suffixes cannot steal mappings from real contract entities. +- VLM correction is optional and bounded; deterministic reports remain available offline. +- Historical sample-specific scripts stay under `experiments/legacy_reference_rebuilds/` and are not imported by production code. + +## Planned package refinement + +As interfaces stabilize, split `rfs/` internally into `contracts`, `providers`, `analysis`, `planning`, `generation`, `composition`, `evaluation`, and `workflows`. Do this incrementally with compatibility imports, not as a mass rename that obscures behavior changes. diff --git a/docs/gpt-reproduction-workflow.md b/docs/gpt-reproduction-workflow.md index e80ba79..c3811db 100644 --- a/docs/gpt-reproduction-workflow.md +++ b/docs/gpt-reproduction-workflow.md @@ -21,7 +21,7 @@ rfs rebuild-editable --reference input.png --out output\demo --asset-mode api -- For higher-quality reference-only reproduction, use the scripted professional workflow. It asks the VLM to generate a controlled Figure DSL that mimics the best one-off rebuild scripts, then compiles that DSL safely: ```powershell -rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --repair-rounds 2 --export-preview +rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --asset-policy smart-api --repair-rounds 2 --export-preview ``` The evaluation command is: @@ -57,7 +57,7 @@ Existing one-off scripts in `scripts/` are historical examples. Do not treat the Install locally: ```powershell -cd D:\ResearchFigureStudio +cd python -m pip install --upgrade pip python -m pip install -e . ``` @@ -99,7 +99,7 @@ Never write API keys into source files, docs, fixtures, or committed outputs. ### 1. Start With A Cost-Safe Evaluation -Use crop assets first. This validates layout/control/semantic VLM quality without spending image-generation credits: +Use crop/placeholder assets first only as an engineering check. The evaluator may use crop for cost-safe structure comparison; production/pro runs should use `smart-api` so crop is not inserted as a final PPT asset: ```powershell rfs rebuild-editable-eval ` @@ -136,6 +136,7 @@ rfs rebuild-editable ` --reference "C:\path\figure.png" ` --out "output\editable_rebuild" ` --asset-mode api ` + --asset-policy smart-api ` --layout-mode hybrid ` --control-mode hybrid ` --text-mode ocr ` @@ -215,6 +216,7 @@ A result is acceptable only if: - Complex visuals are slot-level assets under `assets/`. - `rebuild_vlm_validation_report.json` has `status: pass` or only explainable warnings. - `asset_generation_report.json` records API request counts and fallbacks. +- `asset_decision_report.json`, `text_asset_filter_report.json`, and `api_asset_plan.json` explain which slots are API-generated, reused, or rejected as text. - `reference_geometry_overlay.png` and `reference_controls_overlay.png` visually match the reference closely enough for the current stage. For `rfs rebuild-editable-pro`, additionally check: @@ -258,10 +260,10 @@ Use these modes deliberately: rfs rebuild-editable --reference input.png --out output\placeholder --asset-mode placeholder --layout-mode heuristic --control-mode heuristic # VLM structure test without image generation cost. -rfs rebuild-editable --reference input.png --out output\crop --asset-mode crop --layout-mode hybrid --control-mode hybrid +rfs rebuild-editable --reference input.png --out output\crop --asset-mode crop --asset-policy smart-api --layout-mode hybrid --control-mode hybrid # Full quality run. -rfs rebuild-editable --reference input.png --out output\api --asset-mode api --layout-mode hybrid --control-mode hybrid +rfs rebuild-editable --reference input.png --out output\api --asset-mode api --asset-policy smart-api --layout-mode hybrid --control-mode hybrid ``` Default economy behavior: @@ -325,7 +327,7 @@ Action: - Inspect `asset_generation_specs.json` for `generation_aspect_ratio`, `main_subject_fill_target`, and `prompt_subject`. - Regenerate only failed slots with `--regenerate-slots`. -- Do not globally crop assets unless the user explicitly approves a controlled trim experiment. +- Do not use reference crops as final PPT assets in smart-api/pro runs. Crops are context for API generation only. ## Test Commands diff --git a/docs/help-wanted.md b/docs/help-wanted.md index 2000b47..63e80e0 100644 --- a/docs/help-wanted.md +++ b/docs/help-wanted.md @@ -67,8 +67,8 @@ We welcome help on: - `rfs/ppt_compiler.py` - `rfs/visual_critic.py` - `rfs/validator.py` -- `codex-skills/research-figure-making/references/reference_image_alignment.md` -- `codex-skills/research-figure-making/references/figure_program.md` +- `skills/research-figure-studio/references/reference_image_alignment.md` +- `skills/research-figure-studio/references/figure_program.md` ## Desired End State diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 190b341..ce5ca36 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -1,6 +1,86 @@ # Summary -`rfs paper-to-image` produces a reviewed raster scientific framework image without creating PPTX. Production mode requires successful VLM paper review, a content-free architecture blueprint, reference-conditioned Image2 edit generation, and four production quality gates. +`rfs paper-to-image` produces a reviewed raster scientific framework image without creating PPTX. Production mode requires successful VLM paper review, a paper-grounded semantic blueprint, reference-conditioned Image2 visual generation, deterministic exact-label/connector composition, and the full production quality gates. + +## Three-Minute Fast Contract + +```powershell +rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --json +``` + +The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/contract caches. Layout, style, prompt, and editable overlays are compiled deterministically after the semantic graph. A warm paper normally completes in well under a second. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. + +Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. OCR lines use the same detector, including a three-to-two-column fallback for first pages where centered title fragments otherwise resemble a third column. Quality gates count all Unicode letters and numbers. English, Spanish, French, German, Portuguese, Chinese, Japanese, and Korean section aliases are recognized; localized `Figure/Table` prefixes and Chinese `图/表` captions are indexed. Caption prefixes must include a real numeric, Roman, or appendix identifier, so ordinary prose such as `figura editable` is not misclassified. CJK cross-extractor agreement uses character bigrams so per-character spacing differences do not resemble corruption. Cross-extractor quality uses directional lexical coverage while retaining order-sensitive agreement for diagnostics, preventing multi-column order differences, equation variables, and Poppler-only hidden text from causing wasteful OCR. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. Up to three selected pages run concurrently, and short narrow or vertical fragments in the outer page margins are removed as likely watermarks/page furniture. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. `RFS_OCR_WORKERS` overrides page concurrency. + +If a long fully scanned paper cannot reach the 60% full-document readability gate within the deadline, the workflow may continue only when at least six high-confidence sampled pages cover both Abstract and Method. The result records `semantic_scope: sampled_pages_only`, includes an explicit uncertainty about unprocessed pages, and can never be production-ready. Missing Abstract/Method coverage or weak OCR still stops semantic planning. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. + +Run all or selected paper benchmarks with aggregate reliability metrics: + +```powershell +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/fast-suite --planner-mode heuristic --json +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/unseen --case-id 106_detr_set_prediction --case-id 107_clip_contrastive --json +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/nlp --case-id 109_transformer_encoder_decoder --case-id 110_bert_pretrain_finetune --case-id 111_rag_retrieval_generation --planner-mode vlm --json +``` + +For OCR performance diagnosis, `extraction_report.json` and `benchmark pdf-suite` now record page rendering, text detection, orientation classification, text recognition, and post-processing time separately. RapidOCR defaults are deliberately conservative: detector limit `512`, recognition batch `6`, and an adaptive page-worker cap (4 on 8+ logical CPUs, 2 on 4-7, 1 below 4). Advanced local experiments can override these without changing repository defaults: + +```powershell +$env:RFS_RAPIDOCR_DET_LIMIT = "512" +$env:RFS_RAPIDOCR_BATCH = "6" +$env:RFS_OCR_WORKERS = "3" +rfs benchmark pdf-suite --out output/benchmarks/pdf_profile --ocr-engine rapidocr --json +``` + +Lower detector limits are not assumed to be faster. On dense two-column scans, recognition usually dominates and smaller detector settings can have unstable latency; accept a change only after text agreement, section recovery, caption recovery, confidence, and wall-clock gates all pass. + +Deadline-limited or failed OCR schedules are explicitly marked with `ocr_schedule_complete=false` / `ocr_run_complete=false`. Such document models are never written to the global document cache, and scientifically partial scan results are never written to the semantic contract cache; a transient slow or failed OCR run therefore cannot poison later fast runs. + +When a deadline is active, every local OCR page runs in an isolated worker process. Completed pages are preserved, while workers still running at the reserved cutoff are terminated and recorded with `timed_out=true`. Single-page RapidOCR retains two intra-op threads; multi-page waves use one thread per worker. This prevents a slow ONNX recognition call from holding the fast workflow beyond its parsing budget or leaving background OCR processes behind. + +For a fully scanned long paper with no native section signals, the six-page fallback schedule covers pages 1-4 first, then a page near 85% of the document and finally a middle page. Under a deadline, the first wave uses adaptive page concurrency; the two coverage-rescue pages run one at a time with two OCR threads whenever at least 45 seconds remain. This biases scarce OCR time toward the abstract, overview figure, early method details, and likely appendix/conclusion architecture evidence without weakening the hard cutoff. + +In the 180-second fast path with `planner_mode=vlm`, rescue pages require at least 90 seconds remaining so the workflow can preserve a model-call and validation budget. Heuristic planning and PDF inspection retain the 45-second rescue threshold. The effective value is recorded as `ocr_rescue_min_remaining_seconds`. + +Planner and review retries share one absolute provider deadline rather than receiving a fresh timeout per attempt. If the service is slow or unavailable, each request is clamped to the remaining budget, further retries stop at the cutoff, and any later semantic stage switches to the deterministic evidence-grounded fallback. + +Born-digital and OCR-recovered pages also pass through a conservative repeated-margin filter. Short header or footer lines are removed only when the same digit-normalized pattern appears in the same margin on at least three pages and at least one quarter of the document. The report records `repeated_margin_noise_removed_count` so benchmark runs can detect publisher-header and page-number contamination without hiding the cleanup. + +For pages carrying a PDF rotation flag, native blocks are ordered in the unrotated media-box coordinate system and only then transformed into displayed page coordinates. This preserves Abstract-to-Method-to-Figure reading order on 90/180/270-degree pages while keeping overlay coordinates aligned with rendered output. + +Fast VLM planning preserves the paper's original writing system for every visible scientific label. Entity names and relation labels must either occur verbatim in their cited evidence or remain in the same source script; cross-script translations such as `文档编码器` to `Document encoder` fail planning validation and cannot be marked production-ready. Explanatory prose may follow the requested output language, but editable overlay labels do not. + +For Spanish, French, German, and Portuguese papers, validation also detects same-script English translation. An English-looking visible label that does not occur verbatim anywhere in the paper evidence fails production validation even though both strings use Latin characters. This closes the gap where `codificador de documentos` could otherwise be silently changed to `Document Encoder`. + +Relation labels are optional: if a model supplies a connector label that is not verbatim in the relation's cited evidence, normalization clears the label while preserving the grounded direction and type. Overlay compilation also emits an exact scientific label only once when the same evidence-backed concept is both a module and an innovation callout. + +Native PDF lines split by publisher hyphenation are joined inside the same source text block before evidence IDs are created. Known scientific words such as `trans-` + `former` and `de-` + `coder` lose the layout-only hyphen, while unknown long compound parts retain a literal hyphen. The extraction report records `native_hyphenation_repair_count`. + +OCR normalization also repairs fused numbered section boundaries such as `2方法`, `2Metodo`, `3Experimentos`, and `4Conclusion` before heading classification. The same rule covers common French, German, and Portuguese headings. English word segmentation now requires at least two scientific anchor words, preventing a single English-looking suffix from splitting valid non-English words such as Spanish `procesamiento`. + +Production validation treats visible inputs, outputs, and innovations without evidence IDs as hard errors. Before validation, exact entity labels may be grounded deterministically to matching evidence; innovation labels are grounded only when the same evidence also contains an explicit novelty cue such as `we propose`, `we introduce`, or `本文提出`. Otherwise the result remains engineering-only instead of silently presenting an unsupported contribution. + +Repeated-margin filtering interprets page edges in the PDF's semantic orientation. On a 90-degree page, the original header is detected at the displayed right edge and the original footer at the left edge; corresponding mappings are applied for 180 and 270 degrees. Mixed-orientation papers therefore use one header/footer signature space without leaking rotated publisher furniture into evidence. + +Section headings are deduplicated only within their concrete page/block occurrence, not globally by title text. Repeated `Methods`, `Experiments`, or appendix headings therefore create new evidence boundaries after publisher headers have been removed. + +PDF inputs receive a lightweight container preflight before pypdf, Poppler, or PyMuPDF extraction. Missing `%PDF-` headers and missing tail `%%EOF` markers return stable `ValueError` messages immediately; encrypted documents retain the explicit password-required error. This keeps `--json` failures concise and prevents damaged files from entering either cache. + +When `benchmark pdf-suite` runs with a real OCR engine, it now includes fully rasterized two-column and 2-degree skewed two-column pages in addition to the mixed native/scan fixture. These cases enforce column reconstruction, left-column-first reading order, section recovery, caption recovery, and OCR confidence using the actual runtime engine. + +RapidOCR pages render at 84 DPI by default. This retains the detector limit and recognition batch used by the fast path while recovering long Latin-script lines that could disappear at 72 DPI. The generated stress suite includes native and rasterized Spanish papers and records `render_dpi` for each OCR page; the current suite contains 10 parser-only cases and 16 cases with runtime OCR. + +English OCR pages with at least two explicit English section anchors also receive a lexical plausibility check backed by the bundled word-frequency model. A known-word ratio below 0.72 raises a warning and below 0.50 fails extraction even when the OCR engine reports high confidence. The gate is not applied to CJK or unrecognized non-English Latin documents, preventing an English dictionary from rejecting unrelated languages. + +Table captions now seed coordinate-based table regions. Header cells define column anchors, later cells are assigned by row and nearest column, and the document index stores `columns`, `rows`, `cells`, and a full table bounding box. Individual table cells are excluded from narrative evidence and replaced by one row-major structured table evidence record, preventing OCR detection order from turning `Model | Depth | Accuracy` into misleading prose. + +Table reconstruction is intentionally conservative: a candidate needs a plausible labeled header and at least two data rows. Pure numeric rows and lowercase prose fragments such as `are | obtained` are rejected without changing the original block kinds, avoiding false table structure on captions, equations, and surrounding sentences. + +If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. + +`fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. +The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. It also normalizes string entities plus `source_id`/`target_id`/`relation_type` variants, resolves a missing endpoint only when the relation label exactly identifies one declared entity, grounds short uppercase acronyms through exact token matches, and removes unresolved relations into explicit uncertainties. These repairs are paper-name agnostic and are covered by the NLP suite. + +When a paper exceeds the evidence character budget, the extractor reserves representative evidence from every page, then prioritizes topology-defining statements and Abstract, Conclusion, Method, Introduction, and Experiments content. This prevents long introductions or appendices from silently excluding later conclusions and framework definitions. ## Production Command @@ -27,6 +107,48 @@ rfs paper-to-image ` --json ``` +Resume a failed candidate with one localized repair instead of generating a new +initial batch: + +```powershell +rfs paper-to-image ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_image_repair" ` + --planner-mode vlm ` + --asset-mode image2 ` + --review-mode vlm ` + --repair-source "output\paper_to_image\candidates\candidate_01.png" ` + --repair-rounds 1 ` + --image-retries 0 ` + --ocr-engine off ` + --json +``` + +`--repair-source` skips fresh initial candidate generation. The source is first +reviewed against the current exact-label and scientific contract. If it fails, +the best source is edited once and reviewed again. The request manifest marks +the reused source separately from the real Image2 repair request. This remains a +compatibility route for older images whose labels and arrows are already baked +in, so deterministic overlay mode is disabled for an explicit external repair +source. + +Re-audit an existing complete or failed run without regenerating an image: + +```powershell +rfs benchmark audit ` + --run output/paper_to_image/existing_run ` + --case benchmarks/paper-to-image/cases/101_vit_linear ` + --candidate-image output/paper_to_image/existing_run/candidates/repair_02.png ` + --json +``` + +The audit rebuilds `review_ground_truth.json`, scans the selected image against +quoted paper evidence, exact labels, visible connector topology, and aesthetics, +then writes `benchmark_review.json`, `paper_alignment_report.json`, +`reaudit_repair_plan.json`, and `paper_image_reaudit.json`. If neither +`selected_image.png` nor a selected candidate exists, the highest-scoring +recorded candidate with an existing image is audited automatically. + Never reuse a key that has appeared in chat, source control, an issue, or a terminal transcript. The command records only `api_key_present`; it never records a key value. ## Paper Review Contract @@ -44,18 +166,290 @@ Available profiles: ## Template Contract -Positive references are classified as `arbor`, `linear`, `tripanel`, or `dense-multimodal`. Each becomes a content-free profile containing normalized panels, topology, connector rhythm, density, palette, and forbidden copied content. Automatic selection uses module count, loops/tree structure, multimodality, retrieval structure, and requested ratio. +Built-in templates include `dense-multiframe`, `multimodal`, `branch`, +`feedback`, `arbor`, `linear`, `tripanel`, and `dense-multimodal`. Dense +task/model/data-engine overviews with nested local flows use `dense-multiframe`; +multi-input systems that converge through a modality encoder bank into one joint +space use `multimodal`; compact iterative +generation/feedback/refinement systems use `feedback`; shared-trunk systems with +parallel output heads use `branch`; true search-tree systems use `arbor`. +Positive references are classified into +the closest ratio-based reference archetype and become +content-free profiles containing normalized panels, topology, connector rhythm, +density, palette, and forbidden copied content. Automatic selection uses module +count, loop/tree/parallel-branch structure, multimodality, retrieval structure, +and requested ratio. + +The selected profile is rendered as `layout_blueprint.png` for review. When a +semantic plan is available, Image2 receives `visual_substrate_blueprint.png` +instead. That guide contains the exact node and panel geometry plus reserved +header bands, but no scientific text and no connectors. Reference-derived +profiles remain free of copied reference text and reference-specific objects. + +## Deterministic Structure Overlay + +New semantic-plan runs enable deterministic overlay mode by default: + +1. Image2 generates `candidates/candidate_XX_raw.png`, containing only the + image-rich visual substrate. +2. The geometry normalizer detects the visual cards, requires the detected card + count to equal the semantic node count, rejects ambiguous matching, removes + only the non-semantic generated header band, and contain-fits each visual + region into the exact semantic node box. It writes + `candidates/candidate_XX_substrate.png` and + `geometry/candidate_XX_geometry.json`. +3. `overlay_spec.json` supplies exact paper terminology and declared directed + relations. +4. `layout_blueprint.json.semantic_plan` supplies normalized node boxes and + connector `path_percent` geometry. +5. The compiler writes `candidates/candidate_XX.png` and + `overlays/candidate_XX_overlay.json`. +6. OCR, scientific, topology, template, aesthetic, and substrate-geometry gates review the + composed candidate, not the raw substrate. + +Image2 prompts explicitly forbid words, letters, numbers, formulas, arrows, +lines, loops, and pseudo-text. Exact labels and relation geometry are therefore +owned by the deterministic compiler. The same semantic structure can later be +emitted as editable PowerPoint labels and connectors; the raster overlay exists +so the current whole-image review path scans the same final appearance. + +The connector compiler uses global candidate routing rather than independent +per-edge elbows. It sorts fan-in/fan-out ports by geometry, reserves distinct +lanes, routes skip-layer and feedback edges through outside channels, and scores +every candidate against node obstacles, crossings, near-collinear shared +segments, bend count, and path length. `semantic_plan.route_quality` records: + +- `crossing_count` +- `shared_segment_overlap` +- `node_obstacle_intersections` +- per-connector `lane_assignments` +- the routing algorithm and pass state + +Geometry normalization is a hard contract step before VLM review. Production +requires exact card/node cardinality, unique matching, exact post-normalization +semantic boxes, restored connector corridors, and `semantic_crop_used=false`. +Failures use `failure_stage=substrate_geometry`, never provider failure. A VLM +critic cannot override this gate. + +Overlay compilation is also a hard contract step. Missing label targets, missing +connector geometry, or incomplete paths fail as `failure_stage=overlay_compilation` +and are not misreported as Image2 provider failures. + +Repair ownership is also separated. OCR, spelling, exact-label, arrow, and +topology findings are assigned to `overlay_compiler`; visual density, mechanism +illustration, hierarchy, balance, and substrate aesthetics remain assigned to +`image2_visual_substrate`. Image2 repair prompts never ask the model to modify +text or connectors in deterministic overlay mode. If only compiler-owned issues +remain, the repair loop stops with `overlay_recompile_required` instead of +waiting on an unnecessary image-provider edit. + +`--resume-candidates` prefers the raw/normalized/composed chain. A raw candidate is +re-normalized, re-overlaid, and re-reviewed using the current semantic boxes and routes. A legacy run that +contains only the composed PNG is reviewed without applying the overlay twice; +it cannot pass the new substrate-geometry gate or produce an editable substrate +PPT until the missing raw candidate is regenerated. + +With `--editable-overlay-ppt`, the selected normalized substrate becomes the +bottom PPT image layer. Node outlines, exact labels, relation labels, halo paths, +and arrowheads are native editable PowerPoint objects. The PPT never uses the +geometrically drifting raw Image2 canvas as its structural source. + +For supported normalized topologies (`linear`, `branch`, `multimodal`, +`feedback`, and `dense_multiframe`), contracts with 2-16 visible entities are +compiled into `layout_blueprint.json.semantic_plan`. The plan records exact +paper labels, normalized node boxes, graph ranks, distinct source/target ports, +relation types, connector labels, multi-point paths, and route styles. The +generic compiler: + +- puts independent inputs in one layer and separates their fan-in ports; +- keeps parallel branch outputs in one layer with separate fan-out ports; +- moves source-only conditioning nodes immediately before their target; +- routes skip-layer edges around intermediate nodes instead of through them; +- routes backward/feedback edges through an outer canvas lane; +- avoids connector/node intersections and near-overlapping long connector lanes; +- assigns fan-in and fan-out ports in geometric order to prevent avoidable crossings; +- records global route quality and per-connector lane ownership; +- refuses to force more than 16 visible nodes into one raster guide. + +When the contract matches a stricter built-in topology, the renderer uses a +specialized semantic layout instead: + +- `feedback`: the forward generation chain, two independent inputs to refinement, + the self-feedback node, and the return loop terminating at feedback; +- `dense-multiframe`: task/model/data-engine panels, separate image and prompt + encoder paths, decoder-to-mask output, the data-engine stage chain, and a + separate model-to-data-engine support arrow. + +This semantic blueprint is still generated from the evidence-backed contract; +it does not copy paper figures or invent labels. It reduces topology drift by +making Image2 preserve an explicit scientific source of truth instead of +inferring node mapping from empty placeholder boxes. + +Entity roles are part of that source of truth. Data sources, modalities, +processing modules, innovations, and outputs are reviewed separately even when +all their words are present. Explicit provenance text is normalized into a +source-to-modality-to-method chain, and multiple named sources may be collapsed +into one evidence-backed source group to avoid redundant nodes and crossed +edges. Explicit internal/external evaluation boundaries and named paper methods +or techniques are repaired deterministically when overview evidence states +them. + +Image-2 candidates must enrich the semantic blueprint rather than merely +restyling its boxes. Candidate review records `blueprint_enrichment_ratio` and +`blueprint_mean_abs_difference`; production defaults require at least `0.08` +changed-pixel enrichment inside the blueprint's detected active figure region. +Reports also retain `blueprint_global_enrichment_ratio`, +`blueprint_active_region_bbox`, and `blueprint_active_region_fraction` for +diagnosis. This keeps the threshold strict without penalizing feedback or +branch layouts that intentionally leave large white margins. The VLM aesthetic review separately scores visual +information density, mechanism visualization, and publication polish. Sparse +text-only nodes therefore fail even when labels, arrows, spacing, and colors +are otherwise correct. + +For branch figures, evidence-backed parallel leaf heads are normalized to the +same output-head role. Relations from an upstream ancestor directly into one +head are removed when they bypass the shared branch point already feeding that +head. If an innovation label is identical to an existing module or output, the +role contract highlights the existing node rather than requesting a duplicate. +For nested feedback layouts, the deterministic blueprint starts the feedback-to- +refinement connector at the inner feedback artifact, not the outer container +boundary. + +Dense multi-frame overview matching uses scientific head terms rather than +requiring exact bare labels. Descriptive variants such as `heavyweight image +encoder`, `prompt encoder`, and `lightweight mask decoder` are normalized to +their canonical overview nodes before the 13-entity SAM-style contract is +rebuilt. This prevents detailed implementation entities from changing the +intended Figure 1 topology. + +The same generic semantic plan can now bypass raster generation entirely: + +```powershell +rfs fast-framework-prompt ` + --paper paper.pdf ` + --out output\fast_result ` + --editable-ppt +``` + +This writes `figure_program.json` and `editable_composition.pptx`. Nodes are +native PowerPoint shapes with exact editable labels; multi-segment routes and +feedback loops are native connectors created behind the nodes. The compiler +also writes `semantic_ppt_report.json` and records every editable node and +connector in `composition_quality_report.json`. The same provenance contract is +used here, so data sources feed their declared modalities instead of appearing +as peer sensor inputs. + +Before semantic layout, contract completion prefers true early model-overview +figures over late scaling, timing, attention, or visualization figures. It also +removes isolated appendix-only diagram labels, innovations that cannot be +grounded to novelty evidence, duplicate aliases such as `MLP`/`MLP Head`, and +input shortcuts that bypass an explicit intermediate representation. + +The general production review and focused topology review execute concurrently +for every contract that declares visible relations, including linear pipelines. +Their default request timeout is 90 seconds with no automatic retry, preventing +two independent judges from creating a multi-minute sequential stall. Configure +`RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and +`RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT` when a slower provider requires a larger +budget. Existing candidates can be passed through `--repair-source` so review or +localized repair does not regenerate the correct parts of the image. + +The focused topology review is authoritative only for visible connector +geometry. When it cleanly passes every declared relation, connector-only false +positives from the general critic are suppressed and recorded in +`cross_critic_reconciliation`; paper semantics, entity roles, OCR, and +aesthetics remain under their original hard gates. If both critics identify the +same missing, bypassed, reversed, or invented connector, the failure remains +blocking. + +Image-2 edit requests default to a 120-second timeout controlled by +`RFS_IMAGE2_TIMEOUT`. Network timeouts stop the candidate immediately instead of +blindly regenerating the entire image. Retryable HTTP or provider failures may +retry up to `--image-retries`; the default is one. + +A full provider read/connect timeout also stops the multi-round repair loop with +`repair_stop_reason=provider_timeout`; another nominal repair round must not +repeat the same 120-second blind wait. Fast failures such as HTTP 429 remain +provider failures rather than semantic non-improvement and may use later repair +rounds. + +## Stability Audit + +Production stability requires repeated independent candidates rather than a +single selected best image. Run at least three candidates: + +```powershell +rfs paper-to-image ` + --paper paper.pdf ` + --out output/stability ` + --asset-mode image2 ` + --review-mode vlm ` + --candidates 3 ` + --repair-rounds 0 +``` + +The workflow writes `stability_report.json` with `seed_count`, +`production_pass_rate`, `mean_score`, `worst_case_score`, +`standard_deviation`, per-candidate scores, and failure-mode counts. Repair +candidates and reused `--repair-source` images are excluded because they are not +independent generations. `rfs benchmark score` consumes the generated report +automatically. + +The review records every `unexpected_label`, but only +`unsupported_unexpected_labels` are blocking. A visible detail outside +`required_labels` may remain when the scientific critic or paper-evidence critic +supports it; a label explicitly identified as invented or unsupported still +fails OCR/scientific production gates. Duplicate required labels remain errors, +while repeated non-required implementation details are recorded as +`noncritical_duplicate_labels` instead of causing a false failure. + +`training_flow` and `inference_flow` remain part of the paper contract, but they +are contextual by default. They become mandatory visual priorities only when an +item explicitly sets `required`, `must_appear_in_figure`, or +`visible_label_required`, or when the same content is listed in `must_show` or +`required_labels`. + +The focused topology judge also supports explicit containment semantics. If a +declared operation container visibly contains an evidence-supported repeatable +shared component, an arrow from that nested component to the target may satisfy +the container's outgoing relation. This is limited to repeatable labels inside +the declared source container; it cannot excuse missing cross-panel arrows, +shortcuts, reversed edges, or invented nodes. + +If a stability run is interrupted or one provider request fails, resume it in +place: + +```powershell +rfs paper-to-image ` + --paper paper.pdf ` + --out output/stability ` + --candidates 3 ` + --resume-candidates +``` + +Existing raw candidates are re-composed and re-reviewed with the current gates; +legacy composed-only candidates are re-reviewed without duplicate overlay, and +only missing candidates are generated. A three-seed stability run also permits +one bounded replacement for a provider-failed seed; successful candidates are +never regenerated by that replacement path. -The selected profile is rendered as `layout_blueprint.png`. The blueprint contains no reference text or reference-specific objects and is the only image supplied to Image2 edit for initial candidates. +When `--aspect-ratio auto` is used, the template keeps its internal normalized +geometry while the generation canvas uses the nearest native Image2 ratio +(`3:2`, `2:3`, or `1:1`). This avoids semantic cropping after generation. ## Production Gates Every Image2 candidate must pass: +- exact one-to-one visual-card detection and semantic geometry normalization, + with no ambiguous mapping and no semantic crop; - exact-label OCR with no missing, misspelled, duplicate, or copied reference labels; - scientific module and relation checks with no invention or reversal; -- template alignment score of at least `0.72` with no copied reference content; -- aesthetic score of at least `0.75`; +- for every declared visible relation, a second focused topology review that + verifies visible connector endpoints and arrowheads, rejects + bypasses, and writes `topology_critic_report.json`; +- template alignment score of at least `0.70` with no copied reference content; +- aesthetic score of at least `0.70`; - valid image resolution and blueprint aspect ratio. Scientific score must be at least `0.95`; OCR must be exact. If three candidates fail, the best candidate receives one localized Image2 edit repair. If the repair still fails, the command stops without writing `selected_image.png`. diff --git a/docs/rebuild-editable.md b/docs/rebuild-editable.md index aac01ac..2d9a42a 100644 --- a/docs/rebuild-editable.md +++ b/docs/rebuild-editable.md @@ -31,13 +31,13 @@ rfs rebuild-editable --reference input.png --out output\demo --asset-mode api -- Professional scripted rebuild: ```powershell -rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --repair-rounds 2 --export-preview +rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --asset-policy smart-api --repair-rounds 2 --export-preview ``` Compare against a known high-quality specialized rebuild: ```powershell -rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode crop --benchmark-out output\autofigure_architecture_ai_rebuild +rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode crop --asset-policy smart-api --benchmark-out output\autofigure_architecture_ai_rebuild ``` Offline professional smoke test: @@ -52,6 +52,7 @@ rfs rebuild-editable-pro --reference input.png --out output\demo_pro_placeholder --reference Input reference image. --out Output directory. --asset-mode api | crop | placeholder. Default: api. +--asset-policy legacy | smart-api. Default: smart-api. Use legacy only for old crop-fallback behavior. --asset-workers Parallel asset workers. Default: 4. --asset-retries Retries used by strict regeneration. Default: 1. --economy-mode Reuse accepted/passing assets. Enabled by default. @@ -77,6 +78,8 @@ rebuild-editable-pro Use the controlled professional Figure DSL workflow. --compile-only Recompile from professional_rebuild_script.dsl.json without VLM planning or asset API calls. ``` +`smart-api` disables final reference-crop assets. Local crops are still saved and passed as API reference context, but final PPT assets are API-generated, reused from another generated slot, or skipped when the region is primarily editable text. If `--asset-mode crop --asset-policy smart-api` is used for a low-cost dry run, the final visual asset is a placeholder and the report records `crop_disabled_by_smart_api_policy_placeholder`. + ## API Environment `--asset-mode api` uses the same image-generation route as the specialized rebuild scripts: @@ -112,7 +115,7 @@ $env:GEMINI_API_KEY=$env:API_KEY $env:GEMINI_GEN_IMG_URL='https://your-provider/v1beta/models/your-image-model:generateContent' ``` -If the API call fails for a slot, v1 falls back to the reference crop for that slot and records the failure in `asset_generation_report.json`. +With `--asset-policy legacy`, if the API call fails for a slot, v1 falls back to the reference crop for that slot and records the failure in `asset_generation_report.json`. With `--asset-policy smart-api`, crop fallback is disabled; API failure falls back to a placeholder and records `api_failed_placeholder_fallback`. ## Output Files @@ -131,6 +134,9 @@ asset_generation_specs.json asset_generation_report.json asset_economy_report.json asset_ratio_fit_report.json +asset_decision_report.json +text_asset_filter_report.json +api_asset_plan.json figure_program.json composition_quality_report.json rebuild_vlm_validation_report.json diff --git a/docs/workflow.md b/docs/workflow.md index 929790f..94d846e 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -29,20 +29,31 @@ The workflow is designed to keep scientific content, layout, image generation, a Offline engineering check: ```powershell -rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out D:\ResearchFigureStudio\output\offline_check --asset-mode placeholder --locator-mode heuristic --control-localizer-mode heuristic --arrow-style-mode reference --prompt-plan-mode heuristic --slot-count 36 --candidates-per-slot 3 --asset-review-mode heuristic --critic-mode heuristic --json -rfs validate --out D:\ResearchFigureStudio\output\offline_check --json +rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out .\output\offline_check --asset-mode placeholder --locator-mode heuristic --control-localizer-mode heuristic --arrow-style-mode reference --prompt-plan-mode heuristic --slot-count 36 --candidates-per-slot 3 --asset-review-mode heuristic --critic-mode heuristic --json +rfs validate --out .\output\offline_check --json ``` +Contract-driven engineering check (recommended after a fast/paper-to-image run): + +```powershell +rfs make-framework --paper "C:\path\paper.pdf" --reference ".\output\paper_to_image\run\selected_image.png" --paper-contract-dir ".\output\paper_to_image\run" --out .\output\contract_check --asset-mode placeholder --asset-workers 3 --locator-mode heuristic --control-localizer-mode off --arrow-style-mode reference --prompt-plan-mode heuristic --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --text-extractor-mode heuristic --ocr-engine off --no-export --json +``` + +This route does not call the legacy paper planner. `panel_graphs` instance IDs +remain distinct across panels, local relations are authoritative, exact labels +are editable PPT text, and any extra slots needed to reach 25 are reference +visual splits rather than summary-sentence fillers. + Small real image check: ```powershell -rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out D:\ResearchFigureStudio\output\real_small --asset-mode gemini --asset-workers 3 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --json +rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out .\output\real_small --asset-mode gemini --asset-workers 3 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --json ``` Full quality run: ```powershell -rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out D:\ResearchFigureStudio\output\full_quality --asset-mode gemini --asset-workers 4 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 36 --candidates-per-slot 3 --asset-review-mode vlm --critic-mode vlm --critic-iterations 1 --json +rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out .\output\full_quality --asset-mode gemini --asset-workers 4 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 36 --candidates-per-slot 3 --asset-review-mode vlm --critic-mode vlm --critic-iterations 1 --json ``` ## Parallel Asset Generation diff --git a/experiments/legacy_reference_rebuilds/README.md b/experiments/legacy_reference_rebuilds/README.md new file mode 100644 index 0000000..0eeed27 --- /dev/null +++ b/experiments/legacy_reference_rebuilds/README.md @@ -0,0 +1,5 @@ +# Legacy reference-specific rebuilds + +These scripts document the repository's early image-to-PPT experiments. They intentionally remain outside the reusable `rfs` package because they contain sample-specific assumptions and historical local paths. + +Use them only as implementation references for visual details. Do not import them from production workflows, plugin skills, tests, or packaging code. New reusable behavior belongs in `rfs/` with tests and path-independent inputs. diff --git a/scripts/rebuild_autofigure_architecture_ai_assets.py b/experiments/legacy_reference_rebuilds/rebuild_autofigure_architecture_ai_assets.py similarity index 100% rename from scripts/rebuild_autofigure_architecture_ai_assets.py rename to experiments/legacy_reference_rebuilds/rebuild_autofigure_architecture_ai_assets.py diff --git a/scripts/rebuild_c60_editable.py b/experiments/legacy_reference_rebuilds/rebuild_c60_editable.py similarity index 100% rename from scripts/rebuild_c60_editable.py rename to experiments/legacy_reference_rebuilds/rebuild_c60_editable.py diff --git a/scripts/rebuild_virtual_interview_pipeline_editable.py b/experiments/legacy_reference_rebuilds/rebuild_virtual_interview_pipeline_editable.py similarity index 100% rename from scripts/rebuild_virtual_interview_pipeline_editable.py rename to experiments/legacy_reference_rebuilds/rebuild_virtual_interview_pipeline_editable.py diff --git a/experiments/literature_review/README.md b/experiments/literature_review/README.md new file mode 100644 index 0000000..a0f8eb0 --- /dev/null +++ b/experiments/literature_review/README.md @@ -0,0 +1,8 @@ +# Literature review archive + +This directory contains early research notes and extracted paper text used to understand related diagram-generation systems. + +- `raw_text/` contains archived text extracts for internal literature analysis. +- `scripts/` contains historical extraction and translation helpers. Some scripts preserve machine-specific paths and are not production utilities. + +Nothing under this directory may be imported by `rfs/`, the Codex plugin skill, or automated production workflows. diff --git a/tmp/pdfs/diagrammergpt.txt b/experiments/literature_review/raw_text/diagrammergpt.txt similarity index 100% rename from tmp/pdfs/diagrammergpt.txt rename to experiments/literature_review/raw_text/diagrammergpt.txt diff --git a/tmp/pdfs/scidoc.txt b/experiments/literature_review/raw_text/scidoc.txt similarity index 100% rename from tmp/pdfs/scidoc.txt rename to experiments/literature_review/raw_text/scidoc.txt diff --git a/tmp/pdfs/scirex.txt b/experiments/literature_review/raw_text/scirex.txt similarity index 100% rename from tmp/pdfs/scirex.txt rename to experiments/literature_review/raw_text/scirex.txt diff --git a/tmp/pdfs/structured_ie.txt b/experiments/literature_review/raw_text/structured_ie.txt similarity index 100% rename from tmp/pdfs/structured_ie.txt rename to experiments/literature_review/raw_text/structured_ie.txt diff --git a/tmp/pdfs/extract_layout_text.py b/experiments/literature_review/scripts/extract_layout_text.py similarity index 100% rename from tmp/pdfs/extract_layout_text.py rename to experiments/literature_review/scripts/extract_layout_text.py diff --git a/tmp/pdfs/translate_papers.py b/experiments/literature_review/scripts/translate_papers.py similarity index 100% rename from tmp/pdfs/translate_papers.py rename to experiments/literature_review/scripts/translate_papers.py diff --git a/pyproject.toml b/pyproject.toml index a78f4df..cc3c97a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,16 +8,21 @@ version = "0.1.0" description = "PPTX-first research figure generation pipeline guided by papers and reference images." requires-python = ">=3.10" dependencies = [ + "numpy", "pillow", "python-pptx", "PyMuPDF", + "pypdf", + "pdfplumber", "requests", "opencv-python-headless", + "wordninja", "pywin32; platform_system == 'Windows'" ] [project.optional-dependencies] ocr = [ + "rapidocr-onnxruntime", "paddleocr", "easyocr" ] diff --git a/rfs/analysis/__init__.py b/rfs/analysis/__init__.py new file mode 100644 index 0000000..c0522fe --- /dev/null +++ b/rfs/analysis/__init__.py @@ -0,0 +1,10 @@ +"""Input and visual analysis entry points. + +Implementations remain in their existing modules while interfaces stabilize. +""" + +from ..layout_planner import plan_reference_layout +from ..paper_to_image.analyzer import parse_paper +from ..reference_text_extractor import extract_reference_text + +__all__ = ["extract_reference_text", "parse_paper", "plan_reference_layout"] diff --git a/rfs/arrow_router.py b/rfs/arrow_router.py index 99aef2b..91cd8c4 100644 --- a/rfs/arrow_router.py +++ b/rfs/arrow_router.py @@ -476,6 +476,18 @@ def style_and_route_arrows( panels = program.get("panels", []) if isinstance(program.get("panels"), list) else [] slots = program.get("slots", []) if isinstance(program.get("slots"), list) else [] + program_style = program.get("style") if isinstance(program.get("style"), dict) else {} + color_tokens = program_style.get("color_tokens", []) if isinstance(program_style.get("color_tokens"), list) else [] + default_style_token_id = next( + ( + str(item.get("token_id") or "") + for item in color_tokens + if isinstance(item, dict) and any(term in str(item.get("usage") or "").casefold() for term in ("arrow", "connector", "line")) + ), + "", + ) + if not default_style_token_id: + default_style_token_id = next((str(item.get("token_id") or "") for item in color_tokens if isinstance(item, dict) and item.get("token_id")), "") objects = {str(item.get("id")): item for item in panels + slots if isinstance(item, dict) and item.get("id")} slot_objects = {str(item.get("id")): item for item in slots if isinstance(item, dict) and item.get("id")} panel_ids = {str(panel.get("id")) for panel in panels if isinstance(panel, dict)} @@ -504,6 +516,8 @@ def style_and_route_arrows( arrow["target_id"] = target arrow["source"] = source arrow["target"] = target + if not str(arrow.get("style_token_id") or "").strip(): + arrow["style_token_id"] = default_style_token_id points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] locked = _reference_locked(arrow) route_meta: dict[str, Any] = {"routing_algorithm": "preserve_reference_path", "route_generation_status": "reference_locked"} diff --git a/rfs/asset_generator.py b/rfs/asset_generator.py index 51abb66..bca97af 100644 --- a/rfs/asset_generator.py +++ b/rfs/asset_generator.py @@ -12,6 +12,7 @@ from typing import Iterable import requests +import numpy as np from PIL import Image, ImageDraw, ImageFont from .utils import ensure_dir, write_json, write_text @@ -196,7 +197,7 @@ def build_slot_prompt(slot: dict, style: dict, candidate_index: int = 1) -> str: def _draw_placeholder(slot: dict, path: Path, style: dict, candidate_index: int = 1) -> None: - width, height = _target_size(slot["target_canvas_ratio"]) + width, height = _target_size(slot["target_canvas_ratio"], max_side=512) palette = style.get("palette", ["#EAF5FF", "#DFF7EF", "#8EC9E8", "#66C6A4", "#163B4D"]) bg = palette[(candidate_index - 1) % 2] line = palette[4] if len(palette) > 4 else "#163B4D" @@ -404,22 +405,17 @@ def _content_bbox(rgba: Image.Image) -> tuple[list[int], float, float]: width, height = rgba.size pixels = rgba.load() background = _median_corner_color(pixels, width, height) - xs: list[int] = [] - ys: list[int] = [] - color_threshold = 30.0 - alpha_threshold = 12 - for y in range(height): - for x in range(width): - r, g, b, a = pixels[x, y] - if a <= alpha_threshold: - continue - if a < 245 or _color_distance((r, g, b), background) > color_threshold: - xs.append(x) - ys.append(y) - if not xs or not ys: + rgba_array = np.asarray(rgba, dtype=np.int16) + rgb = rgba_array[:, :, :3] + alpha = rgba_array[:, :, 3] + background_array = np.asarray(background, dtype=np.float32) + color_distance_sq = np.sum((rgb.astype(np.float32) - background_array) ** 2, axis=2) + mask = (alpha > 12) & ((alpha < 245) | (color_distance_sq > 30.0 ** 2)) + ys, xs = np.nonzero(mask) + if xs.size == 0 or ys.size == 0: return [0, 0, width, height], 0.0, 100.0 - left, right = min(xs), max(xs) - top, bottom = min(ys), max(ys) + left, right = int(xs.min()), int(xs.max()) + top, bottom = int(ys.min()), int(ys.max()) bbox_w = max(1, right - left + 1) bbox_h = max(1, bottom - top + 1) content_fill = bbox_w * bbox_h / float(width * height) * 100.0 @@ -524,40 +520,7 @@ def _estimate_quality(path: Path, slot: dict, candidate_index: int, source: str) with Image.open(path) as image: rgba = image.convert("RGBA") width, height = rgba.size - pixels = rgba.load() - background = _median_corner_color(pixels, width, height) - - xs: list[int] = [] - ys: list[int] = [] - color_threshold = 30.0 - alpha_threshold = 12 - for y in range(height): - for x in range(width): - r, g, b, a = pixels[x, y] - if a <= alpha_threshold: - continue - if a < 245 or _color_distance((r, g, b), background) > color_threshold: - xs.append(x) - ys.append(y) - - if not xs or not ys: - content_fill = 0.0 - empty_margin = 100.0 - bbox = [0, 0, width, height] - else: - left, right = min(xs), max(xs) - top, bottom = min(ys), max(ys) - bbox_w = max(1, right - left + 1) - bbox_h = max(1, bottom - top + 1) - bbox = [left, top, right + 1, bottom + 1] - content_fill = bbox_w * bbox_h / float(width * height) * 100.0 - margins = ( - left / width * 100.0, - (width - right - 1) / width * 100.0, - top / height * 100.0, - (height - bottom - 1) / height * 100.0, - ) - empty_margin = max(margins) + bbox, content_fill, empty_margin = _content_bbox(rgba) image_ratio = width / max(height, 1) target_ratio = _ratio_value(slot["target_canvas_ratio"]) @@ -760,7 +723,6 @@ def _generate_one_candidate(slot: dict, style: dict, path: Path, asset_mode: str support_color = str(style["reference_palette"][0]) if asset_mode == "placeholder": _draw_placeholder(slot, path, style, candidate_index=candidate_index) - _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) return "explicit_placeholder_candidate" if asset_mode == "gemini": diff --git a/rfs/cli.py b/rfs/cli.py index 7839330..f3ab372 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -1,20 +1,26 @@ from __future__ import annotations import argparse +import importlib.util import json import os +import shutil +import subprocess import sys from pathlib import Path from . import __version__ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable -from .paper_to_image import run_paper_to_image +from .evaluation import audit_paper_image_run, fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, run_pdf_extraction_stress_suite, score_benchmark_case, validate_benchmark_case +from .paper_to_image import inspect_paper, run_fast_framework_prompt, run_paper_to_image +from .workflows import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro from .professional_repair import vlm_professional_repair_adapter from .presentations_qa import run_presentations_qa from .rebuild_vlm_adapters import build_rebuild_vlm_adapters from .rebuild_eval import evaluate_rebuild_vlm +from .reference_text_extractor import _rapidocr_detector_limit from .utils import env_present, mask_secret from .validator import validate_output from .workflow import make_framework @@ -24,9 +30,24 @@ def _json_print(data: dict) -> None: print(json.dumps(data, indent=2, ensure_ascii=False)) +def _probe_executable(name: str, version_arg: str = "-v") -> dict: + path = shutil.which(name) + if not path: + return {"available": False, "path": None, "error": "not found on PATH"} + try: + completed = subprocess.run([path, version_arg], capture_output=True, text=True, timeout=5, check=False) + except Exception as exc: + return {"available": False, "path": path, "error": str(exc)} + output = "\n".join(part.strip() for part in (completed.stdout, completed.stderr) if part and part.strip()) + result = {"available": completed.returncode == 0, "path": path, "returncode": completed.returncode} + if completed.returncode != 0: + result["error"] = output.splitlines()[0] if output else "version probe failed" + return result + + def _doctor() -> dict: deps = {} - for name, module in [("Pillow", "PIL"), ("python-pptx", "pptx"), ("PyMuPDF", "fitz"), ("requests", "requests"), ("opencv-python-headless", "cv2")]: + for name, module in [("Pillow", "PIL"), ("python-pptx", "pptx"), ("PyMuPDF", "fitz"), ("pypdf", "pypdf"), ("pdfplumber", "pdfplumber"), ("requests", "requests"), ("opencv-python-headless", "cv2")]: try: __import__(module) deps[name] = {"available": True} @@ -59,6 +80,34 @@ def _doctor() -> dict: "RFS_IMAGE_EDIT_URL": {"present": env_present("RFS_IMAGE_EDIT_URL"), "value": os.getenv("RFS_IMAGE_EDIT_URL") if env_present("RFS_IMAGE_EDIT_URL") else None}, "MODEL_VLM": {"present": env_present("MODEL_VLM"), "value": os.getenv("MODEL_VLM") if env_present("MODEL_VLM") else None}, } + easyocr_model_dir = Path(os.getenv("EASYOCR_MODULE_PATH", "").strip() or (Path.home() / ".EasyOCR" / "model")) + easyocr_files = {path.name: path.stat().st_size for path in easyocr_model_dir.glob("*.pth")} if easyocr_model_dir.exists() else {} + easyocr_models = { + "model_dir": str(easyocr_model_dir), + "detector_ready": "craft_mlt_25k.pth" in easyocr_files, + "en_ready": "english_g2.pth" in easyocr_files, + "ch_ready": "zh_sim_g2.pth" in easyocr_files, + "en_ch_ready": "craft_mlt_25k.pth" in easyocr_files and "zh_sim_g2.pth" in easyocr_files, + "allow_download": str(os.getenv("RFS_OCR_ALLOW_DOWNLOAD") or "").strip().casefold() in {"1", "true", "yes", "on"}, + "files": easyocr_files, + } + optional_pdf = { + "pdftotext": _probe_executable("pdftotext"), + "pdftoppm": _probe_executable("pdftoppm"), + "easyocr": {"available": importlib.util.find_spec("easyocr") is not None, "models": easyocr_models}, + "rapidocr": { + "available": importlib.util.find_spec("rapidocr_onnxruntime") is not None, + "runtime": "onnxruntime", + "worker_policy": "up to 4 page workers on 8+ CPUs, 2 on 4-7 CPUs, 1 below 4; deadline runs use killable worker processes; 2 intra-op threads for a single page", + "worker_override": os.getenv("RFS_OCR_WORKERS"), + "thread_override": os.getenv("RFS_RAPIDOCR_THREADS"), + "detector_limit_override": os.getenv("RFS_RAPIDOCR_DET_LIMIT"), + "effective_detector_limit": _rapidocr_detector_limit(), + }, + "paddleocr": {"available": importlib.util.find_spec("paddleocr") is not None}, + "wordninja": {"available": importlib.util.find_spec("wordninja") is not None, "purpose": "conservative English spacing repair for OCR lines"}, + "fast_contract_cache": {"available": True, "path": str(Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")))}, + } ok = all(item["available"] for item in deps.values()) return { "summary": "ResearchFigureStudio doctor report.", @@ -66,6 +115,7 @@ def _doctor() -> dict: "version": __version__, "python": sys.executable, "dependencies": deps, + "pdf_tools": optional_pdf, "powerpoint": powerpnt, "auth": auth, "notes": [ @@ -79,6 +129,7 @@ def _doctor() -> dict: "Use rfs presentations-qa as an optional inspection pass; RFS remains the authoritative PPTX compiler.", "Use rfs coevolve-image to run whole-image Creator Agent and Online/Frozen Judge refinement before PPTX conversion.", "Use rfs paper-to-image for evidence-grounded paper summarization, whole-image prompt compilation, candidate generation, and selected_image.png without PPTX.", + "Use rfs fast-framework-prompt for the one-call, cached paper-to-contract fast path; RFS_FAST_FRAMEWORK_MODEL defaults to gemini-2.5-flash.", "Production paper-to-image uses a content-free template blueprint and requires an Image2 edit endpoint; placeholder output is engineering-only.", ], } @@ -92,10 +143,37 @@ def build_parser() -> argparse.ArgumentParser: doctor = sub.add_parser("doctor", help="Check dependencies, PowerPoint, and auth env vars.") doctor.add_argument("--json", action="store_true", help="Emit JSON.") + inspect_pdf = sub.add_parser("inspect-pdf", help="Extract and diagnose a paper without calling a model.") + inspect_pdf.add_argument("--paper", required=True, help="Paper PDF or supported document path.") + inspect_pdf.add_argument("--out", required=True, help="Output directory for the document model and extraction report.") + inspect_pdf.add_argument("--deadline", type=int, default=180, help="Soft processing deadline in seconds. Default: 180.") + inspect_pdf.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") + inspect_pdf.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + inspect_pdf.add_argument("--json", action="store_true", help="Emit JSON.") + + fast_prompt = sub.add_parser("fast-framework-prompt", help="Compile a paper-grounded framework prompt without generating images.") + fast_prompt.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + fast_prompt.add_argument("--out", required=True, help="Output directory for evidence, semantic contract, prompt, and overlay specification.") + fast_prompt.add_argument("--deadline", type=int, default=180, help="Soft end-to-end deadline in seconds. Default: 180.") + fast_prompt.add_argument("--preferences", help="Optional JSON style and output preferences.") + fast_prompt.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm") + fast_prompt.add_argument("--planner-model") + fast_prompt.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") + fast_prompt.add_argument("--aspect-ratio", default="16:9") + fast_prompt.add_argument("--language", default="English") + fast_prompt.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") + fast_prompt.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + fast_prompt.add_argument("--editable-ppt", action="store_true", help="Also compile the semantic contract into native editable PowerPoint shapes, text, and connectors.") + fast_prompt.add_argument("--verify-ppt-roundtrip", action="store_true", help="Compile the editable PPTX, render it through PowerPoint, and write ppt_roundtrip_report.json.") + fast_prompt.add_argument("--json", action="store_true", help="Emit JSON.") + make = sub.add_parser("make-framework", help="Create a paper-grounded, reference-guided editable PPTX framework figure.") make.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") make.add_argument("--reference", required=True, help="User-provided visual reference image path.") make.add_argument("--out", required=True, help="Output directory.") + contract_source = make.add_mutually_exclusive_group() + contract_source.add_argument("--paper-contract-dir", help="Fast paper-to-image/fast-framework output directory containing figure_specification.json. Bypasses the legacy paper planner.") + contract_source.add_argument("--semantic-contract", help="Semantic contract JSON or figure_specification.json. Bypasses the legacy paper planner.") make.add_argument("--profile", default="ai-ml-paper", help="Figure profile. Default: ai-ml-paper.") make.add_argument("--slot-count", type=int, default=36, help="Target slot count, clamped to 25-50. Default: 36.") make.add_argument("--slot-source", choices=["paper", "reference-primary"], default="reference-primary", help="Slot content source. Default reference-primary makes the reference figure drive visual objects, layout, color, and flow logic.") @@ -116,7 +194,7 @@ def build_parser() -> argparse.ArgumentParser: make.add_argument("--critic-model", help="Optional VLM model for asset review and final critic. Defaults to RFS_CRITIC_MODEL/MODEL_VLM.") make.add_argument("--critic-iterations", type=int, default=0, help="VLM layout correction iterations, clamped to 0-3. Default: 0.") make.add_argument("--text-extractor-mode", choices=["heuristic", "ocr"], default="ocr", help="Editable text layer source. Default ocr uses local OCR when available and falls back to heuristic.") - make.add_argument("--ocr-engine", choices=["paddle", "easyocr", "off"], default="paddle", help="Local OCR engine for reference text extraction. Default: paddle.") + make.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="Local OCR engine for reference text extraction. Default: paddle.") make.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") make.add_argument("--presentations-qa", action="store_true", help="Run optional Presentations plugin import/render/layout QA after export. This never mutates the PPTX.") make.add_argument("--presentations-workspace", help="Optional workspace for Presentations QA scratch artifacts.") @@ -124,7 +202,7 @@ def build_parser() -> argparse.ArgumentParser: make.add_argument("--no-export", action="store_true", help="Skip PDF/PNG export and only create PPTX/artifacts.") make.add_argument("--json", action="store_true", help="Emit JSON.") - paper_image = sub.add_parser("paper-to-image", help="Summarize a paper, plan a scientific framework figure, and generate raster image candidates without PPTX.") + paper_image = sub.add_parser("paper-to-image", help="Summarize a paper, plan a scientific framework figure, and generate reviewed raster candidates with an optional editable PPT overlay.") paper_image.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") paper_image.add_argument("--out", required=True, help="Output directory for review, templates, prompts, candidates, and production-only selected_image.png.") paper_image.add_argument("--preferences", help="Optional JSON file containing style and output preferences.") @@ -133,29 +211,73 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Paper summarization and figure planning mode. Default: vlm with heuristic fallback.") paper_image.add_argument("--planner-model", help="Optional planning VLM. Defaults to RFS_PAPER_TO_IMAGE_MODEL/RFS_PAPER_PLANNER_MODEL/MODEL_VLM.") paper_image.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto", help="Universal review profile plus optional domain extension. Default: auto.") - paper_image.add_argument("--template", choices=["auto", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") + paper_image.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") paper_image.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Whole-image generation backend. Placeholder is for offline validation only.") paper_image.add_argument("--candidates", type=int, default=3, help="Image candidate count, clamped to 1-4. Default: 3.") paper_image.add_argument("--aspect-ratio", default="auto", help="Target image aspect ratio or auto to inherit the selected template. Default: auto.") paper_image.add_argument("--language", default="English", help="Visible label language. Default: English.") paper_image.add_argument("--image-model", help="Optional image model. Defaults to RFS_IMAGE_MODEL/IMAGE_MODEL.") - paper_image.add_argument("--image-retries", type=int, default=2, help="Retries per image candidate, clamped to 0-5. Default: 2.") + paper_image.add_argument("--image-retries", type=int, default=1, help="Retries per image candidate for retryable HTTP/provider errors, clamped to 0-5. Network timeouts are not blindly retried. Default: 1.") paper_image.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Candidate review mode. Production Image2 requires VLM review. Default: vlm.") paper_image.add_argument("--review-model", help="Optional VLM candidate-review model. Defaults to RFS_PAPER_TO_IMAGE_REVIEW_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") - paper_image.add_argument("--repair-rounds", type=int, default=1, help="Localized Image2 edit repair rounds, clamped to 0-1. Default: 1.") - paper_image.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") + paper_image.add_argument("--repair-rounds", type=int, default=3, help="Localized Image2 edit repair rounds, clamped to 0-4 with no-regression acceptance. Default: 3.") + paper_image.add_argument("--repair-source", help="Reuse an existing failed Image2 candidate as the starting point and run review/repair without generating fresh initial candidates.") + paper_image.add_argument("--resume-candidates", action="store_true", help="Reuse existing candidate_XX images in --out, re-review them with current gates, and generate only missing candidates.") + paper_image.add_argument("--editable-overlay-ppt", action="store_true", help="Place the selected geometry-normalized Image2 visual substrate beneath native editable PPT labels and connectors. Requires deterministic overlay mode and a raw candidate that passes substrate geometry normalization.") + paper_image.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") paper_image.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") paper_image.add_argument("--json", action="store_true", help="Emit JSON.") + paper_editable = sub.add_parser("paper-to-editable", help="Generate a paper-grounded visual reference and rebuild it as an editable PowerPoint figure.") + paper_editable.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + paper_editable.add_argument("--out", required=True, help="Output directory for paper analysis, image candidates, semantic contracts, and editable PPTX.") + paper_editable.add_argument("--preferences", help="Optional JSON file containing style and output preferences.") + paper_editable.add_argument("--positive-reference", action="append", default=[], help="Optional positive visual reference image. Repeat for multiple files.") + paper_editable.add_argument("--negative-reference", action="append", default=[], help="Optional negative visual reference image. Repeat for multiple files.") + paper_editable.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm") + paper_editable.add_argument("--planner-model") + paper_editable.add_argument("--image-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2") + paper_editable.add_argument("--image-candidates", type=int, default=3) + paper_editable.add_argument("--aspect-ratio", default="auto") + paper_editable.add_argument("--language", default="English") + paper_editable.add_argument("--image-model") + paper_editable.add_argument("--image-retries", type=int, default=2) + paper_editable.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm") + paper_editable.add_argument("--review-model") + paper_editable.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") + paper_editable.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") + paper_editable.add_argument("--repair-rounds", type=int, default=3) + paper_editable.add_argument("--editable-route", choices=["rebuild", "framework"], default="rebuild", help="Editable compiler. Use framework for the authoritative 25-50 slot paper-contract route; rebuild preserves the legacy image reconstruction route.") + paper_editable.add_argument("--framework-slot-count", type=int, default=25, help="Non-arrow slot count for --editable-route framework. Default: 25.") + paper_editable.add_argument("--framework-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Slot asset backend for --editable-route framework.") + paper_editable.add_argument("--framework-candidates-per-slot", type=int, default=1, help="Asset candidates per slot for --editable-route framework. Default: 1.") + paper_editable.add_argument("--framework-asset-workers", type=int, default=3, help="Parallel slot generation workers for --editable-route framework. Default: 3.") + paper_editable.add_argument("--rebuild-asset-mode", choices=["api", "crop", "placeholder"], default="api") + paper_editable.add_argument("--rebuild-asset-policy", choices=["legacy", "smart-api"], default="smart-api") + paper_editable.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid") + paper_editable.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid") + paper_editable.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr") + paper_editable.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm") + paper_editable.add_argument("--allow-engineering-preview", action="store_true", help="Allow offline/non-production image previews to exercise the editable pipeline. Never enabled by default.") + paper_editable.add_argument("--no-export-preview", action="store_true") + paper_editable.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto") + paper_editable.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + paper_editable.add_argument("--json", action="store_true", help="Emit JSON.") + rebuild = sub.add_parser("rebuild-editable", help="Rebuild a reference image into a reusable editable PowerPoint composition.") rebuild.add_argument("--reference", required=True, help="Reference image path.") rebuild.add_argument("--out", required=True, help="Output directory.") rebuild.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="api", help="Slot asset source. Default api uses GEMINI_GEN_IMG_URL.") + rebuild.add_argument("--asset-policy", choices=["legacy", "smart-api"], default="smart-api", help="Slot asset decision policy. Default smart-api filters text slots, reuses duplicate assets, and disables final crop assets.") rebuild.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") rebuild.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") rebuild.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") rebuild.add_argument("--no-economy-mode", dest="economy_mode", action="store_false", help="Disable economy reuse decisions.") rebuild.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") + rebuild.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Whole-reference design planning mode. Default: vlm with explicit fallback reporting.") + rebuild.add_argument("--design-plan-model", help="Optional VLM for whole-reference design planning.") + rebuild.add_argument("--text-grouping-mode", choices=["off", "heuristic", "vlm", "hybrid"], default="heuristic", help="Group OCR lines into editable paragraphs. Default: heuristic.") + rebuild.add_argument("--text-grouping-model", help="Optional VLM for OCR text grouping.") rebuild.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Panel/card/slot layout extraction mode. Default: hybrid.") rebuild.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Arrow/control extraction mode. Default: hybrid.") rebuild.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") @@ -163,7 +285,7 @@ def build_parser() -> argparse.ArgumentParser: rebuild.add_argument("--strict-asset-regeneration", action="store_true", help="Use stricter asset thresholds and --asset-retries for high-cost regeneration.") rebuild.add_argument("--skip-analysis", action="store_true", help="Reuse existing JSON contracts in --out instead of re-running layout/control/semantic analysis.") rebuild.add_argument("--compile-only", action="store_true", help="Compile editable_composition.pptx from existing JSON contracts and assets without regenerating analysis or assets.") - rebuild.add_argument("--ocr-engine", choices=["paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") + rebuild.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") rebuild.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") rebuild.add_argument("--json", action="store_true", help="Emit JSON.") @@ -179,6 +301,7 @@ def build_parser() -> argparse.ArgumentParser: rebuild_pro.add_argument("--reference", required=True, help="Reference image path.") rebuild_pro.add_argument("--out", required=True, help="Output directory.") rebuild_pro.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="api", help="Slot asset source. Default api uses GEMINI_GEN_IMG_URL.") + rebuild_pro.add_argument("--asset-policy", choices=["legacy", "smart-api"], default="smart-api", help="Slot asset decision policy. Default smart-api disables final crop assets, filters text slots, and reuses duplicate API assets.") rebuild_pro.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") rebuild_pro.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") rebuild_pro.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") @@ -193,7 +316,7 @@ def build_parser() -> argparse.ArgumentParser: rebuild_pro.add_argument("--repair-rounds", type=int, default=2, help="Preview repair rounds to record/run. V1 records conservative no-mutation repair reports by default.") rebuild_pro.add_argument("--repair-mode", choices=["report", "vlm"], default="report", help="Professional repair mode. report records rounds without mutation; vlm applies controlled DSL patches.") rebuild_pro.add_argument("--benchmark-out", help="Optional specialized rebuild output directory to compare against in professional_gap_report.json.") - rebuild_pro.add_argument("--ocr-engine", choices=["paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") + rebuild_pro.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") rebuild_pro.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") rebuild_pro.add_argument("--json", action="store_true", help="Emit JSON.") @@ -213,6 +336,24 @@ def build_parser() -> argparse.ArgumentParser: coevolution_report.add_argument("--run", required=True, help="Existing co-evolution output directory.") coevolution_report.add_argument("--json", action="store_true", help="Emit JSON.") + benchmark = sub.add_parser("benchmark", help="List, validate, run, or score ResearchFigureStudio benchmark cases.") + benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "fast-suite", "pdf-suite", "run", "score", "audit"]) + benchmark.add_argument("--suite", choices=["paper-to-image", "image-to-ppt"], help="Optional suite filter for list.") + benchmark.add_argument("--root", default="benchmarks", help="Benchmark root for list. Default: benchmarks.") + benchmark.add_argument("--case", help="Benchmark case directory for validate, fetch, run, or score.") + benchmark.add_argument("--case-id", action="append", default=[], help="Optional case id filter for fast-suite. Repeat to select multiple cases.") + benchmark.add_argument("--run", dest="run_dir", help="Existing workflow output directory for score.") + benchmark.add_argument("--out", help="Output directory for run, or optional score report destination.") + benchmark.add_argument("--force", action="store_true", help="Re-fetch an existing local benchmark paper input.") + benchmark.add_argument("--deadline", type=int, default=180, help="Soft deadline for benchmark fast. Default: 180.") + benchmark.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Planner mode for benchmark fast.") + benchmark.add_argument("--planner-model", help="Optional model for benchmark fast.") + benchmark.add_argument("--review-model", help="Optional frozen VLM model for benchmark audit.") + benchmark.add_argument("--candidate-image", help="Optional explicit image to re-audit inside an existing paper-to-image run.") + benchmark.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for fast or pdf-suite benchmarks.") + benchmark.add_argument("--rasterize-dpi", type=int, help="For benchmark fast/fast-suite, convert local PDF inputs to image-only scanned PDFs at this DPI before extraction.") + benchmark.add_argument("--json", action="store_true", help="Emit JSON.") + validate = sub.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") validate.add_argument("--out", required=True, help="Output directory to validate.") validate.add_argument("--json", action="store_true", help="Emit JSON.") @@ -230,7 +371,7 @@ def build_parser() -> argparse.ArgumentParser: def _print_human(data: dict) -> None: if "ok" in data: print(f"ok: {data['ok']}") - for key in ["out_dir", "selected_image", "engineering_preview", "candidate_count", "selected_candidate_id", "selected_passed_all_checks", "planner_mode", "planner_model", "paper_review_mode", "domain_profile", "template_id", "review_mode", "approved_image", "thresholds_met", "stop_reason", "rounds_completed", "online_judge_model", "frozen_judge_model", "weak_judge_isolation", "pptx", "pdf", "png", "preview", "asset_count", "slot_count", "slot_source", "asset_mode", "asset_workers", "asset_retries", "economy_mode", "api_requests_attempted", "text_count", "connector_count", "text_mode", "layout_mode", "control_mode", "professional_mode", "repair_rounds", "planner_status", "compile_only", "candidates_per_slot", "asset_review_mode", "locator_mode", "control_localizer_mode", "arrow_style_mode", "prompt_plan_mode", "prompt_plan_workers", "complexity_profile", "critic_mode", "critic_iterations", "text_extractor_mode", "ocr_engine", "ocr_lang"]: + for key in ["out_dir", "selected_image", "engineering_preview", "candidate_count", "selected_candidate_id", "selected_passed_all_checks", "planner_mode", "planner_model", "paper_review_mode", "domain_profile", "template_id", "review_mode", "approved_image", "thresholds_met", "stop_reason", "rounds_completed", "online_judge_model", "frozen_judge_model", "weak_judge_isolation", "pptx", "pdf", "png", "preview", "asset_count", "slot_count", "slot_source", "asset_mode", "asset_policy", "asset_workers", "asset_retries", "economy_mode", "api_requests_attempted", "text_count", "connector_count", "text_mode", "layout_mode", "control_mode", "professional_mode", "repair_rounds", "planner_status", "compile_only", "candidates_per_slot", "asset_review_mode", "locator_mode", "control_localizer_mode", "arrow_style_mode", "prompt_plan_mode", "prompt_plan_workers", "complexity_profile", "critic_mode", "critic_iterations", "text_extractor_mode", "ocr_engine", "ocr_lang"]: if key in data: print(f"{key}: {data[key]}") if data.get("presentations_qa"): @@ -264,6 +405,30 @@ def main(argv: list[str] | None = None) -> int: try: if args.command == "doctor": result = _doctor() + elif args.command == "inspect-pdf": + result = inspect_paper( + paper=args.paper, + out=args.out, + deadline_seconds=args.deadline, + ocr_engine=args.ocr_engine, + ocr_lang=args.ocr_lang, + ) + elif args.command == "fast-framework-prompt": + result = run_fast_framework_prompt( + paper=args.paper, + out=args.out, + deadline_seconds=args.deadline, + planner_mode=args.planner_mode, + planner_model=args.planner_model, + ocr_engine=args.ocr_engine, + ocr_lang=args.ocr_lang, + preferences_path=args.preferences, + aspect_ratio=args.aspect_ratio, + language=args.language, + domain_profile=args.domain_profile, + editable_ppt=args.editable_ppt, + verify_ppt_roundtrip=args.verify_ppt_roundtrip, + ) elif args.command == "make-framework": result = make_framework( paper=args.paper, @@ -295,6 +460,8 @@ def main(argv: list[str] | None = None) -> int: presentations_workspace=args.presentations_workspace, presentations_scale=args.presentations_scale, export=not args.no_export, + semantic_contract=args.semantic_contract, + paper_contract_dir=args.paper_contract_dir, ) elif args.command == "paper-to-image": result = run_paper_to_image( @@ -316,8 +483,47 @@ def main(argv: list[str] | None = None) -> int: domain_profile=args.domain_profile, template=args.template, repair_rounds=args.repair_rounds, + repair_source=args.repair_source, + resume_candidates=args.resume_candidates, + editable_overlay_ppt=args.editable_overlay_ppt, + ocr_engine=args.ocr_engine, + ocr_lang=args.ocr_lang, + ) + elif args.command == "paper-to-editable": + result = run_paper_to_editable( + paper=args.paper, + out=args.out, + preferences_path=args.preferences, + positive_references=args.positive_reference, + negative_references=args.negative_reference, + planner_mode=args.planner_mode, + planner_model=args.planner_model, + image_asset_mode=args.image_asset_mode, + image_candidates=args.image_candidates, + aspect_ratio=args.aspect_ratio, + language=args.language, + image_model=args.image_model, + image_retries=args.image_retries, + review_mode=args.review_mode, + review_model=args.review_model, + domain_profile=args.domain_profile, + template=args.template, + repair_rounds=args.repair_rounds, + rebuild_asset_mode=args.rebuild_asset_mode, + rebuild_asset_policy=args.rebuild_asset_policy, + layout_mode=args.layout_mode, + control_mode=args.control_mode, + text_mode=args.text_mode, + design_plan_mode=args.design_plan_mode, + export_preview=not args.no_export_preview, + allow_engineering_preview=args.allow_engineering_preview, ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, + editable_route=args.editable_route, + framework_slot_count=args.framework_slot_count, + framework_asset_mode=args.framework_asset_mode, + framework_candidates_per_slot=args.framework_candidates_per_slot, + framework_asset_workers=args.framework_asset_workers, ) elif args.command == "rebuild-editable": rebuild_adapters = build_rebuild_vlm_adapters(args.out) @@ -341,6 +547,12 @@ def main(argv: list[str] | None = None) -> int: vlm_layout_adapter=rebuild_adapters["layout"] if args.layout_mode in {"vlm", "hybrid"} else None, control_adapter=rebuild_adapters["control"] if args.control_mode in {"vlm", "hybrid"} else None, semantic_adapter=rebuild_adapters["semantic"] if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else None, + asset_policy=args.asset_policy, + design_plan_mode=args.design_plan_mode, + design_plan_model=args.design_plan_model, + design_adapter=rebuild_adapters["design"] if args.design_plan_mode == "vlm" else None, + text_grouping_mode=args.text_grouping_mode, + text_grouping_model=args.text_grouping_model, ) elif args.command == "rebuild-editable-eval": result = evaluate_rebuild_vlm( @@ -374,6 +586,7 @@ def main(argv: list[str] | None = None) -> int: semantic_adapter=rebuild_adapters["semantic"] if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else None, repair_adapter=vlm_professional_repair_adapter if args.repair_mode == "vlm" else None, benchmark_out=args.benchmark_out, + asset_policy=args.asset_policy, ) elif args.command == "coevolve-image": result = run_image_coevolution( @@ -389,6 +602,41 @@ def main(argv: list[str] | None = None) -> int: ) elif args.command == "coevolution-report": result = analyze_coevolution_run(args.run) + elif args.command == "benchmark": + if args.benchmark_action == "list": + result = list_benchmark_cases(args.root, suite=args.suite) + elif args.benchmark_action == "validate": + if not args.case: + parser.error("benchmark validate requires --case") + result = validate_benchmark_case(args.case) + elif args.benchmark_action == "fetch": + if not args.case: + parser.error("benchmark fetch requires --case") + result = fetch_benchmark_case(args.case, force=args.force) + elif args.benchmark_action == "run": + if not args.case or not args.out: + parser.error("benchmark run requires --case and --out") + result = run_benchmark_case(args.case, args.out) + elif args.benchmark_action == "fast": + if not args.case or not args.out: + parser.error("benchmark fast requires --case and --out") + result = run_fast_benchmark_case(args.case, args.out, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine, rasterize_pdf_dpi=args.rasterize_dpi) + elif args.benchmark_action == "fast-suite": + if not args.out: + parser.error("benchmark fast-suite requires --out") + result = run_fast_benchmark_suite(args.root, args.out, case_ids=args.case_id, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine, rasterize_pdf_dpi=args.rasterize_dpi) + elif args.benchmark_action == "pdf-suite": + if not args.out: + parser.error("benchmark pdf-suite requires --out") + result = run_pdf_extraction_stress_suite(args.out, ocr_engine=args.ocr_engine) + elif args.benchmark_action == "audit": + if not args.run_dir: + parser.error("benchmark audit requires --run; --case is optional but enables benchmark scoring") + result = audit_paper_image_run(args.run_dir, case_dir=args.case, candidate_image=args.candidate_image, review_model=args.review_model, ocr_engine=args.ocr_engine) + else: + if not args.case or not args.run_dir: + parser.error("benchmark score requires --case and --run") + result = score_benchmark_case(args.case, args.run_dir, args.out) elif args.command == "validate": result = validate_output(args.out) elif args.command == "presentations-qa": diff --git a/rfs/composition/__init__.py b/rfs/composition/__init__.py new file mode 100644 index 0000000..0d26f49 --- /dev/null +++ b/rfs/composition/__init__.py @@ -0,0 +1,6 @@ +"""Deterministic editable-figure composition and preview rendering.""" + +from .pptx import compile_ppt +from .preview import render_rebuild_preview + +__all__ = ["compile_ppt", "render_rebuild_preview"] diff --git a/rfs/composition/pptx.py b/rfs/composition/pptx.py new file mode 100644 index 0000000..f14c435 --- /dev/null +++ b/rfs/composition/pptx.py @@ -0,0 +1,687 @@ +from __future__ import annotations + +from pathlib import Path + +from PIL import Image +from pptx import Presentation +from pptx.dml.color import RGBColor +from pptx.enum.dml import MSO_LINE_DASH_STYLE +from pptx.enum.shapes import MSO_CONNECTOR, MSO_SHAPE +from pptx.enum.text import PP_ALIGN, MSO_ANCHOR +from pptx.oxml import parse_xml +from pptx.util import Inches, Pt + +from ..utils import pct_to_inches, write_json + + +def _rgb(hex_color: str) -> RGBColor: + value = hex_color.strip().lstrip("#") or "000000" + return RGBColor(int(value[:2], 16), int(value[2:4], 16), int(value[4:6], 16)) + + +def _apply_arrow(connector, size: str = "sm") -> None: + ln = connector.line._get_or_add_ln() + ln.append(parse_xml(f'')) + + +def _apply_line_cap(connector, cap: str = "round") -> None: + value = {"round": "rnd", "square": "sq"}.get(str(cap).lower()) + if not value: + return + ln = connector.line._get_or_add_ln() + ln.set("cap", value) + + +def _connector_type_for_route(route_style: str): + if str(route_style).lower() in {"soft_curve", "dashed_loop", "dashed_spline_like"}: + return MSO_CONNECTOR.CURVE + return MSO_CONNECTOR.STRAIGHT + + +def _set_text(shape, text: str, font_size: float = 10, bold: bool = False, color: str = "#163B4D", align=PP_ALIGN.CENTER, font_family: str | None = None) -> None: + tf = shape.text_frame + tf.clear() + tf.word_wrap = True + tf.vertical_anchor = MSO_ANCHOR.MIDDLE + p = tf.paragraphs[0] + p.alignment = align + run = p.add_run() + run.text = text + run.font.size = Pt(float(font_size)) + run.font.bold = bold + if font_family: + run.font.name = str(font_family) + run.font.color.rgb = _rgb(color) + + +def _add_round_rect(slide, x: float, y: float, w: float, h: float, fill: str, stroke: str, width_pt: float = 1.2): + shape = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x), Inches(y), Inches(w), Inches(h)) + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(fill) + shape.line.color.rgb = _rgb(stroke) + shape.line.width = Pt(width_pt) + shape.shadow.inherit = False + return shape + + +def _add_card_frame(slide, card: dict, width_in: float, height_in: float): + x, y, w, h = pct_to_inches(card["bbox_percent"], width_in, height_in) + shape_type = MSO_SHAPE.RECTANGLE if str(card.get("shape_kind") or "rounded_rect").lower() == "rect" else MSO_SHAPE.ROUNDED_RECTANGLE + shape = slide.shapes.add_shape(shape_type, Inches(x), Inches(y), Inches(w), Inches(h)) + if float(card.get("fill_transparency", 1.0)) >= 1.0: + shape.fill.background() + else: + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(str(card.get("fill_color") or "#FFFFFF")) + shape.line.color.rgb = _rgb(str(card.get("stroke_color") or "#59AFCB")) + shape.line.width = Pt(float(card.get("stroke_width_pt") or 1.5)) + dash_style = str(card.get("dash_style") or "solid").lower() + if dash_style in {"dash", "dashed"}: + shape.line.dash_style = MSO_LINE_DASH_STYLE.DASH + elif dash_style in {"dot", "dotted"}: + shape.line.dash_style = getattr(MSO_LINE_DASH_STYLE, "ROUND_DOT", MSO_LINE_DASH_STYLE.DASH) + shape.shadow.inherit = False + return shape + + +def _add_label(slide, text: str, x: float, y: float, w: float, h: float, font_size: float = 9, bold: bool = False, align=PP_ALIGN.CENTER, color: str = "#163B4D", font_family: str | None = None): + box = slide.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h)) + _set_text(box, text, font_size=font_size, bold=bold, color=color, align=align, font_family=font_family) + return box + + +def _add_title_block(slide, program: dict, width_in: float, height_in: float) -> None: + title_block = program.get("title_block") + if not isinstance(title_block, dict): + return + title = str(title_block.get("title", "")).strip() + subtitle = str(title_block.get("subtitle", "")).strip() + bbox = title_block.get("bbox_percent") + if not title or not isinstance(bbox, dict): + return + x, y, w, h = pct_to_inches(bbox, width_in, height_in) + title_h = min(h * 0.58, 0.42) + subtitle_h = max(0.18, min(h - title_h, 0.26)) + _add_label( + slide, + title, + x, + y, + w, + title_h, + font_size=int(title_block.get("title_font_size", 24)), + bold=True, + align=PP_ALIGN.LEFT, + ) + if subtitle: + subtitle_box = _add_label( + slide, + subtitle, + x, + y + title_h * 0.94, + w, + subtitle_h, + font_size=int(title_block.get("subtitle_font_size", 11)), + bold=False, + align=PP_ALIGN.LEFT, + ) + for paragraph in subtitle_box.text_frame.paragraphs: + for run in paragraph.runs: + run.font.color.rgb = _rgb(str(title_block.get("subtitle_color", "#333333"))) + + +def _short_caption(text: str) -> str: + replacements = { + "participant seated before 49-inch screen": "Participant setup", + "3D virtual interviewer on screen": "Virtual interviewer", + "wide-angle camera capturing full body": "Full-body camera", + "virtual interview room setup": "Interview room", + "interview question prompt on screen": "Question prompt", + "raw interview video recording": "Raw video", + "287 participant video collection": "287 participants", + "36 interview questions": "36 questions", + "full-body video thumbnails": "Full-body clips", + "participant clip grid": "Clip grid", + "FFmpeg extracts audio from video": "FFmpeg audio", + "audio waveform stream": "Audio stream", + "FunASR speech recognition": "FunASR ASR", + "spoken text with timestamps": "Text + timestamps", + "MTCNN face detection": "MTCNN face", + "face clips": "Face clips", + "AlphaPose full-body skeleton extraction": "AlphaPose pose", + "full-body pose skeleton stream": "Pose stream", + "frame sampling from video": "Frame sampling", + "sampled video frames": "Video frames", + "timestamp alignment clock": "Alignment clock", + "aligned audio and text streams": "Audio + text", + "aligned face and pose streams": "Face + pose", + "aligned frame stream": "Frame stream", + "face modality": "Face", + "frame modality": "Frame", + "pose modality": "Pose", + "audio modality": "Audio", + "text modality": "Text", + "NEO-FFI-3 questionnaire": "NEO-FFI-3", + "OCEAN score vector": "OCEAN scores", + "Openness score": "Openness", + "Conscientiousness score": "Conscientiousness", + "Extraversion score": "Extraversion", + "Agreeableness score": "Agreeableness", + "Neuroticism score": "Neuroticism", + } + if text in replacements: + return replacements[text] + if len(text) <= 22: + return text + words = text.replace("/", " ").split() + return " ".join(words[:3]) if len(words) > 3 else text[:22] + + +def _picture_contain_box(image_path: Path, x: float, y: float, w: float, h: float) -> tuple[float, float, float, float, float]: + with Image.open(image_path) as img: + iw, ih = img.size + image_ratio = iw / max(ih, 1) + slot_ratio = w / max(h, 0.001) + if image_ratio > slot_ratio: + fit_w = w + fit_h = w / image_ratio + else: + fit_h = h + fit_w = h * image_ratio + left = x + (w - fit_w) / 2 + top = y + (h - fit_h) / 2 + fill_percent = fit_w * fit_h / max(w * h, 0.001) * 100 + return left, top, fit_w, fit_h, fill_percent + + +def _add_picture_contain(slide, image_path: Path, x: float, y: float, w: float, h: float): + left, top, fit_w, fit_h, _fill_percent = _picture_contain_box(image_path, x, y, w, h) + return slide.shapes.add_picture(str(image_path), Inches(left), Inches(top), width=Inches(fit_w), height=Inches(fit_h)) + + +def _panel_map(program: dict) -> dict[str, dict]: + return {panel["id"]: panel for panel in program.get("panels", [])} + + +def _text_program_items(program: dict) -> list[dict]: + text_program = program.get("text_program") + if not isinstance(text_program, dict): + return [] + items = text_program.get("items", []) + return [item for item in items if isinstance(item, dict) and item.get("visible", True)] + + +def _text_item_for_target(program: dict, target_id: str, role: str) -> dict | None: + for item in _text_program_items(program): + if str(item.get("target_id")) == target_id and str(item.get("role")) == role: + return item + return None + + +def _align_from_text_item(item: dict): + value = str(item.get("align") or "").lower() + if value == "left": + return PP_ALIGN.LEFT + if value == "right": + return PP_ALIGN.RIGHT + return PP_ALIGN.CENTER + + +def _object_map(program: dict) -> dict[str, dict]: + objects = {panel["id"]: panel for panel in program.get("panels", [])} + objects.update({slot["id"]: slot for slot in program.get("slots", [])}) + objects.update({node["id"]: node for node in program.get("semantic_nodes", []) if isinstance(node, dict) and node.get("id")}) + return objects + + +def _bbox_center(bbox: dict, canvas_w: float, canvas_h: float) -> tuple[float, float]: + x, y, w, h = pct_to_inches(bbox, canvas_w, canvas_h) + return x + w / 2, y + h / 2 + + +def _arrow_endpoints(source: dict, target: dict, canvas_w: float, canvas_h: float) -> tuple[float, float, float, float]: + sbox = source["bbox_percent"] + tbox = target["bbox_percent"] + sx, sy = _bbox_center(sbox, canvas_w, canvas_h) + tx, ty = _bbox_center(tbox, canvas_w, canvas_h) + s_left, s_top, s_w, s_h = pct_to_inches(sbox, canvas_w, canvas_h) + t_left, t_top, t_w, t_h = pct_to_inches(tbox, canvas_w, canvas_h) + + dx = tx - sx + dy = ty - sy + if abs(dx) >= abs(dy): + if dx >= 0: + return s_left + s_w + 0.03, sy, t_left - 0.03, ty + return s_left - 0.03, sy, t_left + t_w + 0.03, ty + if dy >= 0: + return sx, s_top + s_h + 0.03, tx, t_top - 0.03 + return sx, s_top - 0.03, tx, t_top + t_h + 0.03 + + +def _arrow_points_from_path(path: list, width_in: float, height_in: float) -> list[tuple[float, float]]: + points = [] + for point in path: + if isinstance(point, list) and len(point) >= 2: + points.append((float(point[0]) * width_in, float(point[1]) * height_in)) + return points + + +def _draw_program_arrows(slide, program: dict, width_in: float, height_in: float) -> list[dict]: + objects_by_id = _object_map(program) + style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} + token_map = {str(item.get("token_id")): item for item in style.get("color_tokens", []) if isinstance(item, dict)} + rendered: list[dict] = [] + for arrow in program.get("arrows", []): + if arrow.get("type") == "custom_bus": + continue + source = objects_by_id.get(arrow.get("source") or arrow.get("source_id")) + target = objects_by_id.get(arrow.get("target") or arrow.get("target_id")) + path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + if len(path) >= 2 and all(isinstance(point, list) and len(point) >= 2 for point in path): + points = _arrow_points_from_path(path, width_in, height_in) + elif source and target: + x1, y1, x2, y2 = _arrow_endpoints(source, target, width_in, height_in) + points = [(x1, y1), (x2, y2)] + else: + continue + token = token_map.get(str(arrow.get("style_token_id"))) + arrow_color = str(token.get("hex")) if token else str(arrow.get("stroke_color") or "#1F6F8B") + control_kind = str(arrow.get("control_kind") or arrow.get("type", "")).lower() + dashed = control_kind in {"dashed_loop", "dashed", "loop"} + if str(arrow.get("line_pattern", "")).lower() in {"dash", "dashed"}: + dashed = True + line_width = float(arrow.get("stroke_width_pt") or style.get("arrow_weight_pt") or 1.7) + route_style = str(arrow.get("route_style") or "") + connector_type = _connector_type_for_route(route_style) + halo_width = float(arrow.get("halo_width_pt") or 0.0) + halo_color = str(arrow.get("halo_color") or "#FFFFFF") + arrowhead_size = str(arrow.get("arrowhead_size") or "sm").lower() + if arrowhead_size not in {"sm", "med", "lg"}: + arrowhead_size = "sm" + segment_count = 0 + for idx, ((x1, y1), (x2, y2)) in enumerate(zip(points[:-1], points[1:])): + if halo_width > 0: + halo = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) + halo.line.color.rgb = _rgb(halo_color) + halo.line.width = Pt(max(line_width + 1.2, halo_width)) + _apply_line_cap(halo, str(arrow.get("line_cap") or "round")) + if dashed: + halo.line.dash_style = MSO_LINE_DASH_STYLE.DASH + connector = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) + connector.name = f"RFS Connector {arrow.get('id') or 'unnamed'} {idx + 1}" + connector.line.color.rgb = _rgb(arrow_color) + connector.line.width = Pt(line_width) + _apply_line_cap(connector, str(arrow.get("line_cap") or "round")) + if dashed: + connector.line.dash_style = MSO_LINE_DASH_STYLE.DASH + if idx == len(points) - 2: + _apply_arrow(connector, arrowhead_size) + segment_count += 1 + rendered.append({ + "arrow_id": arrow.get("id"), + "control_kind": control_kind or "straight_arrow", + "semantic_role": arrow.get("semantic_role"), + "route_style": route_style, + "bundle_id": arrow.get("bundle_id"), + "lane_index": arrow.get("lane_index"), + "line_cap": arrow.get("line_cap", "round"), + "line_pattern": "dash" if dashed else "solid", + "connector_type": "curve" if connector_type == MSO_CONNECTOR.CURVE else "straight", + "halo_width_pt": halo_width, + "halo_color": halo_color if halo_width > 0 else None, + "stroke_width_pt": line_width, + "arrowhead_size": arrowhead_size, + "routing_algorithm": arrow.get("routing_algorithm"), + "route_generation_status": arrow.get("route_generation_status"), + "segment_count": segment_count, + "point_count": len(points), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + "status": "ok" if segment_count else "not_rendered", + }) + return rendered + + +def _ocean_label(text: str) -> str: + low = text.lower() + if "openness" in low: + return "Openness" + if "conscientiousness" in low: + return "Conscientiousness" + if "extraversion" in low: + return "Extraversion" + if "agreeableness" in low: + return "Agreeableness" + if "neuroticism" in low: + return "Neuroticism" + return _short_caption(text) + + +def compile_ppt(program: dict, out_dir: str | Path) -> Path: + out = Path(out_dir) + canvas = program["canvas"] + width_in = float(canvas["width_in"]) + height_in = float(canvas["height_in"]) + style = program.get("style", {}) + palette = style.get("palette") or style.get("reference_palette") or ["#2D6FB7", "#E17721", "#6B57C8", "#1B9A94", "#4B9B52", "#D44E5D"] + panel_styles = style.get("panel_styles", {}) if isinstance(style.get("panel_styles"), dict) else {} + + prs = Presentation() + prs.slide_width = Inches(width_in) + prs.slide_height = Inches(height_in) + slide = prs.slides.add_slide(prs.slide_layouts[6]) + + background_visual = None + background = program.get("background_image") + if isinstance(background, dict) and str(background.get("path") or "").strip(): + background_path = Path(str(background["path"])) + if not background_path.is_absolute(): + background_path = out / background_path + if not background_path.exists(): + raise FileNotFoundError(f"PPT background visual substrate is missing: {background_path}") + left, top, fit_w, fit_h, fill_percent = _picture_contain_box(background_path, 0.0, 0.0, width_in, height_in) + picture = slide.shapes.add_picture( + str(background_path), + Inches(left), + Inches(top), + width=Inches(fit_w), + height=Inches(fit_h), + ) + picture.name = str(background.get("name") or "RFS Image2 Visual Substrate") + background_visual = { + "path": str(background_path), + "role": str(background.get("role") or "image2_visual_substrate"), + "contains_scientific_text": bool(background.get("contains_scientific_text", False)), + "contains_connectors": bool(background.get("contains_connectors", False)), + "canvas_area_fill_percent": round(fill_percent, 2), + "fit_policy": "contain_no_crop", + "rendered_as": "ppt_picture_background", + } + + _add_title_block(slide, program, width_in, height_in) + + panels_by_id = _panel_map(program) + panel_shapes = {} + has_text_program = bool(_text_program_items(program)) + ocr_panel_title_targets = { + str(item.get("target_id")) + for item in _text_program_items(program) + if str(item.get("role")) == "panel_title" and "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() + } + for idx, panel in enumerate(program["panels"]): + x, y, w, h = pct_to_inches(panel["bbox_percent"], width_in, height_in) + local_style = panel_styles.get(panel["id"], {}) if isinstance(panel_styles.get(panel["id"], {}), dict) else {} + fill = local_style.get("fill_color") or palette[idx % len(palette)] + stroke = local_style.get("stroke_color") or palette[(idx + 1) % len(palette)] + header_color = local_style.get("header_color") or stroke + shape = _add_round_rect(slide, x, y, w, h, fill, stroke, width_pt=1.4) + panel_shapes[panel["id"]] = shape + header_h = min(0.34, h * 0.17) + header = _add_round_rect(slide, x, y, w, header_h, header_color, header_color, width_pt=0.8) + title_text_item = _text_item_for_target(program, panel["id"], "panel_title") + _set_text( + header, + "" if str(panel["id"]) in ocr_panel_title_targets else panel["title"], + font_size=float(title_text_item.get("font_size_pt")) if title_text_item else (9 if w < 2.0 else 10), + bold=True, + color=str(title_text_item.get("color_hex")) if title_text_item else "#FFFFFF", + font_family=str(title_text_item.get("font_family_guess") or "") if title_text_item else None, + ) + + rendered_cards = [] + for card in sorted(program.get("cards", []), key=lambda item: int(item.get("z_index", 12))): + if not isinstance(card, dict) or not isinstance(card.get("bbox_percent"), dict): + continue + _add_card_frame(slide, card, width_in, height_in) + x, y, w, h = pct_to_inches(card["bbox_percent"], width_in, height_in) + if card.get("title") and not has_text_program: + _add_label(slide, str(card.get("title")), x + 0.03, y + 0.03, max(0.05, w - 0.06), min(0.22, h * 0.32), font_size=7, bold=True) + rendered_cards.append({ + "card_id": card.get("id"), + "semantic_role": card.get("semantic_role"), + "shape_kind": card.get("shape_kind", "rounded_rect"), + "bbox_percent": card.get("bbox_percent"), + "dash_style": card.get("dash_style", "solid"), + "fill_transparency": float(card.get("fill_transparency", 1.0)), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + "status": "ok", + }) + + rendered_arrows = _draw_program_arrows(slide, program, width_in, height_in) + + # Paper semantic nodes are rendered after connectors so every edge stays + # behind the editable node shape and its exact paper label. + rendered_semantic_nodes = [] + for node in sorted(program.get("semantic_nodes", []), key=lambda item: int(item.get("z_index", 30))): + if not isinstance(node, dict) or not isinstance(node.get("bbox_percent"), dict): + continue + x, y, w, h = pct_to_inches(node["bbox_percent"], width_in, height_in) + shape_kind = str(node.get("shape_kind") or "rounded_rect").casefold() + shape_type = MSO_SHAPE.RECTANGLE if shape_kind == "rect" else MSO_SHAPE.ROUNDED_RECTANGLE + shape = slide.shapes.add_shape(shape_type, Inches(x), Inches(y), Inches(w), Inches(h)) + shape.name = f"RFS Semantic Node {node.get('id') or 'unnamed'}" + if float(node.get("fill_transparency", 0.0)) >= 1.0: + shape.fill.background() + else: + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(str(node.get("fill_color") or "#F5FAFC")) + shape.line.color.rgb = _rgb(str(node.get("stroke_color") or "#1F6F8B")) + shape.line.width = Pt(float(node.get("stroke_width_pt") or 1.6)) + shape.shadow.inherit = False + label_bbox = node.get("label_bbox_percent") + _set_text( + shape, + "" if isinstance(label_bbox, dict) else str(node.get("label") or node.get("name") or node.get("id") or ""), + font_size=float(node.get("font_size_pt") or 16), + bold=bool(node.get("bold", True)), + color=str(node.get("text_color") or node.get("stroke_color") or "#163B4D"), + font_family=str(node.get("font_family") or "Arial"), + ) + label_rendered_as = "node_shape_text" + if isinstance(label_bbox, dict): + lx, ly, lw, lh = pct_to_inches(label_bbox, width_in, height_in) + label_shape = _add_round_rect( + slide, + lx, + ly, + lw, + lh, + str(node.get("label_fill_color") or node.get("fill_color") or "#F5FAFC"), + str(node.get("label_stroke_color") or "#FFFFFF"), + width_pt=float(node.get("label_stroke_width_pt") or 0.5), + ) + label_shape.name = f"RFS Semantic Label {node.get('id') or 'unnamed'}" + _set_text( + label_shape, + str(node.get("label") or node.get("name") or node.get("id") or ""), + font_size=float(node.get("font_size_pt") or 16), + bold=bool(node.get("bold", True)), + color=str(node.get("text_color") or node.get("stroke_color") or "#163B4D"), + font_family=str(node.get("font_family") or "Arial"), + ) + label_rendered_as = "separate_editable_header_shape" + rendered_semantic_nodes.append({ + "node_id": node.get("id"), + "label": node.get("label") or node.get("name"), + "field": node.get("field"), + "role": node.get("role"), + "bbox_percent": node.get("bbox_percent"), + "shape_kind": shape_kind, + "fill_transparency": float(node.get("fill_transparency", 0.0)), + "label_bbox_percent": label_bbox, + "label_rendered_as": label_rendered_as, + "editable_in": "pptx", + "render_policy": "ppt_transparent_node_plus_editable_header" if isinstance(label_bbox, dict) else "ppt_shape_with_editable_text", + "status": "ok", + }) + + # Slot image layer and editable captions. + composition_items = [] + caption_queue = [] + ordered_slots = sorted(program["slots"], key=lambda item: int(item.get("z_index", 20))) + for slot in ordered_slots: + x, y, w, h = pct_to_inches(slot["bbox_percent"], width_in, height_in) + asset_path = out / "assets" / f"{slot['asset_id']}.png" + tile_added = False + if slot.get("composition_type") == "symbol_cutout": + if asset_path.exists(): + _add_picture_contain(slide, asset_path, x, y, w, h) + _left, _top, fit_w, fit_h, fill_percent = _picture_contain_box(asset_path, x, y, w, h) + else: + fill_percent = 0.0 + if bool(slot.get("show_slot_caption", False)) and not has_text_program: + parent = panels_by_id.get(slot.get("panel_id")) + if parent: + px, py, pw, ph = pct_to_inches(parent["bbox_percent"], width_in, height_in) + label_x = x + w + 0.035 + label_w = max(0.28, min(0.86, px + pw - label_x - 0.03)) + caption_queue.append(( + _ocean_label(slot.get("display_label") or slot["paper_concept"]), + label_x, + y + h * 0.10, + label_w, + h * 0.80, + 5 if label_w < 0.55 else 6, + PP_ALIGN.LEFT, + )) + composition_items.append({ + "slot_id": slot["id"], + "asset_id": slot["asset_id"], + "slot_frame_policy": slot.get("slot_frame_policy", "frameless_slot"), + "picture_fill_policy": slot.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), + "tile_frame_added": tile_added, + "caption_inside_image_slot": False, + "slot_bbox_percent": slot["bbox_percent"], + "image_slot_area_fill_percent": round(fill_percent, 2), + "status": "ok" if fill_percent >= 95 else "image_area_below_95", + }) + continue + if asset_path.exists(): + _add_picture_contain(slide, asset_path, x, y, w, h) + _left, _top, fit_w, fit_h, fill_percent = _picture_contain_box(asset_path, x, y, w, h) + else: + fill_percent = 0.0 + if bool(slot.get("show_slot_caption", False)) and not has_text_program: + caption_h = min(0.20, h * 0.18) + caption = slot.get("display_label") or _short_caption(slot["paper_concept"]) + caption_queue.append((caption, x + w * 0.03, y + h + 0.01, w * 0.94, caption_h, 5 if w < 0.8 else 6, PP_ALIGN.CENTER)) + composition_items.append({ + "slot_id": slot["id"], + "asset_id": slot["asset_id"], + "slot_frame_policy": slot.get("slot_frame_policy", "frameless_slot"), + "picture_fill_policy": slot.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), + "tile_frame_added": tile_added, + "caption_inside_image_slot": False, + "slot_bbox_percent": slot["bbox_percent"], + "image_slot_area_fill_percent": round(fill_percent, 2), + "status": "ok" if fill_percent >= 95 else "image_area_below_95", + }) + + for caption, x, y, w, h, font_size, align in caption_queue: + _add_label(slide, caption, x, y, w, h, font_size=font_size, align=align) + + rendered_text_items = [] + for item in _text_program_items(program): + is_ocr_text = "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() + if str(item.get("role")) == "panel_title" and not is_ocr_text: + rendered_text_items.append({ + "text_id": item.get("id"), + "role": item.get("role"), + "target_id": item.get("target_id"), + "rendered_as": "panel_header_text", + "editable_in": "pptx", + }) + continue + bbox = item.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = pct_to_inches(bbox, width_in, height_in) + _add_label( + slide, + str(item.get("text") or ""), + x, + y, + w, + h, + font_size=float(item.get("font_size_pt") or 6), + bold=bool(item.get("bold")), + align=_align_from_text_item(item), + color=str(item.get("color_hex") or "#263747"), + font_family=str(item.get("font_family_guess") or "") or None, + ) + rendered_text_items.append({ + "text_id": item.get("id"), + "role": item.get("role"), + "target_id": item.get("target_id"), + "source_reference_text_id": item.get("source_reference_text_id"), + "bbox_percent": item.get("bbox_percent"), + "font_size_pt": item.get("font_size_pt"), + "color_hex": item.get("color_hex"), + "font_family_guess": item.get("font_family_guess"), + "fit_strategy": item.get("fit_strategy"), + "ocr_confidence": item.get("ocr_confidence"), + "editable_in": "pptx", + "rendered_as": "ppt_textbox", + }) + + for label in program.get("labels", []): + bbox = label.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = pct_to_inches(bbox, width_in, height_in) + _add_label( + slide, + str(label.get("text") or ""), + x, + y, + w, + h, + font_size=float(label.get("font_size_pt") or 9), + bold=bool(label.get("bold", True)), + align=PP_ALIGN.LEFT if str(label.get("align")).lower() == "left" else PP_ALIGN.CENTER, + color=str(label.get("color_hex") or "#263747"), + ) + + # Shared resource bus as editable connectors. + shared = panels_by_id.get("shared_resource_library") + if shared: + sx, sy, sw, sh = pct_to_inches(shared["bbox_percent"], width_in, height_in) + bus_y = sy - 0.30 + line = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(sx + 0.1), Inches(bus_y), Inches(sx + sw - 0.1), Inches(bus_y)) + line.line.color.rgb = _rgb("#1F6F8B") + line.line.width = Pt(1.2) + for panel in program["panels"]: + if panel["id"] == "shared_resource_library": + continue + cx, _cy = _bbox_center(panel["bbox_percent"], width_in, height_in) + if not (sx + 0.1 <= cx <= sx + sw - 0.1): + continue + down = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(cx), Inches(bus_y), Inches(cx), Inches(sy)) + down.line.color.rgb = _rgb("#1F6F8B") + down.line.width = Pt(1.0) + if not has_text_program: + _add_label(slide, "shared resources", sx + 0.2, bus_y - 0.22, 1.5, 0.18, font_size=7, align=PP_ALIGN.LEFT) + + if program.get("show_visible_title"): + title = program.get("paper_brief", {}).get("title_guess") or "Research System Figure" + _add_label(slide, title[:90], width_in - 5.2, height_in - 0.50, 4.9, 0.38, font_size=7, bold=True, align=PP_ALIGN.RIGHT) + + pptx_path = out / "editable_composition.pptx" + prs.save(pptx_path) + write_json(out / "composition_quality_report.json", { + "summary": "PPT composition quality report checking frameless slot insertion, no extra white tiles, and image area fill inside each reference slot.", + "policy": { + "slot_frame_policy": "frameless_slot", + "picture_fill_policy": "direct_full_slot_contain_no_tile", + "min_image_slot_area_fill_percent": 95, + "caption_inside_image_slot_allowed": False, + }, + "slots": composition_items, + "cards": rendered_cards, + "arrows": rendered_arrows, + "semantic_nodes": rendered_semantic_nodes, + "text": rendered_text_items, + "background_visual": background_visual, + }) + return pptx_path diff --git a/rfs/composition/preview.py b/rfs/composition/preview.py new file mode 100644 index 0000000..c2ecef9 --- /dev/null +++ b/rfs/composition/preview.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from pathlib import Path + +from PIL import Image, ImageColor, ImageDraw, ImageFont + + +def _hex_color(value: object, fallback: str = "#000000") -> str: + if not isinstance(value, str): + return fallback + text = value.strip() + if not text: + return fallback + if not text.startswith("#"): + text = "#" + text + try: + ImageColor.getrgb(text) + return text + except ValueError: + return fallback + + +def _bbox_px(bbox: dict, width: int, height: int) -> tuple[int, int, int, int]: + x = int(round(float(bbox.get("x", 0)) * width)) + y = int(round(float(bbox.get("y", 0)) * height)) + w = int(round(float(bbox.get("w", 0)) * width)) + h = int(round(float(bbox.get("h", 0)) * height)) + return x, y, max(1, w), max(1, h) + + +def _font(size_px: int, bold: bool = False) -> ImageFont.ImageFont: + candidates = [ + "arialbd.ttf" if bold else "arial.ttf", + "calibrib.ttf" if bold else "calibri.ttf", + "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf", + ] + for name in candidates: + try: + return ImageFont.truetype(name, max(4, int(size_px))) + except OSError: + continue + return ImageFont.load_default() + + +def _draw_text(draw: ImageDraw.ImageDraw, item: dict, canvas_w: int, canvas_h: int) -> None: + if not item.get("visible", True): + return + if str(item.get("layer_ownership") or "editable_text_layer") != "editable_text_layer": + return + bbox = item.get("bbox_percent") + if not isinstance(bbox, dict): + return + x, y, w, h = _bbox_px(bbox, canvas_w, canvas_h) + text = str(item.get("text") or "") + if not text: + return + font_size_pt = float(item.get("font_size_pt") or 8) + size_px = max(4, int(round(font_size_pt * canvas_h / 387.0))) + font = _font(size_px, bold=bool(item.get("bold"))) + color = _hex_color(item.get("color_hex"), "#263747") + align = str(item.get("align") or "center").lower() + text_bbox = draw.textbbox((0, 0), text, font=font) + text_w = max(1, text_bbox[2] - text_bbox[0]) + text_h = max(1, text_bbox[3] - text_bbox[1]) + if align == "left": + tx = x + elif align == "right": + tx = x + w - text_w + else: + tx = x + (w - text_w) / 2 + ty = y + (h - text_h) / 2 + draw.text((tx, ty), text, fill=color, font=font) + + +def _draw_arrow(draw: ImageDraw.ImageDraw, arrow: dict, canvas_w: int, canvas_h: int) -> None: + path = arrow.get("path_percent") + if not isinstance(path, list) or len(path) < 2: + return + points: list[tuple[int, int]] = [] + for point in path: + if isinstance(point, list) and len(point) >= 2: + points.append((int(round(float(point[0]) * canvas_w)), int(round(float(point[1]) * canvas_h)))) + if len(points) < 2: + return + color = _hex_color(arrow.get("stroke_color") or arrow.get("outline_color"), "#3F5063") + line_width = max(1, int(round(float(arrow.get("stroke_width_pt") or 1.5) * canvas_h / 277.0))) + render_style = str(arrow.get("render_style") or "").lower() + if render_style == "filled_block_arrow": + fill = _hex_color(arrow.get("fill_color"), "#AFC6DE") + start, end = points[0], points[-1] + thickness = max(8, int(round(float(arrow.get("block_arrow_thickness_percent") or 0.052) * canvas_h))) + if abs(end[0] - start[0]) >= abs(end[1] - start[1]): + y = int(round((start[1] + end[1]) / 2)) + left, right = sorted([start[0], end[0]]) + draw.rounded_rectangle((left, y - thickness // 2, right, y + thickness // 2), radius=thickness // 3, fill=fill, outline=color, width=line_width) + else: + x = int(round((start[0] + end[0]) / 2)) + top, bottom = sorted([start[1], end[1]]) + draw.rounded_rectangle((x - thickness // 2, top, x + thickness // 2, bottom), radius=thickness // 3, fill=fill, outline=color, width=line_width) + return + draw.line(points, fill=color, width=line_width, joint="curve") + + +def _draw_card(draw: ImageDraw.ImageDraw, card: dict, canvas_w: int, canvas_h: int) -> None: + bbox = card.get("bbox_percent") + if not isinstance(bbox, dict): + return + x, y, w, h = _bbox_px(bbox, canvas_w, canvas_h) + stroke = _hex_color(card.get("stroke_color"), "#59AFCB") + fill = None if float(card.get("fill_transparency", 1.0)) >= 1.0 else _hex_color(card.get("fill_color"), "#FFFFFF") + line_width = max(1, int(round(float(card.get("stroke_width_pt") or 1.5) * canvas_h / 360.0))) + radius = max(0, int(round(float(card.get("corner_radius") or 0.08) * min(w, h)))) + rect = (x, y, x + w, y + h) + if str(card.get("shape_kind") or "rounded_rect").lower() == "rect": + draw.rectangle(rect, fill=fill, outline=stroke, width=line_width) + else: + draw.rounded_rectangle(rect, radius=radius, fill=fill, outline=stroke, width=line_width) + + +def render_rebuild_preview(program: dict, out_dir: str | Path, preview_path: str | Path | None = None) -> Path: + out = Path(out_dir) + canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} + width = int(canvas.get("width_px") or 1600) + height = int(canvas.get("height_px") or 900) + background = _hex_color(canvas.get("background"), "#FFFFFF") + image = Image.new("RGB", (width, height), background) + draw = ImageDraw.Draw(image) + + style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} + panel_styles = style.get("panel_styles", {}) if isinstance(style.get("panel_styles"), dict) else {} + palette = style.get("palette") or ["#F7F8F7", "#D2DCDE", "#A09596"] + + for idx, panel in enumerate(program.get("panels", [])): + bbox = panel.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = _bbox_px(bbox, width, height) + local_style = panel_styles.get(panel.get("id"), {}) if isinstance(panel_styles.get(panel.get("id")), dict) else {} + fill = _hex_color(local_style.get("fill_color"), palette[idx % len(palette)]) + stroke = _hex_color(local_style.get("stroke_color"), "#FFFFFF") + draw.rounded_rectangle((x, y, x + w, y + h), radius=max(2, min(12, h // 18)), fill=fill, outline=stroke, width=2) + + for card in sorted(program.get("cards", []), key=lambda item: int(item.get("z_index") or 12)): + if isinstance(card, dict): + _draw_card(draw, card, width, height) + + asset_by_id = {str(asset.get("id")): asset for asset in program.get("assets", []) if isinstance(asset, dict)} + for slot in sorted(program.get("slots", []), key=lambda item: int(item.get("z_index") or 20)): + bbox = slot.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = _bbox_px(bbox, width, height) + asset = asset_by_id.get(str(slot.get("asset_id") or slot.get("id"))) + asset_path = out / str(asset.get("path")) if asset else out / "assets" / f"{slot.get('asset_id')}.png" + if asset_path.exists(): + with Image.open(asset_path) as asset_img: + asset_img = asset_img.convert("RGB") + asset_img.thumbnail((w, h), Image.Resampling.LANCZOS) + left = x + (w - asset_img.width) // 2 + top = y + (h - asset_img.height) // 2 + image.paste(asset_img, (left, top)) + + for arrow in program.get("arrows", []): + if isinstance(arrow, dict): + _draw_arrow(draw, arrow, width, height) + + text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {} + text_items = text_program.get("items", []) if isinstance(text_program, dict) else [] + visible_items = [item for item in text_items if isinstance(item, dict)] + for item in sorted(visible_items, key=lambda item: (float(item.get("z_index") or 80), str(item.get("id") or ""))): + _draw_text(draw, item, width, height) + + target = Path(preview_path) if preview_path else out / "rebuild_preview.png" + target.parent.mkdir(parents=True, exist_ok=True) + image.save(target) + return target diff --git a/rfs/contracts/__init__.py b/rfs/contracts/__init__.py new file mode 100644 index 0000000..c866d03 --- /dev/null +++ b/rfs/contracts/__init__.py @@ -0,0 +1,5 @@ +"""Stable data-contract APIs shared across ResearchFigureStudio workflows.""" + +from .semantic import apply_paper_semantic_contract, compile_paper_brief_from_semantic_contract, load_paper_semantic_contract + +__all__ = ["apply_paper_semantic_contract", "compile_paper_brief_from_semantic_contract", "load_paper_semantic_contract"] diff --git a/rfs/contracts/semantic.py b/rfs/contracts/semantic.py new file mode 100644 index 0000000..9cea2ca --- /dev/null +++ b/rfs/contracts/semantic.py @@ -0,0 +1,615 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +from ..utils import read_json, write_json, write_text + + +def load_paper_semantic_contract(run_dir: str | Path) -> dict[str, Any]: + root = Path(run_dir) + if root.is_file(): + payload = read_json(root) + if root.name == "figure_specification.json" or "panel_graphs" in payload or "relations" in payload: + return { + "summary": "Paper-grounded semantic contract loaded from a figure specification.", + "source_run_dir": str(root.parent.resolve()), + "figure_specification": payload, + } + return payload + + def optional(name: str) -> dict[str, Any]: + path = root / name + return read_json(path) if path.exists() else {} + + specification_path = root / "figure_specification.json" + if not specification_path.exists(): + raise FileNotFoundError(f"semantic contract directory has no figure_specification.json: {root}") + return { + "summary": "Paper-grounded semantic contract for editable figure reconstruction.", + "source_run_dir": str(root.resolve()), + "paper_review": optional("paper_review.json"), + "figure_specification": read_json(root / "figure_specification.json"), + "paper_summary": optional("paper_summary.json"), + "design_plan": optional("design_plan.json"), + "layout_intent": optional("layout_intent.json"), + "style_plan": optional("style_plan.json"), + } + + +def _text(item: dict, fallback: str = "") -> str: + for key in ("visible_label", "name", "text", "statement", "label", "id"): + value = str(item.get(key) or "").strip() + if value: + return value + return fallback + + +def _specification(contract: dict) -> dict: + return contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract + + +def _panel_graph_entities(spec: dict) -> list[dict]: + entities: list[dict] = [] + seen_ids: set[str] = set() + for panel_index, panel in enumerate(spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(panel, dict): + continue + panel_id = str(panel.get("id") or f"panel_{panel_index + 1}").strip() + panel_label = str(panel.get("label") or panel.get("description") or panel_id).strip() + for node_index, raw in enumerate(panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []): + if not isinstance(raw, dict): + continue + role = str(raw.get("role") or "module").strip() + if role.casefold() in {"group", "container", "panel", "stage"}: + continue + label = _text(raw) + entity_id = str(raw.get("instance_id") or raw.get("id") or f"{panel_id}_node_{node_index + 1:02d}").strip() + if not label or not entity_id or entity_id in seen_ids: + continue + seen_ids.add(entity_id) + entities.append({ + "id": entity_id, + "semantic_id": str(raw.get("semantic_id") or "").strip() or None, + "label": label, + "role": role, + "panel_id": panel_id, + "panel_label": panel_label, + "bbox_percent": raw.get("bbox_percent"), + "evidence_ids": [str(value) for value in raw.get("evidence_ids", []) if value], + "source": raw, + }) + return entities + + +def _entities(contract: dict) -> list[dict]: + spec = _specification(contract) + panel_entities = _panel_graph_entities(spec) + if panel_entities: + return panel_entities + required_labels = { + re.sub(r"[^a-z0-9]+", "", str(value).casefold()) + for value in spec.get("required_labels", []) + if str(value).strip() + } + entities: list[dict] = [] + for field, role in (("inputs", "input"), ("modules", "module"), ("outputs", "output"), ("innovations", "innovation")): + values = spec.get(field) if isinstance(spec.get(field), list) else [] + for index, raw in enumerate(values, start=1): + if not isinstance(raw, dict): + continue + label = _text(raw) + if not label: + continue + if role == "innovation": + visible_required = bool(raw.get("visible_label_required") or raw.get("must_appear_in_figure")) + normalized_label = re.sub(r"[^a-z0-9]+", "", label.casefold()) + if not visible_required and normalized_label not in required_labels: + continue + entity_id = str(raw.get("id") or raw.get("name") or f"{role}_{index:02d}").strip() + entities.append({ + "id": entity_id, + "label": label, + "role": role, + "evidence_ids": list(raw.get("evidence_ids") or []), + "source": raw, + }) + return entities + + +def _relations(contract: dict) -> tuple[list[dict], str]: + spec = _specification(contract) + panel_entities = _panel_graph_entities(spec) + if panel_entities: + entity_ids = {item["id"] for item in panel_entities} + relations: list[dict] = [] + for panel_index, panel in enumerate(spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(panel, dict): + continue + panel_id = str(panel.get("id") or f"panel_{panel_index + 1}").strip() + for relation_index, raw in enumerate(panel.get("relations", []) if isinstance(panel.get("relations"), list) else []): + if not isinstance(raw, dict): + continue + source = str(raw.get("source") or raw.get("source_id") or "").strip() + target = str(raw.get("target") or raw.get("target_id") or "").strip() + if not source or not target or source == target or source not in entity_ids or target not in entity_ids: + continue + relations.append({ + **raw, + "id": str(raw.get("id") or f"{panel_id}_relation_{relation_index + 1:02d}"), + "source": source, + "target": target, + "panel_id": panel_id, + }) + return relations, "panel_graphs" + return [item for item in spec.get("relations", []) if isinstance(item, dict)] if isinstance(spec.get("relations"), list) else [], "global_graph" + + +def _composition_type(role: str, label: str) -> str: + low = f"{role} {label}".casefold() + if any(token in low for token in ("input", "dataset", "sample", "image", "text", "audio", "video")): + return "scene_thumbnail" + if any(token in low for token in ("output", "prediction", "loss", "objective", "score", "distribution")): + return "full_frame_icon" + return "full_bleed_card" + + +def compile_paper_brief_from_semantic_contract( + contract: dict, + out_dir: str | Path | None = None, + *, + source_path: str | Path | None = None, +) -> dict[str, Any]: + """Compile the fast paper contract into the legacy brief shape without rereading the paper.""" + + spec = _specification(contract) + entities = _entities(contract) + relations, relation_source = _relations(contract) + panels = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict)] if isinstance(spec.get("panel_graphs"), list) else [] + modules = [str(item.get("label") or item.get("description") or item.get("id") or "").strip() for item in panels] + modules = [item for item in modules if item] + if not modules: + modules = list(dict.fromkeys(item["panel_label"] for item in entities if item.get("panel_label"))) + if not modules: + modules = list(dict.fromkeys(item["label"] for item in entities if item["role"].casefold() in {"module", "stage"})) + if not modules: + modules = ["Paper-grounded method"] + + outgoing: dict[str, list[str]] = {} + incoming: dict[str, list[str]] = {} + labels_by_id = {item["id"]: item["label"] for item in entities} + for relation in relations: + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + outgoing.setdefault(source, []).append(labels_by_id.get(target, target)) + incoming.setdefault(target, []).append(labels_by_id.get(source, source)) + + suggestions = [] + for entity in entities: + panel = entity.get("panel_label") or modules[0] + neighbor_cues = [*incoming.get(entity["id"], []), *outgoing.get(entity["id"], [])] + must_show = [f"paper-grounded visual object for {entity['label']}", "local reference shape and visual density"] + if neighbor_cues: + must_show.append("declared local context: " + ", ".join(list(dict.fromkeys(neighbor_cues))[:3])) + suggestions.append({ + "id": entity["id"], + "macro_panel": panel, + "paper_concept": entity["label"], + "composition_type": _composition_type(entity["role"], entity["label"]), + "visual_metaphor": f"reference-grounded scientific visual for the paper term {entity['label']}", + "must_show": must_show, + "avoid_showing": ["unmentioned architecture modules", "fake formulas", "fake numeric chart text", "critical text baked into the image"], + "semantic_instance_id": entity["id"], + "semantic_id": entity.get("semantic_id"), + "semantic_panel_id": entity.get("panel_id"), + "semantic_role": entity["role"], + "evidence_ids": entity["evidence_ids"], + "contract_source": relation_source, + }) + + review = contract.get("paper_review") if isinstance(contract.get("paper_review"), dict) else {} + summary = contract.get("paper_summary") if isinstance(contract.get("paper_summary"), dict) else {} + title = str(review.get("title") or review.get("paper_title") or summary.get("title") or "Paper-grounded framework").strip() + goal = str(spec.get("figure_goal") or "").strip() + if not goal: + central_claim = spec.get("central_claim") + goal = _text(central_claim) if isinstance(central_claim, dict) else str(central_claim or "").strip() + if not goal: + goal = "Visualize only the modules and local relations declared in the paper semantic contract." + + brief = { + "summary": "Paper brief compiled directly from the fast evidence-grounded semantic contract.", + "planner": "semantic_contract", + "semantic_authority": relation_source, + "title_guess": title, + "figure_kind": str(spec.get("topology") or "paper_driven_generic"), + "figure_goal": goal, + "modules": modules[:8], + "concepts": [item["label"] for item in entities], + "variables": [str(item) for item in spec.get("required_labels", []) if str(item).strip()], + "slot_suggestions": suggestions, + "semantic_relations": relations, + "semantic_entity_count": len(entities), + "semantic_relation_count": len(relations), + "source_path": str(source_path or contract.get("source_run_dir") or ""), + "source_loader": "semantic_contract", + "char_count": 0, + "warnings": [] if relation_source == "panel_graphs" else ["panel_graphs unavailable; using the compatibility global graph"], + } + if out_dir is not None: + out = Path(out_dir) + write_json(out / "paper_brief.json", brief) + lines = [ + "# Summary", + "Paper brief compiled directly from the fast evidence-grounded semantic contract.", + "", + "## Source", + f"- Contract authority: {relation_source}", + f"- Source: `{brief['source_path']}`", + f"- Title: {title}", + "", + "## Figure Goal", + goal, + "", + "## Panels", + *[f"- {item}" for item in modules], + "", + "## Contract Entities", + *[f"- {item['id']}: {item['label']} ({item['role']})" for item in entities], + "", + "## Contract Relations", + *[f"- {item.get('source')} -> {item.get('target')} ({item.get('type') or 'data_flow'})" for item in relations], + ] + write_text(out / "paper_brief.md", "\n".join(lines) + "\n") + return brief + + +def _tokens(value: object) -> set[str]: + return {token for token in re.findall(r"[a-z0-9]+", str(value or "").lower()) if len(token) > 1} + + +def _object_text(obj: dict) -> str: + return " ".join(str(obj.get(key) or "") for key in ("id", "title", "label", "name", "semantic_type", "description", "prompt")) + + +def _center(obj: dict) -> tuple[float, float]: + box = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else {} + return float(box.get("x") or 0) + float(box.get("w") or 0) / 2, float(box.get("y") or 0) + float(box.get("h") or 0) / 2 + + +def _assign_entities(entities: list[dict], program: dict) -> tuple[dict[str, dict], list[dict]]: + slots = [item for item in program.get("slots", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + cards = [item for item in program.get("cards", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + panels = [item for item in program.get("panels", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + objects = [*cards, *slots, *panels] + seen: set[str] = set() + objects = [item for item in objects if not (str(item.get("id")) in seen or seen.add(str(item.get("id"))))] + text_items = [ + item for item in (program.get("text_program", {}).get("items", []) if isinstance(program.get("text_program"), dict) else []) + if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) + and not str(item.get("id") or "").startswith("semantic_text_") + and not item.get("semantic_entity_id") + ] + + def object_search_text(obj: dict) -> str: + values = [_object_text(obj)] + box = obj.get("bbox_percent") or {} + for text_item in text_items: + if str(text_item.get("target_id") or "") == str(obj.get("id") or "") or _inside(text_item.get("bbox_percent") or {}, box): + values.append(str(text_item.get("text") or "")) + return " ".join(values) + + def object_priority(obj: dict) -> int: + if obj in cards: + return 2 + if obj in slots: + return 1 + return 0 + + objects.sort(key=lambda item: (_center(item)[1], _center(item)[0], str(item.get("id") or ""))) + available = list(objects) + mapping: dict[str, dict] = {} + records: list[dict] = [] + for entity in entities: + entity_tokens = _tokens(entity["label"]) | _tokens(entity["id"]) + scored = [] + for obj in available: + searchable = object_search_text(obj) + obj_tokens = _tokens(searchable) + score = len(entity_tokens & obj_tokens) / max(1, len(entity_tokens | obj_tokens)) + semantic_instance_id = str(obj.get("semantic_instance_id") or "").strip() + if semantic_instance_id and semantic_instance_id == entity["id"]: + score += 10.0 + semantic_panel_id = str(obj.get("semantic_panel_id") or "").strip() + if semantic_panel_id and semantic_panel_id == str(entity.get("panel_id") or ""): + score += 0.5 + entity_compact = re.sub(r"[^a-z0-9]+", "", str(entity.get("label") or "").casefold()) + object_compact = re.sub(r"[^a-z0-9]+", "", searchable.casefold()) + if entity_compact and entity_compact in object_compact: + score += 1.0 + scored.append((score, object_priority(obj), -_center(obj)[1], -_center(obj)[0], obj)) + if not scored: + records.append({"entity_id": entity["id"], "label": entity["label"], "status": "unmapped"}) + continue + scored.sort(key=lambda value: (value[0], value[1], value[2], value[3]), reverse=True) + chosen = scored[0][4] if scored[0][0] > 0 else available[0] + available.remove(chosen) + mapping[entity["id"]] = chosen + records.append({ + "entity_id": entity["id"], + "label": entity["label"], + "role": entity["role"], + "object_id": chosen.get("id"), + "match_score": round(max(0.0, scored[0][0]), 4), + "status": "mapped" if scored[0][0] > 0 else "mapped_spatial_fallback", + }) + return mapping, records + + +def _inside(inner: dict, outer: dict) -> bool: + cx = float(inner.get("x") or 0) + float(inner.get("w") or 0) / 2 + cy = float(inner.get("y") or 0) + float(inner.get("h") or 0) / 2 + return float(outer.get("x") or 0) <= cx <= float(outer.get("x") or 0) + float(outer.get("w") or 0) and float(outer.get("y") or 0) <= cy <= float(outer.get("y") or 0) + float(outer.get("h") or 0) + + +def _label_bbox(obj: dict) -> dict[str, float]: + box = obj["bbox_percent"] + x, y, w, h = (float(box[key]) for key in ("x", "y", "w", "h")) + label_h = min(0.055, max(0.026, h * 0.18)) + return {"x": round(x, 4), "y": round(max(0.0, y), 4), "w": round(w, 4), "h": round(label_h, 4)} + + +def _anchor(source: dict, target: dict) -> list[list[float]]: + sx, sy = _center(source) + tx, ty = _center(target) + sb, tb = source["bbox_percent"], target["bbox_percent"] + if abs(tx - sx) >= abs(ty - sy): + start = [float(sb["x"]) + (float(sb["w"]) if tx >= sx else 0.0), sy] + end = [float(tb["x"]) + (0.0 if tx >= sx else float(tb["w"])), ty] + mid = (start[0] + end[0]) / 2 + points = [start, [mid, start[1]], [mid, end[1]], end] + else: + start = [sx, float(sb["y"]) + (float(sb["h"]) if ty >= sy else 0.0)] + end = [tx, float(tb["y"]) + (0.0 if ty >= sy else float(tb["h"]))] + mid = (start[1] + end[1]) / 2 + points = [start, [start[0], mid], [end[0], mid], end] + clean: list[list[float]] = [] + for point in points: + rounded = [round(max(0.0, min(1.0, point[0])), 4), round(max(0.0, min(1.0, point[1])), 4)] + if not clean or clean[-1] != rounded: + clean.append(rounded) + return clean + + +def _preserve_local_reference_arrows(program: dict, mapping: dict[str, dict]) -> tuple[list[dict], list[str]]: + mapped_objects = list(mapping.values()) + all_objects = { + str(item.get("id")): item + for field in ("slots", "cards", "panels") + for item in program.get(field, []) + if isinstance(item, dict) and item.get("id") and isinstance(item.get("bbox_percent"), dict) + } + + def mapped_container(obj: dict | None) -> str | None: + if not obj: + return None + center_box = obj.get("bbox_percent") or {} + for mapped in mapped_objects: + if str(mapped.get("id")) == str(obj.get("id")) or _inside(center_box, mapped.get("bbox_percent") or {}): + return str(mapped.get("id")) + return None + + preserved: list[dict] = [] + replaced: list[str] = [] + for arrow in program.get("arrows", []) if isinstance(program.get("arrows"), list) else []: + if not isinstance(arrow, dict): + continue + source_id = str(arrow.get("source_id") or arrow.get("source") or "") + target_id = str(arrow.get("target_id") or arrow.get("target") or "") + arrow_id = str(arrow.get("id") or "reference_arrow") + if not source_id or not target_id: + replaced.append(arrow_id) + continue + source_container = mapped_container(all_objects.get(source_id)) + target_container = mapped_container(all_objects.get(target_id)) + if source_container and target_container and source_container != target_container: + replaced.append(arrow_id) + continue + preserved.append(arrow) + return preserved, replaced + + +def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | Path) -> tuple[dict, dict]: + out = Path(out_dir) + entities = _entities(contract) + relations, relation_source = _relations(contract) + mapping, mapping_records = _assign_entities(entities, program) + mapped_boxes = [obj["bbox_percent"] for obj in mapping.values()] + text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {"items": []} + existing = [item for item in text_program.get("items", []) if isinstance(item, dict)] + preserved = [ + item for item in existing + if not str(item.get("id") or "").startswith("semantic_text_") + and not item.get("semantic_entity_id") + and not any(_inside(item.get("bbox_percent") or {}, box) for box in mapped_boxes) + ] + semantic_text = [] + semantic_reference_regions = [] + for entity in entities: + obj = mapping.get(entity["id"]) + if not obj: + continue + box = _label_bbox(obj) + reference_text_id = f"semantic_ref_text_{entity['id']}" + center = {"x": round(box["x"] + box["w"] / 2, 4), "y": round(box["y"] + box["h"] / 2, 4)} + estimated_font_ratio = round(max(0.006, box["h"] * 0.55), 6) + semantic_text.append({ + "id": f"semantic_text_{entity['id']}", + "text": entity["label"], + "role": f"paper_{entity['role']}_label", + "target_id": obj.get("id"), + "semantic_entity_id": entity["id"], + "semantic_panel_id": entity.get("panel_id"), + "semantic_evidence_ids": entity["evidence_ids"], + "source_reference_text_id": reference_text_id, + "reference_binding": "reference_slot_semantic_overlay_geometry", + "bbox_percent": box, + "center_percent": center, + "width_percent": box["w"], + "height_percent": box["h"], + "estimated_font_ratio": estimated_font_ratio, + "font_size_pt": 10.0, + "color_hex": "#263747", + "bold": entity["role"] in {"module", "innovation"}, + "align": "center", + "font_family_guess": "Arial", + "font_weight_guess": "bold" if entity["role"] in {"module", "innovation"} else "regular", + "fit_strategy": "paper_semantic_contract_exact_label", + "editable_in": "pptx", + "visible": True, + "layer_ownership": "editable_text_layer", + }) + semantic_reference_regions.append({ + "id": reference_text_id, + "text": entity["label"], + "raw_text": entity["label"], + "role": f"paper_{entity['role']}_label", + "target_id": obj.get("id"), + "semantic_entity_id": entity["id"], + "semantic_panel_id": entity.get("panel_id"), + "evidence_ids": entity["evidence_ids"], + "bbox_percent": box, + "center_percent": center, + "width_percent": box["w"], + "height_percent": box["h"], + "estimated_font_ratio": estimated_font_ratio, + "color_hex": "#263747", + "font_family_guess": "Arial", + "font_weight_guess": "bold" if entity["role"] in {"module", "innovation"} else "regular", + "source": "reference_slot_semantic_overlay_geometry", + "editable_in": "pptx", + }) + text_program = { + **text_program, + "summary": "Editable text with paper-grounded exact labels overriding image-derived labels in mapped semantic objects.", + "semantic_authority": "paper_ground_truth", + "items": preserved + semantic_text, + } + program["text_program"] = text_program + program["text_program_path"] = "text_program.json" + program["reference_text_geometry_path"] = "reference_text_geometry.json" + program["text_alignment_report_path"] = "text_alignment_report.json" + + geometry_path = out / "reference_text_geometry.json" + if geometry_path.exists(): + reference_geometry = read_json(geometry_path) + else: + reference_geometry = { + "summary": "Reference-first text geometry with semantic overlay bindings.", + "policy": "reference_image_is_highest_authority_for_text_size_position_color_and_hierarchy", + "detection_mode": "reference_geometry_and_semantic_overlay", + "text_regions": [], + } + geometry_regions = [ + item for item in reference_geometry.get("text_regions", []) + if isinstance(item, dict) and not str(item.get("id") or "").startswith("semantic_ref_text_") + ] + reference_geometry["text_regions"] = geometry_regions + semantic_reference_regions + reference_geometry["semantic_overlay_policy"] = "exact paper terminology bound to reference-derived slot geometry" + + alignment_items = [] + for item in text_program["items"]: + alignment_items.append({ + "label_id": item.get("id"), + "source_reference_text_id": item.get("source_reference_text_id"), + "target_id": item.get("target_id"), + "center_delta_percent": 0.0, + "width_delta_percent": 0.0, + "height_delta_percent": 0.0, + "font_ratio_delta": 0.0, + "color_match": True, + "editable_in": "pptx", + "status": "pass", + }) + alignment_report = { + "summary": "Reference alignment report after applying exact paper-semantic overlay labels.", + "policy": "reference_alignment_only; semantic labels inherit their mapped reference slot geometry", + "status": "pass", + "max_center_delta_percent": 0.0, + "max_width_delta_percent": 0.0, + "max_height_delta_percent": 0.0, + "items": alignment_items, + } + + semantic_arrows = [] + skipped_relations = [] + for index, relation in enumerate(relations, start=1): + source_id = str(relation.get("source") or relation.get("source_id") or "") + target_id = str(relation.get("target") or relation.get("target_id") or "") + source, target = mapping.get(source_id), mapping.get(target_id) + if not source or not target: + skipped_relations.append({"source": source_id, "target": target_id, "reason": "unmapped_endpoint"}) + continue + semantic_arrows.append({ + "id": str(relation.get("id") or f"semantic_relation_{index:02d}"), + "source_id": str(source.get("id")), + "target_id": str(target.get("id")), + "semantic_source_id": source_id, + "semantic_target_id": target_id, + "semantic_panel_id": relation.get("panel_id"), + "relation_type": str(relation.get("type") or relation.get("relation_type") or "data_flow"), + "label": str(relation.get("label") or relation.get("statement") or ""), + "evidence_ids": list(relation.get("evidence_ids") or []), + "path_percent": _anchor(source, target), + "source_anchor": "auto", + "target_anchor": "auto", + "control_kind": "elbow_connector", + "render_style": "elbow_connector", + "route_intent": "paper_semantic_relation", + "stroke_color": "#3F5063", + "stroke_width_pt": 1.8, + "arrowhead": "triangle", + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + }) + preserved_reference_arrows, replaced_reference_arrow_ids = _preserve_local_reference_arrows(program, mapping) + if relation_source == "panel_graphs": + heuristic_defaults = [ + item for item in preserved_reference_arrows + if str(item.get("binding_source") or "").startswith("heuristic_default_") + ] + if heuristic_defaults: + replaced_reference_arrow_ids.extend(str(item.get("id") or "heuristic_default_arrow") for item in heuristic_defaults) + preserved_reference_arrows = [ + item for item in preserved_reference_arrows + if not str(item.get("binding_source") or "").startswith("heuristic_default_") + ] + if mapping: + program["arrows"] = [*preserved_reference_arrows, *semantic_arrows] + + low_confidence_mappings = [item for item in mapping_records if float(item.get("match_score") or 0) <= 0] + report = { + "summary": "Paper semantic contract application report.", + "status": "pass" if len(mapping) == len(entities) and not skipped_relations and not low_confidence_mappings else "warning", + "semantic_authority": "paper_ground_truth", + "contract_graph_source": relation_source, + "visual_authority": "generated_reference_image", + "entity_count": len(entities), + "mapped_entity_count": len(mapping), + "relation_count": len(relations), + "mapped_relation_count": len(semantic_arrows), + "preserved_reference_arrow_count": len(preserved_reference_arrows), + "replaced_reference_arrow_ids": replaced_reference_arrow_ids, + "mappings": mapping_records, + "low_confidence_mappings": low_confidence_mappings, + "skipped_relations": skipped_relations, + "policy": "exact labels and scientific relations come from the paper contract; the image supplies layout and style only", + } + write_json(out / "paper_semantic_contract.json", contract) + write_json(out / "semantic_binding_report.json", report) + write_json(out / "reference_text_geometry.json", reference_geometry) + write_json(out / "text_program.json", text_program) + write_json(out / "text_alignment_report.json", alignment_report) + write_json(out / "figure_program.json", program) + return program, report diff --git a/rfs/control_localizer.py b/rfs/control_localizer.py index 27761e3..63b8f20 100644 --- a/rfs/control_localizer.py +++ b/rfs/control_localizer.py @@ -6,6 +6,8 @@ from PIL import Image, ImageDraw +from .cv_compat import hough_line_coordinates + def _center(box: dict) -> tuple[float, float]: return float(box["x"]) + float(box["w"]) / 2, float(box["y"]) + float(box["h"]) / 2 @@ -67,7 +69,10 @@ def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str return [] candidates = [] for line in lines[:80]: - x1, y1, x2, y2 = [int(v) for v in line[0]] + coordinates = hough_line_coordinates(line) + if coordinates is None: + continue + x1, y1, x2, y2 = coordinates length = math.hypot(x2 - x1, y2 - y1) if length < min(width, height) * 0.06: continue diff --git a/rfs/cv_compat.py b/rfs/cv_compat.py new file mode 100644 index 0000000..53abbee --- /dev/null +++ b/rfs/cv_compat.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from typing import Any + + +def hough_line_coordinates(line: Any) -> tuple[int, int, int, int] | None: + """Normalize OpenCV HoughLinesP rows across array-shape variants.""" + try: + values = line.reshape(-1).tolist() if hasattr(line, "reshape") else line + while isinstance(values, (list, tuple)) and len(values) == 1 and isinstance(values[0], (list, tuple)): + values = values[0] + if not isinstance(values, (list, tuple)) or len(values) < 4: + return None + return tuple(int(value) for value in values[:4]) + except (TypeError, ValueError): + return None diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index 54b571e..e2a388b 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -12,13 +12,16 @@ from PIL import Image, ImageChops, ImageDraw +from .composition import compile_ppt, render_rebuild_preview +from .contracts import apply_paper_semantic_contract from .control_localizer import localize_reference_controls +from .evaluation import run_rebuild_visual_quality_check from .layout_planner import dominant_palette, estimate_background, plan_reference_layout from .layout_semantic_planner import plan_slot_semantics -from .ppt_compiler import compile_ppt +from .rebuild_design_planner import plan_rebuild_design from .rebuild_vlm_validation import build_rebuild_vlm_validation_report from .text_layer import build_text_layer -from .utils import ensure_dir, write_json, write_text +from .utils import ensure_dir, read_json, write_json, write_text ASSET_THRESHOLDS = { @@ -36,6 +39,10 @@ } +TEXT_ASSET_OVERLAP_THRESHOLD = 0.60 +SIMPLE_PRIMITIVE_TYPES = {"legend_marker", "simple_marker", "status_marker", "node_marker"} + + def _bbox(x: float, y: float, w: float, h: float) -> dict[str, float]: x = max(0.0, min(0.995, x)) y = max(0.0, min(0.995, y)) @@ -385,7 +392,157 @@ def _load_accepted_assets(path: Path) -> dict: return {} -def _make_asset_specs(program: dict, reference_path: Path, out: Path) -> list[dict]: +def _bbox_overlap_ratio(a: dict, b: dict) -> float: + ax1, ay1 = float(a["x"]), float(a["y"]) + ax2, ay2 = ax1 + float(a["w"]), ay1 + float(a["h"]) + bx1, by1 = float(b["x"]), float(b["y"]) + bx2, by2 = bx1 + float(b["w"]), by1 + float(b["h"]) + ix = max(0.0, min(ax2, bx2) - max(ax1, bx1)) + iy = max(0.0, min(ay2, by2) - max(ay1, by1)) + return (ix * iy) / max(float(a["w"]) * float(a["h"]), 0.000001) + + +def _text_program_boxes(program: dict) -> list[dict]: + text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {} + boxes = [] + for item in text_program.get("items", []) or []: + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + if not str(item.get("text") or "").strip(): + continue + boxes.append(item) + return boxes + + +def _normalized_reuse_key(spec: dict) -> str: + import re + + subject = str(spec.get("prompt_subject") or spec.get("semantic_role") or spec.get("slot_id") or "").lower() + subject = re.sub(r"\b(executor|branch|step|slot|icon)\s*[a-z0-9]*\b", " ", subject) + subject = re.sub(r"\b[a-d]\b|\b[0-9]+\b", " ", subject) + subject = re.sub(r"[^a-z0-9]+", " ", subject).strip() + if not subject: + subject = str(spec.get("asset_type") or spec.get("slot_type") or "generic") + return f"{spec.get('asset_type') or spec.get('slot_type') or 'generic'}::{subject}" + + +def _apply_smart_asset_policy(specs: list[dict], program: dict, out: Path, asset_policy: str) -> list[dict]: + text_items = _text_program_boxes(program) + filtered: list[dict] = [] + text_filter_report = [] + decision_report = [] + api_plan = { + "summary": "Smart API asset plan. Reference crops are only prompt/context inputs; they are not final PPT assets.", + "asset_policy": asset_policy, + "unique_api_assets": 0, + "reused_slots": 0, + "skipped_text_regions": 0, + "ppt_primitive_slots": 0, + "estimated_api_requests": 0, + "reuse_groups": [], + "recommended_api_slots": [], + } + + for spec in specs: + slot_id = str(spec["slot_id"]) + bbox = spec["slot_bbox_percent"] + overlaps = [ + { + "text_id": item.get("id"), + "text": item.get("text"), + "overlap_ratio": round(_bbox_overlap_ratio(bbox, item["bbox_percent"]), 4), + } + for item in text_items + ] + best = max(overlaps, key=lambda item: item["overlap_ratio"], default=None) + if best and float(best["overlap_ratio"]) >= TEXT_ASSET_OVERLAP_THRESHOLD: + spec["asset_decision"] = "text_region" + spec["asset_generation_mode"] = "skip_asset" + spec["asset_decision_reason"] = f"OCR/DSL text overlap {best['overlap_ratio']:.2f}; region is primarily editable text." + text_filter_report.append({ + "candidate_id": slot_id, + "rejected_as_asset": True, + "converted_to": "existing_editable_text", + "reason": spec["asset_decision_reason"], + "matched_text_id": best.get("text_id"), + "matched_text": best.get("text"), + "ocr_or_text_overlap": best.get("overlap_ratio"), + }) + decision_report.append({ + "slot_id": slot_id, + "decision": "text_region", + "needs_api": False, + "reason": spec["asset_decision_reason"], + }) + api_plan["skipped_text_regions"] += 1 + continue + if str(spec.get("asset_type") or spec.get("slot_type") or "").lower() in SIMPLE_PRIMITIVE_TYPES: + spec["asset_decision"] = "ppt_primitive" + spec["asset_generation_mode"] = "skip_asset" + spec["asset_decision_reason"] = "Simple marker should be represented by editable PPT primitives, not an image asset." + decision_report.append({ + "slot_id": slot_id, + "decision": "ppt_primitive", + "needs_api": False, + "reason": spec["asset_decision_reason"], + }) + api_plan["ppt_primitive_slots"] += 1 + continue + spec["asset_decision"] = "api_generated" + spec["asset_generation_mode"] = "api" + spec["asset_decision_reason"] = "Complex non-text visual slot; final asset should be API-generated from reference crop context." + spec["reuse_group_key"] = _normalized_reuse_key(spec) + filtered.append(spec) + + primary_by_key: dict[str, dict] = {} + reuse_groups: dict[str, list[str]] = {} + for spec in filtered: + key = str(spec["reuse_group_key"]) + reuse_groups.setdefault(key, []).append(str(spec["slot_id"])) + if key not in primary_by_key: + primary_by_key[key] = spec + decision = "api_generated" + api_plan["unique_api_assets"] += 1 + api_plan["recommended_api_slots"].append(str(spec["slot_id"])) + else: + primary = primary_by_key[key] + spec["asset_decision"] = "reuse_existing" + spec["asset_generation_mode"] = "reuse_existing" + spec["reuse_source_slot_id"] = primary["slot_id"] + spec["reuse_source_asset_id"] = primary["asset_id"] + decision = "reuse_existing" + api_plan["reused_slots"] += 1 + decision_report.append({ + "slot_id": spec["slot_id"], + "decision": decision, + "reused_asset_id": spec.get("reuse_source_asset_id"), + "reuse_group_key": key, + "needs_api": decision == "api_generated", + "reason": spec.get("asset_decision_reason"), + }) + api_plan["estimated_api_requests"] = api_plan["unique_api_assets"] + api_plan["reuse_groups"] = [ + {"reuse_group_key": key, "slots": slots} + for key, slots in sorted(reuse_groups.items()) + if len(slots) > 1 + ] + kept_slot_ids = {str(spec["slot_id"]) for spec in filtered} + program["slots"] = [slot for slot in program.get("slots", []) if str(slot.get("id")) in kept_slot_ids] + write_json(out / "asset_decision_report.json", { + "summary": "Slot-level asset decisions. In smart-api mode, final crop assets are disabled; crops are only API reference inputs.", + "asset_policy": asset_policy, + "decisions": decision_report, + }) + write_json(out / "text_asset_filter_report.json", { + "summary": "Text-like candidate slots rejected before asset generation.", + "overlap_threshold": TEXT_ASSET_OVERLAP_THRESHOLD, + "items": text_filter_report, + }) + write_json(out / "api_asset_plan.json", api_plan) + return filtered + + +def _make_asset_specs(program: dict, reference_path: Path, out: Path, asset_policy: str = "legacy") -> list[dict]: specs = [] background = str(program["canvas"].get("background") or "#FFFFFF") for slot in program["slots"]: @@ -426,6 +583,22 @@ def _make_asset_specs(program: dict, reference_path: Path, out: Path) -> list[di "no_crop_postprocess": True, } specs.append(spec) + if asset_policy == "smart-api": + return _apply_smart_asset_policy(specs, program, out, asset_policy) + write_json(out / "asset_decision_report.json", { + "summary": "Legacy asset decision report.", + "asset_policy": asset_policy, + "decisions": [{"slot_id": spec["slot_id"], "decision": asset_policy, "needs_api": asset_policy == "api"} for spec in specs], + }) + write_json(out / "text_asset_filter_report.json", { + "summary": "Text asset filter disabled in legacy asset policy.", + "items": [], + }) + write_json(out / "api_asset_plan.json", { + "summary": "API asset plan disabled in legacy asset policy.", + "asset_policy": asset_policy, + "estimated_api_requests": len(specs) if asset_policy == "api" else 0, + }) return specs @@ -439,6 +612,7 @@ def _generate_assets( economy_mode: bool, regenerate_slots: set[str], strict_asset_regeneration: bool, + asset_policy: str = "legacy", ) -> tuple[list[dict], dict]: asset_dir = ensure_dir(out / "assets") accepted = _load_accepted_assets(out / "accepted_assets.json") @@ -448,6 +622,14 @@ def _generate_assets( def run_one(spec: dict) -> dict: slot_id = spec["slot_id"] out_path = asset_dir / f"{spec['asset_id']}.png" + if spec.get("asset_decision") == "reuse_existing": + source_asset = asset_dir / f"{spec['reuse_source_asset_id']}.png" + if source_asset.exists(): + shutil.copyfile(source_asset, out_path) + metrics = _foreground_metrics(out_path, spec["background_color_hex"]) + decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=False) + return {**spec, **metrics, **decision, "status": "reused_from_group", "asset_path": str(out_path), "api_requests_attempted": 0, "economy_decision": "reuse_group_asset", "fallback_used": False} + return {**spec, "status": "reuse_source_missing", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": f"reuse source missing: {source_asset}"} locked = bool(accepted.get(slot_id, {}).get("accepted")) or bool(accepted.get(spec["asset_id"], {}).get("accepted")) if out_path.exists() and economy_mode and slot_id not in regenerate_slots and not strict_asset_regeneration: metrics = _foreground_metrics(out_path, spec["background_color_hex"]) @@ -463,10 +645,14 @@ def run_one(spec: dict) -> dict: _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]) request_count = 0 status = "placeholder" - elif asset_mode == "crop": + elif asset_mode == "crop" and asset_policy != "smart-api": shutil.copyfile(crop_path, out_path) request_count = 0 status = "reference_crop" + elif asset_mode == "crop" and asset_policy == "smart-api": + _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]) + request_count = 0 + status = "crop_disabled_by_smart_api_policy_placeholder" else: result = _api_generate_asset(spec, crop_path, out_path) request_count = int(result.get("api_requests_attempted", 1)) @@ -486,7 +672,7 @@ def run_one(spec: dict) -> dict: } except Exception as exc: last_error = str(exc) - if asset_mode == "api": + if asset_mode == "api" and asset_policy != "smart-api": try: shutil.copyfile(crop_path, out_path) metrics = _foreground_metrics(out_path, spec["background_color_hex"]) @@ -505,20 +691,47 @@ def run_one(spec: dict) -> dict: } except Exception: pass + if asset_mode == "api" and asset_policy == "smart-api": + try: + _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]) + metrics = _foreground_metrics(out_path, spec["background_color_hex"]) + decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=False) + return { + **spec, + **metrics, + **decision, + "status": "api_failed_placeholder_fallback", + "asset_path": str(out_path), + "api_requests_attempted": 1, + "attempt": attempt, + "economy_decision": "placeholder_after_api_error_crop_disabled", + "fallback_used": True, + "error": last_error, + } + except Exception: + pass return {**spec, "status": "failed", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": last_error} + primary_specs = [spec for spec in specs if spec.get("asset_decision") != "reuse_existing"] + reuse_specs = [spec for spec in specs if spec.get("asset_decision") == "reuse_existing"] workers = max(1, min(int(asset_workers), 12)) with ThreadPoolExecutor(max_workers=workers) as pool: - futures = [pool.submit(run_one, spec) for spec in specs] + futures = [pool.submit(run_one, spec) for spec in primary_specs] for future in as_completed(futures): report = future.result() reports.append(report) api_requests += int(report.get("api_requests_attempted") or 0) + for spec in reuse_specs: + report = run_one(spec) + reports.append(report) + api_requests += int(report.get("api_requests_attempted") or 0) reports.sort(key=lambda item: str(item.get("slot_id"))) program["assets"] = [{"id": spec["asset_id"], "path": f"assets/{spec['asset_id']}.png", "source": "slot_asset"} for spec in specs] summary = { "summary": "Asset generation report for reusable editable rebuild workflow.", "asset_mode": asset_mode, + "asset_policy": asset_policy, + "final_crop_assets_allowed": asset_policy != "smart-api", "economy_mode": economy_mode, "asset_retries": asset_retries, "strict_asset_regeneration": strict_asset_regeneration, @@ -531,6 +744,7 @@ def run_one(spec: dict) -> dict: "accepted_assets_file": str(out / "accepted_assets.json"), "regenerate_slots": sorted(regenerate_slots), "thresholds": {key: {"min": value[0], "max": value[1]} for key, value in ASSET_THRESHOLDS.items()}, + "asset_policy": asset_policy, "assets": [{ "slot_id": item.get("slot_id"), "slot_type": item.get("slot_type"), @@ -568,6 +782,9 @@ def _export_preview(pptx_path: Path, out: Path) -> Path | None: preview = out / "rebuild_preview.png" if platform.system() != "Windows": write_text(out / "preview_export_error.txt", "PowerPoint preview export is only available on Windows with Microsoft PowerPoint installed.") + program_path = out / "figure_program.json" + if program_path.exists(): + return render_rebuild_preview(read_json(program_path), out, preview) return None try: import win32com.client @@ -580,6 +797,9 @@ def _export_preview(pptx_path: Path, out: Path) -> Path | None: return preview except Exception as exc: write_text(out / "preview_export_error.txt", str(exc)) + program_path = out / "figure_program.json" + if program_path.exists(): + return render_rebuild_preview(read_json(program_path), out, preview) return None @@ -710,6 +930,14 @@ def rebuild_editable( vlm_layout_adapter: Callable | None = None, control_adapter: Callable | None = None, semantic_adapter: Callable | None = None, + asset_policy: str = "legacy", + design_plan_mode: str = "vlm", + design_plan_model: str | None = None, + design_adapter: Callable | None = None, + text_grouping_mode: str = "heuristic", + text_grouping_model: str | None = None, + text_grouping_adapter: Callable | None = None, + semantic_contract: dict | str | Path | None = None, ) -> dict: reference_path = Path(reference) if not reference_path.exists(): @@ -728,10 +956,33 @@ def rebuild_editable( "text_mode": text_mode, "control_mode": control_mode, "layout_mode": layout_mode, + "asset_policy": asset_policy, + "design_plan_mode": design_plan_mode, + "design_plan_model": design_plan_model, + "text_grouping_mode": text_grouping_mode, + "text_grouping_model": text_grouping_model, + "semantic_contract": str(semantic_contract) if isinstance(semantic_contract, (str, Path)) else ("inline" if semantic_contract else None), "skip_analysis": skip_analysis, "compile_only": compile_only, }) + if skip_analysis and (out_path / "reference_logic_plan.json").exists(): + design_bundle = { + "logic": _load_json_or_empty(out_path / "reference_logic_plan.json"), + "layer_plan": _load_json_or_empty(out_path / "reference_layer_plan.json"), + "generation_plan": _load_json_or_empty(out_path / "reference_generation_plan.json"), + "flow_graph": _load_json_or_empty(out_path / "reference_flow_graph.json"), + } + else: + design_bundle = plan_rebuild_design( + archived_reference, + out_path, + mode=design_plan_mode, + model=design_plan_model, + adapter=design_adapter, + fallback_on_error=True, + ) + if skip_analysis and (out_path / "reference_geometry.json").exists(): program = _program_from_contracts(out_path) reference_geometry = _load_json_or_empty(out_path / "reference_geometry.json") @@ -753,7 +1004,14 @@ def rebuild_editable( program["arrows"] = reference_controls.get("arrows", []) write_json(out_path / "reference_controls.json", reference_controls) - if text_mode != "off": + reuse_text_layer = bool( + skip_analysis + and (out_path / "reference_text_geometry.json").exists() + and (out_path / "text_program.json").exists() + ) + if reuse_text_layer: + pass + elif text_mode != "off": extractor_mode = "ocr" if text_mode == "ocr" else "heuristic" build_text_layer( archived_reference, @@ -764,6 +1022,9 @@ def rebuild_editable( ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, + text_grouping_mode=text_grouping_mode, + text_grouping_model=text_grouping_model, + text_grouping_adapter=text_grouping_adapter, ) else: write_json(out_path / "reference_text_geometry.json", {"summary": "Text extraction skipped.", "detection_mode": "off", "regions": []}) @@ -787,17 +1048,32 @@ def rebuild_editable( elif not semantic_report: semantic_report = {"summary": "Semantic planning skipped by --skip-analysis.", "semantic_vlm_status": "skipped", "slots": []} write_json(out_path / "slot_semantic_report.json", semantic_report) + semantic_binding_report = {} + if semantic_contract: + contract = read_json(semantic_contract) if isinstance(semantic_contract, (str, Path)) else semantic_contract + program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_path) write_json(out_path / "slot_inventory.json", {"summary": "Visual asset slot inventory.", "slots": program["slots"]}) - specs = _make_asset_specs(program, archived_reference, out_path) + specs = _make_asset_specs(program, archived_reference, out_path, asset_policy=asset_policy) + if asset_policy == "smart-api": + reference_geometry["slots"] = program.get("slots", []) + write_json(out_path / "reference_geometry.json", reference_geometry) + write_json(out_path / "slot_inventory.json", {"summary": "Visual asset slot inventory.", "slots": program["slots"]}) write_json(out_path / "asset_generation_specs.json", {"summary": "Slot-level asset generation specs.", "asset_mode": asset_mode, "specs": specs}) regen = set() if isinstance(regenerate_slots, str): regen = {item.strip() for item in regenerate_slots.split(",") if item.strip()} elif isinstance(regenerate_slots, list): regen = {str(item).strip() for item in regenerate_slots if str(item).strip()} - asset_reports, asset_summary = _generate_assets(specs, program, out_path, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration) + asset_reports, asset_summary = _generate_assets(specs, program, out_path, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration, asset_policy=asset_policy) vlm_validation_report = build_rebuild_vlm_validation_report(out_path, reference_geometry, reference_controls, semantic_report, asset_summary) + visual_quality_report = run_rebuild_visual_quality_check( + out_path, + program, + reference_geometry=reference_geometry, + reference_controls=reference_controls, + ownership_report=_load_json_or_empty(out_path / "text_layer_ownership_report.json"), + ) write_json(out_path / "figure_program.json", program) pptx_path = compile_ppt(program, out_path) @@ -828,25 +1104,42 @@ def rebuild_editable( "asset_workers": asset_workers, "asset_retries": asset_retries, "economy_mode": economy_mode, + "asset_policy": asset_policy, + "design_plan_mode": design_plan_mode, + "design_plan_effective_mode": design_bundle.get("logic", {}).get("effective_mode"), + "design_plan_model": design_bundle.get("logic", {}).get("model"), + "text_grouping_mode": text_grouping_mode, "api_requests_attempted": asset_summary.get("api_requests_attempted", 0), "asset_count": len(asset_reports), "slot_count": len(program.get("slots", [])), "text_count": len(program.get("text_program", {}).get("items", [])) if isinstance(program.get("text_program"), dict) else 0, "connector_count": len(program.get("arrows", [])), "text_mode": text_mode, + "text_layer_reused": reuse_text_layer, "control_mode": control_mode, "layout_mode": layout_mode, + "rebuild_visual_quality_status": visual_quality_report.get("status"), + "semantic_binding_status": semantic_binding_report.get("status") if semantic_binding_report else None, "reports": { "input_manifest": str(out_path / "input_manifest.json"), + "reference_logic_plan": str(out_path / "reference_logic_plan.json"), + "reference_layer_plan": str(out_path / "reference_layer_plan.json"), + "reference_generation_plan": str(out_path / "reference_generation_plan.json"), + "reference_flow_graph": str(out_path / "reference_flow_graph.json"), "reference_geometry": str(out_path / "reference_geometry.json"), "reference_text_geometry": str(out_path / "reference_text_geometry.json"), "reference_controls": str(out_path / "reference_controls.json"), "slot_inventory": str(out_path / "slot_inventory.json"), "slot_semantic_report": str(out_path / "slot_semantic_report.json"), "rebuild_vlm_validation_report": str(out_path / "rebuild_vlm_validation_report.json"), + "rebuild_visual_quality_report": str(out_path / "rebuild_visual_quality_report.json"), + "semantic_binding_report": str(out_path / "semantic_binding_report.json") if semantic_binding_report else None, "asset_generation_specs": str(out_path / "asset_generation_specs.json"), "asset_generation_report": str(out_path / "asset_generation_report.json"), "asset_economy_report": str(out_path / "asset_economy_report.json"), + "asset_decision_report": str(out_path / "asset_decision_report.json"), + "text_asset_filter_report": str(out_path / "text_asset_filter_report.json"), + "api_asset_plan": str(out_path / "api_asset_plan.json"), "figure_program": str(out_path / "figure_program.json"), "composition_quality_report": str(out_path / "composition_quality_report.json"), }, diff --git a/rfs/evaluation/__init__.py b/rfs/evaluation/__init__.py new file mode 100644 index 0000000..8df7580 --- /dev/null +++ b/rfs/evaluation/__init__.py @@ -0,0 +1,18 @@ +"""Deterministic and model-assisted quality evaluation APIs.""" + +from .rebuild_visual import run_rebuild_visual_quality_check +from .benchmarking import audit_paper_image_run, fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, score_benchmark_case, validate_benchmark_case +from .pdf_extraction_benchmark import run_pdf_extraction_stress_suite + +__all__ = [ + "list_benchmark_cases", + "audit_paper_image_run", + "fetch_benchmark_case", + "run_rebuild_visual_quality_check", + "run_benchmark_case", + "run_fast_benchmark_case", + "run_fast_benchmark_suite", + "run_pdf_extraction_stress_suite", + "score_benchmark_case", + "validate_benchmark_case", +] diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py new file mode 100644 index 0000000..ece8ee7 --- /dev/null +++ b/rfs/evaluation/benchmarking.py @@ -0,0 +1,1206 @@ +from __future__ import annotations + +import hashlib +import json +import math +import re +from pathlib import Path +from typing import Any + +from PIL import Image, ImageChops, ImageFilter, ImageStat +from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE +import requests + +from ..utils import ensure_dir, read_json, write_json, write_text + + +SUITES = {"paper-to-image", "image-to-ppt"} + + +def _load(path: Path) -> dict: + if not path.exists(): + return {} + try: + value = read_json(path) + except Exception: + return {} + return value if isinstance(value, dict) else {} + + +def _mean(values: list[float]) -> float: + return round(sum(values) / max(1, len(values)), 4) + + +def _clamp(value: object) -> float: + try: + return round(max(0.0, min(1.0, float(value))), 4) + except Exception: + return 0.0 + + +def _threshold_failures(metrics: dict, thresholds: dict) -> list[str]: + maximum_metrics = { + "hallucination_count", + "forbidden_content_count", + "plan_forbidden_content_count", + "full_slide_image_count", + "blocking_visual_issue_count", + "evidence_conflict_count", + "unsupported_visual_claim_count", + "missing_paper_priority_count", + } + failures = [] + for key, threshold in thresholds.items(): + value = metrics.get(key) + if not isinstance(value, (int, float)) or not isinstance(threshold, (int, float)): + continue + if key in maximum_metrics and float(value) > float(threshold): + failures.append(f"{key}={value} above maximum {threshold}") + elif key not in maximum_metrics and float(value) < float(threshold): + failures.append(f"{key}={value} below minimum {threshold}") + return failures + + +def _normalized_text(value: object) -> str: + return re.sub(r"[^a-z0-9]+", "", str(value or "").casefold()) + + +def _normalized_label_variants(value: object) -> set[str]: + label = str(value or "").strip() + variants = {_normalized_text(label)} + without_suffix = re.sub(r"\s*\([^()]{1,24}\)\s*$", "", label).strip() + if without_suffix: + variants.add(_normalized_text(without_suffix)) + return {item for item in variants if item} + + +def _score_planning_contract(expected: dict, specification: dict) -> dict[str, Any]: + if not specification: + return {"available": False} + actual_entities = [] + for field in ("inputs", "modules", "outputs", "innovations", "must_show"): + for index, item in enumerate(specification.get(field, []) if isinstance(specification.get(field), list) else []): + if not isinstance(item, dict): + continue + label = next((str(item.get(key) or "").strip() for key in ("visible_label", "name", "text", "statement", "label", "id") if str(item.get(key) or "").strip()), "") + actual_entities.append({"id": str(item.get("id") or f"{field}_{index}"), "label": label, "normalized": _normalized_text(label), "normalized_variants": _normalized_label_variants(label), "field": field}) + expanded_relations = set() + for index, item in enumerate(specification.get("relations", []) if isinstance(specification.get("relations"), list) else []): + if not isinstance(item, dict): + continue + source = str(item.get("source") or item.get("source_id") or "") + target = str(item.get("target") or item.get("target_id") or "") + if source and target: + expanded_relations.add((source, target)) + label = str(item.get("label") or "").strip() + if source and target and label: + label_id = f"relation_label_{index}" + actual_entities.append({"id": label_id, "label": label, "normalized": _normalized_text(label), "normalized_variants": _normalized_label_variants(label), "field": "relation_label"}) + expanded_relations.add((source, label_id)) + expanded_relations.add((label_id, target)) + actual_relations = expanded_relations + adjacency: dict[str, set[str]] = {} + for source, target in actual_relations: + adjacency.setdefault(source, set()).add(target) + + def connected(source: str, target: str, max_hops: int = 2) -> bool: + if (source, target) in actual_relations: + return True + frontier = {source} + visited = {source} + for _ in range(max_hops): + frontier = {neighbor for node in frontier for neighbor in adjacency.get(node, set()) if neighbor not in visited} + if target in frontier: + return True + visited.update(frontier) + return False + + expected_entities = [item for item in expected.get("entities", []) if isinstance(item, dict) and item.get("required", True)] + mapping: dict[str, str] = {} + matched = [] + used_actual_ids: set[str] = set() + candidate_scores: dict[str, dict[str, int]] = {} + for item in expected_entities: + expected_id = str(item.get("id")) + primary = _normalized_text(item.get("label")) + aliases = [_normalized_text(value) for value in item.get("aliases", []) if _normalized_text(value)] + def match_score(actual: dict[str, Any]) -> int: + value = actual["normalized"] + variants = actual.get("normalized_variants") or {value} + if not value: + return 0 + bonus = 2 if actual.get("field") in {"inputs", "modules", "outputs", "innovations"} else 1 if actual.get("field") == "relation_label" else 0 + if primary and primary in variants: + return 5 + bonus + if any(alias in variants for alias in aliases): + return 4 + bonus + if any(alias in value or value in alias for alias in aliases): + return 3 + bonus + if primary and (primary in value or value in primary): + return 1 + bonus + return 0 + candidate_scores[expected_id] = {actual["id"]: match_score(actual) for actual in actual_entities} + ranked = sorted( + ((score, actual) for actual in actual_entities if actual["id"] not in used_actual_ids and (score := candidate_scores[expected_id][actual["id"]]) > 0), + key=lambda pair: pair[0], + reverse=True, + ) + match = ranked[0][1] if ranked and ranked[0][0] > 0 else None + if match: + mapping[expected_id] = match["id"] + used_actual_ids.add(match["id"]) + matched.append(expected_id) + expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict) and item.get("required", True)] + for expected_id, current_actual in list(mapping.items()): + blocked = set(mapping.values()) - {current_actual} + + def relation_support(actual_id: str) -> int: + support = 0 + for relation in expected_relations: + source_expected = str(relation.get("source")) + target_expected = str(relation.get("target")) + if source_expected == expected_id and target_expected in mapping: + support += int(connected(actual_id, mapping[target_expected])) + elif target_expected == expected_id and source_expected in mapping: + support += int(connected(mapping[source_expected], actual_id)) + return support + + alternatives = [ + actual_id + for actual_id, score in candidate_scores.get(expected_id, {}).items() + if score > 0 and actual_id not in blocked + ] + if alternatives: + mapping[expected_id] = max( + alternatives, + key=lambda actual_id: ( + candidate_scores[expected_id][actual_id], + relation_support(actual_id), + actual_id == current_actual, + ), + ) + relation_matches = 0 + matched_relations = [] + missing_relations = [] + for relation in expected_relations: + source = mapping.get(str(relation.get("source"))) + target = mapping.get(str(relation.get("target"))) + if source and target and connected(source, target): + relation_matches += 1 + matched_relations.append(f"{relation.get('source')}->{relation.get('target')}") + else: + missing_relations.append(f"{relation.get('source')}->{relation.get('target')}") + actual_text = " ".join(item["label"] for item in actual_entities).casefold() + forbidden = [label for label in expected.get("forbidden_labels", []) if str(label).casefold() in actual_text] + return { + "available": True, + "entity_recall": _clamp(len(matched) / max(1, len(expected_entities))), + "relation_recall": _clamp(relation_matches / max(1, len(expected_relations))), + "matched_entity_ids": matched, + "entity_mapping": mapping, + "missing_entity_ids": [str(item.get("id")) for item in expected_entities if str(item.get("id")) not in matched], + "matched_relations": matched_relations, + "missing_relations": missing_relations, + "forbidden_labels_found": forbidden, + } + + +def validate_benchmark_case(case_dir: str | Path) -> dict[str, Any]: + root = Path(case_dir).resolve() + case = _load(root / "case.json") + errors: list[str] = [] + warnings: list[str] = [] + suite = str(case.get("suite") or "") + if suite not in SUITES: + errors.append(f"case.json suite must be one of {sorted(SUITES)}") + if not str(case.get("case_id") or "").strip(): + errors.append("case.json requires case_id") + if not isinstance(case.get("thresholds"), dict): + errors.append("case.json requires thresholds object") + + required_files = ["case.json"] + if suite == "paper-to-image": + paper_relative = str(case.get("paper") or "paper.md") + required_files.append(str(case.get("expected_semantics") or "expected_semantics.json")) + if not (root / paper_relative).exists(): + source_path = root / str(case.get("source") or "source.json") + if source_path.exists(): + warnings.append(f"paper input not fetched yet: {paper_relative}") + else: + errors.append(f"missing required file: {paper_relative}; no source.json is available") + expected = _load(root / str(case.get("expected_semantics") or "expected_semantics.json")) + if not expected.get("entities"): + errors.append("expected_semantics.json requires entities") + if not isinstance(expected.get("relations"), list): + errors.append("expected_semantics.json requires relations list") + if not isinstance(expected.get("forbidden_labels", []), list): + errors.append("expected_semantics.json forbidden_labels must be a list") + elif suite == "image-to-ppt": + required_files.extend([str(case.get("reference_image") or "reference.png"), str(case.get("expected_objects") or "expected_objects.json")]) + expected = _load(root / str(case.get("expected_objects") or "expected_objects.json")) + if not isinstance(expected.get("objects"), list): + errors.append("expected_objects.json requires objects list") + if not isinstance(expected.get("relations", []), list): + errors.append("expected_objects.json relations must be a list") + + for relative in required_files: + if relative and not (root / relative).exists(): + errors.append(f"missing required file: {relative}") + references = case.get("positive_references", []) + case.get("negative_references", []) + for relative in references: + if not (root / str(relative)).exists(): + warnings.append(f"missing optional reference: {relative}") + + return { + "summary": "Benchmark case validation report.", + "ok": not errors, + "case_dir": str(root), + "case_id": case.get("case_id"), + "suite": suite, + "errors": errors, + "warnings": warnings, + } + + +def fetch_benchmark_case(case_dir: str | Path, force: bool = False) -> dict[str, Any]: + root = Path(case_dir).resolve() + case = _load(root / "case.json") + source_path = root / str(case.get("source") or "source.json") + source = _load(source_path) + if str(case.get("suite") or "") != "paper-to-image": + return {"summary": "Benchmark source fetch is only required for paper-to-image cases.", "ok": True, "case_dir": str(root), "status": "not_required"} + if not source: + return {"summary": "Benchmark source metadata is missing.", "ok": False, "case_dir": str(root), "status": "missing_source", "error": str(source_path)} + target = root / str(case.get("paper") or "inputs/paper.pdf") + if target.exists() and not force: + return { + "summary": "Benchmark paper source already exists.", + "ok": True, + "case_dir": str(root), + "status": "reused", + "paper": str(target), + "sha256": hashlib.sha256(target.read_bytes()).hexdigest(), + } + urls = source.get("paper_urls") if isinstance(source.get("paper_urls"), list) else [] + errors = [] + for url in urls: + try: + response = requests.get(str(url), headers={"User-Agent": "ResearchFigureStudio benchmark fetch/0.1"}, timeout=120) + response.raise_for_status() + content = response.content + if not content.startswith(b"%PDF"): + raise ValueError("downloaded response is not a PDF") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + digest = hashlib.sha256(content).hexdigest() + report = { + "summary": "Benchmark paper source fetched for local use.", + "ok": True, + "case_dir": str(root), + "status": "downloaded", + "paper": str(target), + "source_url": str(url), + "bytes": len(content), + "sha256": digest, + "license_note": source.get("license_note"), + } + write_json(target.parent / "fetch_report.json", report) + return report + except Exception as exc: + errors.append({"url": str(url), "error": str(exc)}) + return {"summary": "Unable to fetch benchmark paper source.", "ok": False, "case_dir": str(root), "status": "failed", "errors": errors} + + +def list_benchmark_cases(benchmarks_root: str | Path, suite: str | None = None) -> dict[str, Any]: + root = Path(benchmarks_root).resolve() + cases = [] + suites = [suite] if suite else sorted(SUITES) + for suite_name in suites: + case_root = root / suite_name / "cases" + if not case_root.exists(): + continue + for case_dir in sorted(path for path in case_root.iterdir() if path.is_dir()): + validation = validate_benchmark_case(case_dir) + cases.append({ + "case_id": validation.get("case_id") or case_dir.name, + "suite": suite_name, + "case_dir": str(case_dir), + "valid": validation["ok"], + "errors": validation["errors"], + }) + return {"summary": "Available ResearchFigureStudio benchmark cases.", "benchmarks_root": str(root), "cases": cases} + + +def _select_candidate_review(run: Path) -> dict: + report = _load(run / "candidate_review.json") + candidates = [item for item in report.get("candidates", []) if isinstance(item, dict)] + selected_id = report.get("selected_candidate_id") + selected = next((item for item in candidates if item.get("candidate_id") == selected_id), None) + if not selected and candidates: + selected = max(candidates, key=lambda item: float(item.get("score") or 0.0)) + return selected.get("review", {}) if isinstance(selected, dict) else {} + + +def _unexpected_visible_labels(review: dict, specification: dict) -> list[str]: + ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} + if "unsupported_unexpected_labels" in ocr: + return [str(value) for value in ocr.get("unsupported_unexpected_labels", []) if str(value).strip()] + if ocr.get("unexpected_labels"): + return [str(value) for value in ocr.get("unexpected_labels", []) if str(value).strip()] + allowed = { + _normalized_text(value) + for value in specification.get("required_labels", []) if _normalized_text(value) + } + allowed.update( + _normalized_text(value) + for value in specification.get("repeatable_labels", []) if _normalized_text(value) + ) + allowed.update( + _normalized_text(item.get("label")) + for item in specification.get("relations", []) if isinstance(item, dict) and _normalized_text(item.get("label")) + ) + detected = ocr.get("detected_labels", []) + return [str(value) for value in detected if _normalized_text(value) and _normalized_text(value) not in allowed] + + +def _audit_candidate_stability(candidate_report: dict, specification: dict) -> dict[str, Any]: + candidates = [ + item for item in candidate_report.get("candidates", []) + if isinstance(item, dict) and str(item.get("candidate_id") or "").startswith("candidate_") and not item.get("repair_source") + ] + requested = int(candidate_report.get("requested_candidates") or len(candidates)) + if requested < 2: + return {"summary": "Repeated-generation stability audit has insufficient samples.", "seed_count": requested, "available": False} + records = [] + pass_count = 0 + for item in candidates: + review = item.get("review", {}) if isinstance(item.get("review"), dict) else {} + unexpected = _unexpected_visible_labels(review, specification) + effective_pass = bool(item.get("production_pass")) and not unexpected + pass_count += int(effective_pass) + records.append({ + "candidate_id": item.get("candidate_id"), + "reported_production_pass": bool(item.get("production_pass")), + "effective_production_pass": effective_pass, + "unexpected_labels": unexpected, + }) + pass_rate = round(pass_count / max(1, requested), 4) + return { + "summary": "Repeated-generation stability audited against the normalized visible-label whitelist.", + "available": True, + "seed_count": requested, + "production_pass_count": pass_count, + "production_pass_rate": pass_rate, + "stable": pass_rate >= 0.8, + "candidates": records, + } + + +def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, Any]: + case_root, run = Path(case_dir).resolve(), Path(run_dir).resolve() + case = _load(case_root / "case.json") + expected = _load(case_root / str(case.get("expected_semantics") or "expected_semantics.json")) + candidate_report = _load(run / "candidate_review.json") + review = _select_candidate_review(run) + candidate_records = [item for item in candidate_report.get("candidates", []) if isinstance(item, dict)] + selected_candidate_id = candidate_report.get("selected_candidate_id") + selected_candidate_record = next((item for item in candidate_records if item.get("candidate_id") == selected_candidate_id), None) + if selected_candidate_record is None and candidate_records: + selected_candidate_record = max(candidate_records, key=lambda item: float(item.get("score") or 0.0)) + deterministic_overlay = bool(candidate_report.get("deterministic_overlay")) + overlay_report = selected_candidate_record.get("overlay_report", {}) if isinstance(selected_candidate_record, dict) and isinstance(selected_candidate_record.get("overlay_report"), dict) else {} + geometry_report = selected_candidate_record.get("geometry_alignment_report", {}) if isinstance(selected_candidate_record, dict) and isinstance(selected_candidate_record.get("geometry_alignment_report"), dict) else {} + layout_blueprint = _load(run / "layout_blueprint.json") + semantic_plan = layout_blueprint.get("semantic_plan", {}) if isinstance(layout_blueprint.get("semantic_plan"), dict) else {} + route_quality = semantic_plan.get("route_quality", {}) if isinstance(semantic_plan.get("route_quality"), dict) else {} + benchmark_review = _load(run / "benchmark_review.json") + if benchmark_review: + review = {**review, **benchmark_review} + + scientific = review.get("scientific", {}) if isinstance(review.get("scientific"), dict) else {} + paper_alignment = review.get("paper_alignment", {}) if isinstance(review.get("paper_alignment"), dict) else {} + topology = review.get("topology", {}) if isinstance(review.get("topology"), dict) else {} + ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} + aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} + clarity = review.get("clarity", {}) if isinstance(review.get("clarity"), dict) else {} + information = review.get("information", {}) if isinstance(review.get("information"), dict) else {} + stability = review.get("stability", {}) if isinstance(review.get("stability"), dict) else {} + if not stability: + stability = _load(run / "stability_report.json") or (candidate_report.get("stability", {}) if isinstance(candidate_report.get("stability"), dict) else {}) + if isinstance(benchmark_review.get("stability"), dict): + stability = benchmark_review["stability"] + specification = _load(run / "figure_specification.json") + review_ground_truth = _load(run / "review_ground_truth.json") + repair_history = _load(run / "repair_history.json") + planning_contract = _score_planning_contract(expected, specification) + stability_audit = _audit_candidate_stability(candidate_report, specification) + + expected_entities = [item for item in expected.get("entities", []) if isinstance(item, dict) and item.get("required", True)] + expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict) and item.get("required", True)] + missing_entities = scientific.get("missing_modules", []) or benchmark_review.get("missing_entities", []) + missing_relations = scientific.get("missing_relations", []) or benchmark_review.get("missing_relations", []) + invented = scientific.get("invented_items", []) or benchmark_review.get("invented_items", []) + forbidden_found = ocr.get("forbidden_labels_found", []) or benchmark_review.get("forbidden_labels_found", []) + missing_labels = ocr.get("missing_labels", []) or benchmark_review.get("missing_labels", []) + misspelled = ocr.get("misspelled_labels", []) or benchmark_review.get("misspelled_labels", []) + + entity_recall = _clamp((len(expected_entities) - len(missing_entities)) / max(1, len(expected_entities))) + relation_recall = _clamp((len(expected_relations) - len(missing_relations)) / max(1, len(expected_relations))) + exact_label_rate = _clamp((len(expected_entities) - len(missing_labels) - len(misspelled)) / max(1, len(expected_entities))) + scientific_score = _clamp(scientific.get("score", _mean([entity_recall, relation_recall]))) + aesthetic_dimensions = [aesthetic.get(key) for key in ("hierarchy", "balance", "whitespace", "color", "icon_consistency", "readability") if aesthetic.get(key) is not None] + aesthetic_score = _clamp(aesthetic.get("score", _mean([_clamp(value) for value in aesthetic_dimensions]))) + clarity_score = _clamp(clarity.get("score", aesthetic.get("readability", 0.0))) + information_score = _clamp(information.get("score", information.get("coverage", scientific_score))) + stability_score = _clamp( + stability_audit.get("production_pass_rate") + if stability_audit.get("available") + else stability.get("production_pass_rate", stability.get("score", 0.0)) + ) + paper_alignment_score = _clamp(paper_alignment.get("score")) if paper_alignment else None + topology_score = _clamp(topology.get("score")) if topology else None + required_priority_ids = { + str(item.get("id") or "").strip() + for item in review_ground_truth.get("priorities", []) + if isinstance(item, dict) and item.get("required") and str(item.get("id") or "").strip() + } + supported_priority_ids = { + str(item.get("id") or "").strip() + for item in paper_alignment.get("priorities", []) + if isinstance(item, dict) + and str(item.get("status") or "").strip().casefold() in {"supported", "present", "correct"} + and str(item.get("id") or "").strip() + } + evidence_priority_coverage = ( + _clamp(len(required_priority_ids & supported_priority_ids) / max(1, len(required_priority_ids))) + if required_priority_ids else None + ) + evidence_conflicts = list(paper_alignment.get("evidence_conflicts", [])) if paper_alignment else [] + unsupported_visual_claims = list(paper_alignment.get("unsupported_visual_claims", [])) if paper_alignment else [] + missing_priorities = list(paper_alignment.get("missing_priorities", [])) if paper_alignment else [] + repair_rounds = [item for item in repair_history.get("rounds", []) if isinstance(item, dict)] + accepted_repairs = [item for item in repair_rounds if item.get("accepted")] + rejected_repairs = [item for item in repair_rounds if not item.get("accepted")] + regression_rejections = [ + item for item in rejected_repairs + if any("decreased" in str(reason).casefold() or "pass to fail" in str(reason).casefold() for reason in item.get("rejection_reasons", [])) + ] + unexpected_labels = _unexpected_visible_labels(review, specification) + hard_failures = list(review.get("hard_errors", [])) + if not scientific or not aesthetic or scientific.get("engineering_only") or aesthetic.get("engineering_only"): + hard_failures.append("frozen benchmark judge did not provide scientific and aesthetic sections") + if invented: + hard_failures.append("invented scientific content") + if forbidden_found: + hard_failures.append("forbidden labels found") + if relation_recall < 1.0: + hard_failures.append("required relations missing or incorrect") + if exact_label_rate < 1.0: + hard_failures.append("required labels missing or misspelled") + if unexpected_labels: + hard_failures.append("unexpected visible labels outside the normalized figure contract") + evidence_review_required = bool(review_ground_truth.get("strict_evidence_required")) + if evidence_review_required and not paper_alignment: + hard_failures.append("paper-evidence alignment review is missing") + if paper_alignment and (not paper_alignment.get("passed") or missing_priorities): + hard_failures.append("required paper priorities are missing or unverified") + if evidence_conflicts: + hard_failures.append("candidate conflicts with quoted paper evidence") + if unsupported_visual_claims: + hard_failures.append("candidate contains unsupported visual claims") + if topology and not topology.get("passed", False): + hard_failures.append("visible connector topology failed review") + if stability_audit.get("available") and not stability_audit.get("stable"): + hard_failures.append("repeated generation stability below production threshold") + plan_entity_min = float(case.get("thresholds", {}).get("plan_entity_recall", 1.0)) + plan_relation_min = float(case.get("thresholds", {}).get("plan_relation_recall", 1.0)) + if planning_contract.get("available") and float(planning_contract.get("entity_recall", 0.0)) < plan_entity_min: + hard_failures.append("paper planning missed required benchmark entities") + if planning_contract.get("available") and float(planning_contract.get("relation_recall", 0.0)) < plan_relation_min: + hard_failures.append("paper planning missed required benchmark relations") + if planning_contract.get("forbidden_labels_found"): + hard_failures.append("paper planning introduced forbidden benchmark labels") + if deterministic_overlay and not overlay_report: + hard_failures.append("deterministic overlay report is missing for the selected candidate") + if deterministic_overlay and not geometry_report: + hard_failures.append("visual substrate geometry report is missing for the selected candidate") + if deterministic_overlay and geometry_report and not ( + geometry_report.get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("unique_node_card_assignment") + and not geometry_report.get("post_normalization_alignment", {}).get("semantic_crop_used") + ): + hard_failures.append("visual substrate geometry normalization gate failed") + if deterministic_overlay and overlay_report and not overlay_report.get("passed", False): + hard_failures.append("deterministic overlay did not compile every required label and connector") + if deterministic_overlay and route_quality and not route_quality.get("passed", False): + hard_failures.append("deterministic connector routing quality gate failed") + + metrics = { + "entity_recall": entity_recall, + "relation_recall": relation_recall, + "exact_label_rate": exact_label_rate, + "scientific_score": scientific_score, + "information_score": information_score, + "clarity_score": clarity_score, + "aesthetic_score": aesthetic_score, + "stability_score": stability_score, + "paper_alignment_score": paper_alignment_score, + "evidence_priority_coverage": evidence_priority_coverage, + "evidence_conflict_count": len(evidence_conflicts), + "unsupported_visual_claim_count": len(unsupported_visual_claims), + "missing_paper_priority_count": len(missing_priorities), + "topology_score": topology_score, + "repair_attempt_count": len(repair_rounds), + "accepted_repair_count": len(accepted_repairs), + "rejected_repair_count": len(rejected_repairs), + "repair_regression_rejection_count": len(regression_rejections), + "hallucination_count": len(invented), + "forbidden_content_count": len(forbidden_found), + "unexpected_label_count": len(unexpected_labels), + "substrate_geometry_pass": bool(geometry_report.get("passed")) if deterministic_overlay else None, + "plan_entity_recall": planning_contract.get("entity_recall"), + "plan_relation_recall": planning_contract.get("relation_recall"), + "plan_forbidden_content_count": len(planning_contract.get("forbidden_labels_found", [])), + "deterministic_overlay": deterministic_overlay, + "overlay_label_count": overlay_report.get("label_count") if overlay_report else None, + "overlay_connector_count": overlay_report.get("connector_count") if overlay_report else None, + "route_crossing_count": route_quality.get("crossing_count") if route_quality else None, + "route_shared_segment_overlap": route_quality.get("shared_segment_overlap") if route_quality else None, + "route_node_obstacle_intersections": route_quality.get("node_obstacle_intersections") if route_quality else None, + } + thresholds = case.get("thresholds", {}) + threshold_failures = _threshold_failures(metrics, thresholds) + if paper_alignment and topology: + total_score = round(0.28 * scientific_score + 0.20 * float(paper_alignment_score or 0.0) + 0.12 * information_score + 0.12 * clarity_score + 0.10 * float(topology_score or 0.0) + 0.13 * aesthetic_score + 0.05 * stability_score, 4) + else: + total_score = round(0.40 * scientific_score + 0.15 * information_score + 0.15 * clarity_score + 0.20 * aesthetic_score + 0.10 * stability_score, 4) + return { + "summary": "Paper-to-image benchmark score.", + "suite": "paper-to-image", + "case_id": case.get("case_id"), + "case_dir": str(case_root), + "run_dir": str(run), + "metrics": metrics, + "planning_contract": planning_contract, + "deterministic_overlay": { + "enabled": deterministic_overlay, + "selected_candidate_id": selected_candidate_record.get("candidate_id") if isinstance(selected_candidate_record, dict) else None, + "overlay_report": overlay_report, + "route_quality": route_quality, + }, + "paper_alignment": paper_alignment, + "repair_history": repair_history, + "stability_audit": stability_audit, + "total_score": total_score, + "hard_failures": sorted(set(hard_failures)), + "threshold_failures": threshold_failures, + "passed": not hard_failures and not threshold_failures, + "policy": "paper-evidence alignment, scientific content, topology, terminology, and hallucination failures cannot be offset by aesthetics", + } + + +def audit_paper_image_run( + run_dir: str | Path, + case_dir: str | Path | None = None, + candidate_image: str | Path | None = None, + review_model: str | None = None, + ocr_engine: str = "off", + ocr_lang: str = "en_ch", + critic_adapter=None, + topology_adapter=None, +) -> dict[str, Any]: + """Re-scan an existing generated image against its paper evidence contract. + + This deliberately does not regenerate the image. It upgrades legacy runs to + the current frozen review gates and leaves an actionable repair plan when + evidence, topology, OCR, or aesthetic checks fail. + """ + + run = Path(run_dir).resolve() + if not run.exists(): + raise FileNotFoundError(f"Paper-to-image run does not exist: {run}") + from ..paper_to_image.critics import review_candidate + from ..paper_to_image.review_contract import build_repair_plan, compile_review_ground_truth + + plan = {} + for name in ("paper_summary", "figure_specification", "design_plan", "layout_intent", "visual_metaphors", "style_plan"): + value = _load(run / f"{name}.json") + if value: + plan[name] = value + if not isinstance(plan.get("figure_specification"), dict): + raise ValueError(f"figure_specification.json is missing or invalid in {run}") + + document_model = _load(run / "document_model.json") + key_evidence = _load(run / "key_evidence.json") + evidence_records = document_model.get("evidence", []) if isinstance(document_model.get("evidence"), list) else key_evidence.get("evidence", []) + ground_truth = compile_review_ground_truth(plan, evidence_records) + plan["review_ground_truth"] = ground_truth + write_json(run / "review_ground_truth.json", ground_truth) + + candidate_report = _load(run / "candidate_review.json") + selected_id = str(candidate_report.get("selected_candidate_id") or "").strip() + candidate_records = [item for item in candidate_report.get("candidates", []) if isinstance(item, dict) and str(item.get("path") or "").strip()] + selected_record = next((item for item in candidate_records if str(item.get("candidate_id")) == selected_id), None) + candidate = Path(candidate_image).resolve() if candidate_image else run / "selected_image.png" + if candidate_image: + selected_id = "explicit_reaudit_candidate" + elif not candidate.exists(): + if selected_record is None and candidate_records: + selected_record = max(candidate_records, key=lambda item: (bool(item.get("production_pass")), float(item.get("score") or 0.0))) + if selected_record is None: + raise FileNotFoundError(f"No reviewable candidate image is recorded in {run}") + selected_id = str(selected_record.get("candidate_id") or "reaudit_candidate") + candidate = Path(str(selected_record.get("path"))) + if not candidate.is_absolute(): + candidate = run / candidate + blueprint = run / "layout_blueprint.png" + if not candidate.exists(): + raise FileNotFoundError(f"Selected candidate image is missing in {run}") + if not blueprint.exists(): + raise FileNotFoundError(f"layout_blueprint.png is missing in {run}") + template = _load(run / "selected_template.json") + preferences = _load(run / "preferences.json") + acceptable_ratios = [ + str(value) for value in (preferences.get("requested_aspect_ratio"), preferences.get("aspect_ratio")) + if value and str(value) != "auto" + ] + generation_parameters = _load(run / "generation_parameters.json") + layout_blueprint = _load(run / "layout_blueprint.json") + semantic_plan = layout_blueprint.get("semantic_plan", {}) if isinstance(layout_blueprint.get("semantic_plan"), dict) else {} + route_quality = semantic_plan.get("route_quality", {}) if isinstance(semantic_plan.get("route_quality"), dict) else {} + overlay_report = selected_record.get("overlay_report", {}) if isinstance(selected_record, dict) and isinstance(selected_record.get("overlay_report"), dict) else {} + geometry_report = selected_record.get("geometry_alignment_report", {}) if isinstance(selected_record, dict) and isinstance(selected_record.get("geometry_alignment_report"), dict) else {} + review = review_candidate( + candidate, + blueprint, + plan, + template, + mode="vlm", + model=review_model, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + critic_adapter=critic_adapter, + topology_adapter=topology_adapter, + acceptable_aspect_ratios=acceptable_ratios, + require_visual_enrichment=str(generation_parameters.get("asset_mode") or "image2") == "image2", + ) + repair_plan = build_repair_plan(review, selected_id, 0) + write_json(run / "benchmark_review.json", review) + write_json(run / "paper_alignment_report.json", { + "summary": "Independent paper-evidence re-audit for the selected complete image.", + "candidates": [{"candidate_id": selected_id, **review.get("paper_alignment", {})}], + }) + write_json(run / "reaudit_repair_plan.json", repair_plan) + benchmark = score_benchmark_case(case_dir, run, run) if case_dir else None + result = { + "summary": "Existing complete image re-audited against quoted paper evidence and current visual gates.", + "ok": True, + "run_dir": str(run), + "candidate_id": selected_id, + "candidate": str(candidate), + "review_ground_truth": str(run / "review_ground_truth.json"), + "required_priority_count": ground_truth.get("required_priority_count"), + "grounded_required_priority_count": ground_truth.get("grounded_required_priority_count"), + "production_pass": bool( + review.get("production_pass") + and ( + not candidate_report.get("deterministic_overlay") + or geometry_report.get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("unique_node_card_assignment") + and not geometry_report.get("post_normalization_alignment", {}).get("semantic_crop_used") + ) + ), + "overall_score": review.get("overall_score"), + "paper_alignment_score": review.get("paper_alignment", {}).get("score"), + "topology_score": review.get("topology", {}).get("score"), + "deterministic_overlay": bool(candidate_report.get("deterministic_overlay")), + "overlay_report": overlay_report, + "geometry_alignment_report": geometry_report, + "route_quality": route_quality, + "repair_item_count": repair_plan.get("item_count"), + "benchmark": benchmark, + } + write_json(run / "paper_image_reaudit.json", result) + return result + + +def _image_similarity(reference: Path, preview: Path) -> dict[str, float | None]: + if not reference.exists() or not preview.exists(): + return {"pixel_similarity": None, "edge_similarity": None, "color_similarity": None} + with Image.open(reference) as ref_image, Image.open(preview) as out_image: + ref = ref_image.convert("RGB").resize((512, 512), Image.Resampling.LANCZOS) + out = out_image.convert("RGB").resize((512, 512), Image.Resampling.LANCZOS) + difference = ImageChops.difference(ref, out) + rms = math.sqrt(sum(value * value for value in ImageStat.Stat(difference).rms) / 3.0) + pixel_similarity = _clamp(1.0 - rms / 255.0) + ref_edge = ref.convert("L").filter(ImageFilter.FIND_EDGES).point(lambda value: 255 if value > 32 else 0) + out_edge = out.convert("L").filter(ImageFilter.FIND_EDGES).point(lambda value: 255 if value > 32 else 0) + edge_diff = ImageChops.difference(ref_edge, out_edge) + edge_similarity = _clamp(1.0 - ImageStat.Stat(edge_diff).mean[0] / 255.0) + ref_mean, out_mean = ImageStat.Stat(ref).mean, ImageStat.Stat(out).mean + color_error = sum(abs(a - b) for a, b in zip(ref_mean, out_mean)) / (3.0 * 255.0) + return {"pixel_similarity": pixel_similarity, "edge_similarity": edge_similarity, "color_similarity": _clamp(1.0 - color_error)} + + +def _ppt_editability(pptx_path: Path) -> dict[str, Any]: + if not pptx_path.exists(): + return {"status": "missing", "full_slide_image_count": 0, "text_shape_count": 0, "connector_count": 0, "shape_count": 0} + presentation = Presentation(str(pptx_path)) + full_slide_images = 0 + text_shapes = 0 + connectors = 0 + shapes = 0 + slide_width, slide_height = float(presentation.slide_width), float(presentation.slide_height) + for slide in presentation.slides: + for shape in slide.shapes: + shapes += 1 + if getattr(shape, "has_text_frame", False) and str(getattr(shape, "text", "")).strip(): + text_shapes += 1 + if shape.shape_type == MSO_SHAPE_TYPE.LINE: + connectors += 1 + if shape.shape_type == MSO_SHAPE_TYPE.PICTURE: + coverage = float(shape.width) * float(shape.height) / max(1.0, slide_width * slide_height) + if coverage >= 0.90: + full_slide_images += 1 + return { + "status": "pass" if full_slide_images == 0 else "blocked", + "full_slide_image_count": full_slide_images, + "text_shape_count": text_shapes, + "connector_count": connectors, + "shape_count": shapes, + } + + +def _bbox_iou(left: dict, right: dict) -> float: + lx0, ly0 = float(left.get("x", 0)), float(left.get("y", 0)) + lx1, ly1 = lx0 + float(left.get("w", 0)), ly0 + float(left.get("h", 0)) + rx0, ry0 = float(right.get("x", 0)), float(right.get("y", 0)) + rx1, ry1 = rx0 + float(right.get("w", 0)), ry0 + float(right.get("h", 0)) + intersection = max(0.0, min(lx1, rx1) - max(lx0, rx0)) * max(0.0, min(ly1, ry1) - max(ly0, ry0)) + union = max(0.000001, float(left.get("w", 0)) * float(left.get("h", 0)) + float(right.get("w", 0)) * float(right.get("h", 0)) - intersection) + return intersection / union + + +def _center_error(left: dict, right: dict) -> float: + lx = float(left.get("x", 0)) + float(left.get("w", 0)) / 2 + ly = float(left.get("y", 0)) + float(left.get("h", 0)) / 2 + rx = float(right.get("x", 0)) + float(right.get("w", 0)) / 2 + ry = float(right.get("y", 0)) + float(right.get("h", 0)) / 2 + return math.sqrt((lx - rx) ** 2 + (ly - ry) ** 2) + + +def _match_expected_objects(expected_objects: list[dict], geometry: dict) -> tuple[list[dict], dict[str, str]]: + actual = [] + for collection, object_type in (("panels", "panel"), ("cards", "card"), ("slots", "asset")): + actual.extend({**item, "benchmark_type": object_type} for item in geometry.get(collection, []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)) + unmatched = list(actual) + matches = [] + id_mapping: dict[str, str] = {} + for expected in expected_objects: + expected_box = expected.get("bbox_percent") if isinstance(expected.get("bbox_percent"), dict) else {} + expected_type = str(expected.get("type") or "") + same_id = next((item for item in unmatched if str(item.get("id")) == str(expected.get("id"))), None) + candidates = [item for item in unmatched if expected_type in {"", str(item.get("benchmark_type"))}] + if not candidates: + candidates = unmatched + chosen = same_id or max(candidates, key=lambda item: _bbox_iou(expected_box, item["bbox_percent"]), default=None) + if not chosen: + matches.append({"expected_id": expected.get("id"), "status": "unmatched", "iou": 0.0, "center_error": 1.0}) + continue + unmatched.remove(chosen) + iou = _bbox_iou(expected_box, chosen["bbox_percent"]) + center_error = _center_error(expected_box, chosen["bbox_percent"]) + id_mapping[str(expected.get("id"))] = str(chosen.get("id")) + matches.append({ + "expected_id": expected.get("id"), + "actual_id": chosen.get("id"), + "expected_type": expected_type, + "actual_type": chosen.get("benchmark_type"), + "iou": round(iou, 4), + "center_error": round(center_error, 4), + "status": "matched" if iou >= 0.30 else "low_overlap", + }) + return matches, id_mapping + + +def score_image_to_ppt(case_dir: str | Path, run_dir: str | Path) -> dict[str, Any]: + case_root, run = Path(case_dir).resolve(), Path(run_dir).resolve() + case = _load(case_root / "case.json") + expected = _load(case_root / str(case.get("expected_objects") or "expected_objects.json")) + reference = case_root / str(case.get("reference_image") or "reference.png") + preview = run / str(case.get("preview") or "rebuild_preview.png") + pptx = run / str(case.get("pptx") or "editable_composition.pptx") + similarity = _image_similarity(reference, preview) + editability = _ppt_editability(pptx) + visual = _load(run / "rebuild_visual_quality_report.json") + semantic = _load(run / "semantic_binding_report.json") + composition = _load(run / "composition_quality_report.json") + alignment = _load(run / "text_alignment_report.json") + geometry = _load(run / "reference_geometry.json") + + expected_objects = [item for item in expected.get("objects", []) if isinstance(item, dict)] + expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict)] + object_matches, id_mapping = _match_expected_objects(expected_objects, geometry) + object_coverage = _clamp(sum(1 for item in object_matches if item["status"] == "matched") / max(1, len(expected_objects))) if expected_objects else 1.0 + mean_object_iou = _mean([float(item["iou"]) for item in object_matches]) if object_matches else 1.0 + mean_center_error = _mean([float(item["center_error"]) for item in object_matches]) if object_matches else 0.0 + actual_relations = { + (str(item.get("source_id") or item.get("source") or ""), str(item.get("target_id") or item.get("target") or "")) + for item in composition.get("arrows", []) + if isinstance(item, dict) + } + matched_relations = 0 + for relation in expected_relations: + source = id_mapping.get(str(relation.get("source"))) + target = id_mapping.get(str(relation.get("target"))) + if source and target and (source, target) in actual_relations: + matched_relations += 1 + relation_coverage = _clamp(matched_relations / max(1, len(expected_relations))) if expected_relations else 1.0 + max_text_delta = max((float(item.get("center_delta_percent") or 0.0) for item in alignment.get("items", []) if isinstance(item, dict)), default=0.0) + text_alignment_score = _clamp(1.0 - max_text_delta / 0.10) + blocking_visual_issues = int(visual.get("blocking_issue_count") or 0) + semantic_binding_score = _clamp(semantic.get("mapped_entity_count", 0) / max(1, semantic.get("entity_count", 0))) if semantic else 1.0 + editable_score = 0.0 if editability["status"] == "blocked" else _clamp((min(1, editability["text_shape_count"]) + min(1, editability["connector_count"]) + 1) / 3) + fidelity_values = [value for value in similarity.values() if isinstance(value, float)] + render_fidelity = _mean(fidelity_values) if fidelity_values else 0.0 + metrics = { + **similarity, + "render_fidelity": render_fidelity, + "object_coverage": object_coverage, + "mean_object_iou": mean_object_iou, + "mean_center_error": mean_center_error, + "relation_coverage": relation_coverage, + "text_alignment_score": text_alignment_score, + "semantic_binding_score": semantic_binding_score, + "editability_score": editable_score, + "full_slide_image_count": editability["full_slide_image_count"], + "blocking_visual_issue_count": blocking_visual_issues, + } + hard_failures = [] + if editability["full_slide_image_count"]: + hard_failures.append("full-slide reference image detected") + if blocking_visual_issues: + hard_failures.append("blocking visual issues remain") + thresholds = case.get("thresholds", {}) + threshold_failures = _threshold_failures(metrics, thresholds) + total_score = round(0.40 * render_fidelity + 0.20 * object_coverage + 0.15 * relation_coverage + 0.10 * text_alignment_score + 0.15 * editable_score, 4) + return { + "summary": "Image-to-editable-PPT benchmark score.", + "suite": "image-to-ppt", + "case_id": case.get("case_id"), + "case_dir": str(case_root), + "run_dir": str(run), + "metrics": metrics, + "editability": editability, + "object_matches": object_matches, + "total_score": total_score, + "hard_failures": hard_failures, + "threshold_failures": threshold_failures, + "passed": not hard_failures and not threshold_failures, + "limitations": [ + "pixel, edge, and color similarity are baseline metrics; calibrated SSIM/LPIPS and region annotations remain future work", + "mutation-based editability testing requires a dedicated render-after-edit harness", + ], + } + + +def score_benchmark_case(case_dir: str | Path, run_dir: str | Path, out: str | Path | None = None) -> dict[str, Any]: + validation = validate_benchmark_case(case_dir) + if not validation["ok"]: + return {**validation, "passed": False} + result = score_paper_to_image(case_dir, run_dir) if validation["suite"] == "paper-to-image" else score_image_to_ppt(case_dir, run_dir) + target = ensure_dir(out or run_dir) + write_json(target / "benchmark_result.json", result) + lines = [ + "# Benchmark Result", + "", + f"- Suite: {result.get('suite')}", + f"- Case: {result.get('case_id')}", + f"- Passed: {result.get('passed')}", + f"- Total score: {result.get('total_score')}", + "", + "## Metrics", + ] + lines.extend(f"- {key}: {value}" for key, value in result.get("metrics", {}).items()) + lines.extend(["", "## Hard failures"]) + lines.extend(f"- {item}" for item in result.get("hard_failures", [])) + lines.extend(["", "## Threshold failures"]) + lines.extend(f"- {item}" for item in result.get("threshold_failures", [])) + write_text(target / "benchmark_result.md", "\n".join(lines) + "\n") + return result + + +def run_benchmark_case(case_dir: str | Path, out: str | Path) -> dict[str, Any]: + case_root = Path(case_dir).resolve() + case_preflight = _load(case_root / "case.json") + if str(case_preflight.get("suite") or "") == "paper-to-image": + paper_path = case_root / str(case_preflight.get("paper") or "paper.md") + if not paper_path.exists(): + fetched = fetch_benchmark_case(case_root) + if not fetched.get("ok"): + return {**fetched, "passed": False} + validation = validate_benchmark_case(case_dir) + if not validation["ok"]: + return {**validation, "passed": False} + root = Path(case_dir).resolve() + target = ensure_dir(out).resolve() + case = _load(root / "case.json") + config = case.get("run_config", {}) if isinstance(case.get("run_config"), dict) else {} + if validation["suite"] == "paper-to-image": + from ..paper_to_image import run_paper_to_image + + workflow_result = run_paper_to_image( + paper=root / str(case.get("paper") or "paper.md"), + out=target, + preferences_path=(root / str(case["preferences"])) if case.get("preferences") else None, + positive_references=[str(root / str(path)) for path in case.get("positive_references", [])], + negative_references=[str(root / str(path)) for path in case.get("negative_references", [])], + planner_mode=str(config.get("planner_mode") or "heuristic"), + asset_mode=str(config.get("image_asset_mode") or "placeholder"), + candidates=int(config.get("image_candidates") or 1), + aspect_ratio=str(config.get("aspect_ratio") or "auto"), + language=str(config.get("language") or "English"), + review_mode=str(config.get("review_mode") or "heuristic"), + domain_profile=str(config.get("domain_profile") or "auto"), + template=str(config.get("template") or "auto"), + repair_rounds=int(config.get("repair_rounds") or 0), + ocr_engine=str(config.get("ocr_engine") or "off"), + ocr_lang=str(config.get("ocr_lang") or "en_ch"), + ) + else: + from ..editable_rebuild import rebuild_editable + + workflow_result = rebuild_editable( + reference=root / str(case.get("reference_image") or "reference.png"), + out=target, + asset_mode=str(config.get("asset_mode") or "placeholder"), + asset_policy=str(config.get("asset_policy") or "smart-api"), + text_mode=str(config.get("text_mode") or "off"), + layout_mode=str(config.get("layout_mode") or "heuristic"), + control_mode=str(config.get("control_mode") or "heuristic"), + design_plan_mode=str(config.get("design_plan_mode") or "heuristic"), + export_preview=True, + ocr_engine=str(config.get("ocr_engine") or "off"), + ) + score = score_benchmark_case(root, target, target) + result = { + "summary": "Benchmark workflow and scoring completed.", + "ok": bool(workflow_result.get("ok")), + "suite": validation["suite"], + "case_id": validation["case_id"], + "out_dir": str(target), + "workflow_result": workflow_result, + "benchmark_result": score, + } + write_json(target / "benchmark_run.json", result) + return result + + +def run_fast_benchmark_case( + case_dir: str | Path, + out: str | Path, + deadline_seconds: int = 180, + planner_mode: str = "vlm", + planner_model: str | None = None, + ocr_engine: str = "off", + rasterize_pdf_dpi: int | None = None, +) -> dict[str, Any]: + case_root = Path(case_dir).resolve() + case = _load(case_root / "case.json") + if str(case.get("suite") or "") != "paper-to-image": + return {"summary": "Fast framework benchmark only supports paper-to-image cases.", "ok": False, "case_dir": str(case_root)} + paper_path = case_root / str(case.get("paper") or "paper.md") + if not paper_path.exists(): + fetched = fetch_benchmark_case(case_root) + if not fetched.get("ok"): + return {**fetched, "passed": False} + validation = validate_benchmark_case(case_root) + if not validation.get("ok"): + return {**validation, "passed": False} + from ..paper_to_image import run_fast_framework_prompt + from .scanned_pdf import rasterize_pdf_as_scan + + preferences = case_root / str(case.get("preferences")) if case.get("preferences") else None + active_paper = paper_path + rasterized_input = None + if rasterize_pdf_dpi is not None and paper_path.suffix.casefold() == ".pdf": + rasterized_input = rasterize_pdf_as_scan(paper_path, Path(out) / "benchmark_inputs" / "paper_scanned.pdf", dpi=rasterize_pdf_dpi) + active_paper = rasterized_input + run = run_fast_framework_prompt( + paper=active_paper, + out=out, + deadline_seconds=deadline_seconds, + planner_mode=planner_mode, + planner_model=planner_model, + ocr_engine=ocr_engine, + preferences_path=preferences if preferences and preferences.exists() else None, + aspect_ratio=str(case.get("run_config", {}).get("aspect_ratio") or "16:9"), + ) + score = score_benchmark_case(case_root, out) + result = { + "summary": "Fast paper framework benchmark completed.", + "ok": bool(run.get("ok")), + "case_id": case.get("case_id"), + "rasterized_input": str(rasterized_input) if rasterized_input else None, + "rasterize_pdf_dpi": int(rasterize_pdf_dpi) if rasterized_input else None, + "run": run, + "benchmark": score, + "passed_planning_thresholds": not any("plan_" in value for value in score.get("threshold_failures", [])), + } + write_json(Path(out) / "fast_benchmark_result.json", result) + return result + + +def run_fast_benchmark_suite( + benchmarks_root: str | Path, + out: str | Path, + case_ids: list[str] | None = None, + deadline_seconds: int = 180, + planner_mode: str = "vlm", + planner_model: str | None = None, + ocr_engine: str = "off", + rasterize_pdf_dpi: int | None = None, +) -> dict[str, Any]: + root = Path(benchmarks_root).resolve() + target = ensure_dir(out).resolve() + selected = set(case_ids or []) + listing = list_benchmark_cases(root, suite="paper-to-image") + cases = [item for item in listing.get("cases", []) if item.get("valid") and (not selected or str(item.get("case_id")) in selected)] + results = [] + for item in cases: + case_id = str(item.get("case_id")) + case_out = target / case_id + try: + result = run_fast_benchmark_case( + item["case_dir"], + case_out, + deadline_seconds=deadline_seconds, + planner_mode=planner_mode, + planner_model=planner_model, + ocr_engine=ocr_engine, + rasterize_pdf_dpi=rasterize_pdf_dpi, + ) + except Exception as exc: + result = {"summary": "Fast benchmark case raised an exception.", "ok": False, "case_id": case_id, "error": str(exc)} + run = result.get("run", {}) if isinstance(result.get("run"), dict) else {} + benchmark = result.get("benchmark", {}) if isinstance(result.get("benchmark"), dict) else {} + metrics = benchmark.get("metrics", {}) if isinstance(benchmark.get("metrics"), dict) else {} + provider = run.get("provider", {}) if isinstance(run.get("provider"), dict) else {} + extraction_quality = run.get("extraction_quality", {}) if isinstance(run.get("extraction_quality"), dict) else {} + results.append({ + "case_id": case_id, + "ok": bool(result.get("ok")), + "production_ready": bool(run.get("production_ready")), + "passed_planning_thresholds": bool(result.get("passed_planning_thresholds")), + "contract_source": run.get("contract_source"), + "cache_hit": bool(run.get("cache_hit")), + "document_cache_hit": bool(run.get("document_cache_hit")), + "rasterized_input": result.get("rasterized_input"), + "elapsed_seconds": run.get("elapsed_seconds"), + "stage_timings": run.get("stage_timings", {}), + "extraction_quality": extraction_quality, + "provider": provider, + "plan_entity_recall": metrics.get("plan_entity_recall"), + "plan_relation_recall": metrics.get("plan_relation_recall"), + "plan_forbidden_content_count": metrics.get("plan_forbidden_content_count"), + "threshold_failures": benchmark.get("threshold_failures", []), + "error": result.get("error"), + }) + + def numbers(field: str) -> list[float]: + values = [] + for item in results: + value = item.get(field) + if isinstance(value, (int, float)): + values.append(float(value)) + return values + + def timing(name: str) -> list[float]: + return [float(item.get("stage_timings", {}).get(name)) for item in results if isinstance(item.get("stage_timings", {}).get(name), (int, float))] + + def extraction_metric(name: str) -> list[float]: + return [float(item.get("extraction_quality", {}).get(name)) for item in results if isinstance(item.get("extraction_quality", {}).get(name), (int, float))] + + def percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, math.ceil(len(ordered) * fraction) - 1)) + return round(ordered[index], 4) + + provider_attempts = sum(int(item.get("provider", {}).get("attempts") or 0) for item in results) + provider_retries = sum(int(item.get("provider", {}).get("retries_used") or 0) for item in results) + provider_calls = [item for item in results if item.get("provider", {}).get("attempts")] + failure_categories: dict[str, int] = {} + for item in provider_calls: + for category in item.get("provider", {}).get("failure_categories", []) or []: + failure_categories[str(category)] = failure_categories.get(str(category), 0) + 1 + total_times = numbers("elapsed_seconds") + entity_scores = numbers("plan_entity_recall") + relation_scores = numbers("plan_relation_recall") + aggregate = { + "case_count": len(results), + "successful_run_count": sum(bool(item.get("ok")) for item in results), + "production_ready_count": sum(bool(item.get("production_ready")) for item in results), + "planning_threshold_pass_count": sum(bool(item.get("passed_planning_thresholds")) for item in results), + "cache_hit_count": sum(bool(item.get("cache_hit")) for item in results), + "cache_hit_rate": round(sum(bool(item.get("cache_hit")) for item in results) / max(1, len(results)), 4), + "document_cache_hit_count": sum(bool(item.get("document_cache_hit")) for item in results), + "document_cache_hit_rate": round(sum(bool(item.get("document_cache_hit")) for item in results) / max(1, len(results)), 4), + "rasterized_case_count": sum(bool(item.get("rasterized_input")) for item in results), + "mean_plan_entity_recall": _mean(entity_scores), + "mean_plan_relation_recall": _mean(relation_scores), + "forbidden_content_total": sum(int(item.get("plan_forbidden_content_count") or 0) for item in results), + "mean_total_seconds": _mean(total_times), + "p95_total_seconds": percentile(total_times, 0.95), + "mean_document_preparation_seconds": _mean(timing("document_preparation_seconds")), + "mean_document_extraction_seconds": _mean(timing("document_extraction_seconds")), + "mean_semantic_compilation_seconds": _mean(timing("semantic_compilation_seconds")), + "mean_readable_page_ratio": _mean(extraction_metric("readable_page_ratio")), + "mean_evidence_page_coverage_ratio": _mean(extraction_metric("evidence_page_coverage_ratio")), + "mean_evidence_char_count": _mean(extraction_metric("evidence_char_count")), + "max_detected_column_count": max((int(value) for value in extraction_metric("max_column_count")), default=0), + "multi_column_page_total": sum(int(value) for value in extraction_metric("multi_column_page_count")), + "mean_section_count": _mean(extraction_metric("section_count")), + "typographic_heading_total": sum(int(value) for value in extraction_metric("typographic_heading_count")), + "merged_heading_line_total": sum(int(value) for value in extraction_metric("merged_heading_line_count")), + "figure_caption_total": sum(int(value) for value in extraction_metric("figure_caption_count")), + "table_caption_total": sum(int(value) for value in extraction_metric("table_caption_count")), + "formula_total": sum(int(value) for value in extraction_metric("formula_count")), + "ocr_candidate_page_total": sum(int(value) for value in extraction_metric("ocr_candidate_count")), + "ocr_scheduled_page_total": sum(int(value) for value in extraction_metric("ocr_scheduled_count")), + "ocr_attempted_page_total": sum(int(value) for value in extraction_metric("ocr_attempted_count")), + "ocr_completed_page_total": sum(int(value) for value in extraction_metric("ocr_completed_count")), + "ocr_incomplete_run_count": sum(not bool(item.get("extraction_quality", {}).get("ocr_run_complete", True)) for item in results), + "max_ocr_worker_count": max((int(value) for value in extraction_metric("ocr_worker_count")), default=1), + "repeated_margin_noise_removed_total": sum(int(value) for value in extraction_metric("repeated_margin_noise_removed_count")), + "native_hyphenation_repair_total": sum(int(value) for value in extraction_metric("native_hyphenation_repair_count")), + "ocr_margin_noise_removed_total": sum(int(value) for value in extraction_metric("ocr_margin_noise_removed_count")), + "ocr_spacing_repair_total": sum(int(value) for value in extraction_metric("ocr_spacing_repair_count")), + "mean_ocr_latin_known_word_ratio": _mean(extraction_metric("ocr_latin_known_word_ratio")), + "provider_call_count": len(provider_calls), + "provider_success_count": sum(bool(item.get("provider", {}).get("success")) for item in provider_calls), + "provider_success_rate": round(sum(bool(item.get("provider", {}).get("success")) for item in provider_calls) / len(provider_calls), 4) if provider_calls else None, + "provider_attempts": provider_attempts, + "provider_retries_used": provider_retries, + "provider_failure_categories": failure_categories, + } + report = { + "summary": "Fast paper framework benchmark suite completed.", + "ok": bool(results) and all(item.get("ok") for item in results), + "benchmarks_root": str(root), + "out_dir": str(target), + "planner_mode": planner_mode, + "planner_model": planner_model, + "deadline_seconds": deadline_seconds, + "ocr_engine": ocr_engine, + "rasterize_pdf_dpi": rasterize_pdf_dpi, + "selected_case_ids": [item.get("case_id") for item in cases], + "aggregate": aggregate, + "cases": results, + } + write_json(target / "fast_suite_report.json", report) + return report diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py new file mode 100644 index 0000000..667c0e4 --- /dev/null +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -0,0 +1,557 @@ +from __future__ import annotations + +import io +import math +import time +from pathlib import Path +from typing import Any, Callable + +from ..paper_to_image.analyzer import parse_paper +from ..utils import ensure_dir, write_json +from .scanned_pdf import rasterize_pdf_as_scan + + +def _save_document(document: Any, target: Path) -> Path: + target.parent.mkdir(parents=True, exist_ok=True) + document.save(str(target)) + document.close() + return target + + +def _native_two_column_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 48), "Abstract", fontname="hebo", fontsize=13) + page.insert_textbox((45, 68, 555, 125), "A deterministic two-column extraction benchmark with enough scientific text for parser quality gates.", fontsize=10) + page.insert_text((45, 155), "2 Method", fontname="hebo", fontsize=13) + page.insert_textbox((45, 185, 275, 260), "LEFT_TOP encoder evidence begins the method flow and establishes the input representation.", fontsize=10) + page.insert_textbox((45, 300, 275, 375), "LEFT_BOTTOM decoder evidence follows the encoder in the left reading column.", fontsize=10) + page.insert_textbox((325, 185, 555, 260), "RIGHT_TOP retrieval evidence belongs after the complete left column.", fontsize=10) + page.insert_textbox((325, 300, 555, 375), "RIGHT_BOTTOM output evidence finishes the two-column reading order.", fontsize=10) + page.insert_textbox((45, 430, 555, 485), "Figure 1: Overview of the encoder, decoder, retrieval module, and final output.", fontsize=10) + page.insert_text((45, 540), "3 Conclusion", fontname="hebo", fontsize=13) + page.insert_textbox((45, 560, 555, 620), "The benchmark concludes with a complete evidence-grounded pipeline.", fontsize=10) + return _save_document(document, target) + + +def _bold_heading_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 70, 555, 130), "A paper with unnumbered bold sections validates typography-aware section recovery.", fontsize=10) + page.insert_text((45, 175), "Model Architecture", fontname="hebo", fontsize=11) + page.insert_textbox((45, 195, 555, 260), "The architecture contains an image encoder, a prompt encoder, and a mask decoder.", fontsize=10) + page.insert_text((45, 305), "Encoder and Decoder Stacks", fontname="hebo", fontsize=10) + page.insert_textbox((45, 325, 555, 390), "The encoder representation conditions the decoder before the final prediction.", fontsize=10) + page.insert_text((45, 435), "Conclusion", fontname="hebo", fontsize=11) + page.insert_textbox((45, 455, 555, 520), "Typography-based boundaries preserve the scientific section context.", fontsize=10) + return _save_document(document, target) + + +def _rotated_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 75, 555, 145), "A rotated scientific page must retain displayed coordinates and readable text.", fontsize=10) + page.insert_text((45, 190), "2 Method", fontname="hebo", fontsize=12) + page.insert_textbox((45, 215, 555, 290), "The rotated encoder sends its representation to a decoder and final output.", fontsize=10) + page.insert_textbox((45, 335, 555, 390), "Figure 1: Rotated architecture overview with encoder and decoder stages.", fontsize=10) + page.set_rotation(90) + return _save_document(document, target) + + +def _mixed_scan_fixture(target: Path) -> Path: + import fitz + + scan_source = fitz.open() + scan_page = scan_source.new_page(width=600, height=800) + scan_page.insert_text((50, 65), "2 Method", fontname="hebo", fontsize=18) + scan_page.insert_textbox((50, 105, 550, 190), "The scanned encoder passes evidence to the scanned decoder and output module.", fontsize=15) + scan_page.insert_textbox((50, 245, 550, 330), "Figure 1: Scanned architecture overview with encoder decoder and output.", fontsize=15) + pixmap = scan_page.get_pixmap(dpi=180, alpha=False) + scan_png = pixmap.tobytes("png") + scan_source.close() + + document = fitz.open() + page1 = document.new_page(width=600, height=800) + page1.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page1.insert_textbox((45, 75, 555, 155), "A mixed PDF combines native text with one scanned method page for local OCR fallback.", fontsize=10) + page2 = document.new_page(width=600, height=800) + page2.insert_image(page2.rect, stream=scan_png) + page3 = document.new_page(width=600, height=800) + page3.insert_text((45, 50), "3 Conclusion", fontname="hebo", fontsize=12) + page3.insert_textbox((45, 75, 555, 155), "The mixed extraction path preserves native pages and recovers the scanned method evidence.", fontsize=10) + return _save_document(document, target) + + +def _repeated_margin_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + sections = [ + ("Abstract", "A document encoder grounds paper evidence before framework generation."), + ("2 Method", "The document encoder sends grounded entities to a relation decoder and final output."), + ("3 Experiments", "Experiments measure entity recall, relation recall, and extraction latency."), + ("4 Conclusion", "The system preserves evidence citations and editable scientific labels."), + ] + for page_number, (heading, body) in enumerate(sections, 1): + page = document.new_page(width=600, height=800) + page.insert_text((45, 25), "Proceedings of the Example Conference 2026", fontsize=8) + page.insert_text((45, 785), f"Anonymous Paper 1234 | Page {page_number}", fontsize=8) + page.insert_text((45, 70), heading, fontname="hebo", fontsize=12) + page.insert_textbox((45, 95, 555, 220), body, fontsize=10) + return _save_document(document, target) + + +def _native_chinese_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + font = fitz.Font(fontname="china-s") + page.insert_font(fontname="CJK", fontbuffer=font.buffer) + page.insert_text((45, 50), "\u6458\u8981", fontname="CJK", fontsize=13) + page.insert_textbox((45, 75, 555, 145), "\u672c\u6587\u63d0\u51fa\u8bba\u6587\u7f16\u7801\u5668\u548c\u5173\u7cfb\u89e3\u7801\u5668,\u751f\u6210\u53ef\u7f16\u8f91\u6846\u67b6\u56fe\u3002", fontname="CJK", fontsize=10) + page.insert_text((45, 185), "2 \u65b9\u6cd5", fontname="CJK", fontsize=13) + page.insert_textbox((45, 210, 555, 300), "\u8f93\u5165\u8bba\u6587\u9996\u5148\u8fdb\u5165\u6587\u6863\u7f16\u7801\u5668\u3002\u5173\u7cfb\u89e3\u7801\u5668\u8fde\u63a5\u5b9e\u4f53,\u6700\u540e\u8f93\u51fa\u53ef\u7f16\u8f91\u7684\u6f14\u793a\u6587\u7a3f\u56fe\u3002", fontname="CJK", fontsize=10) + page.insert_textbox((45, 340, 555, 400), "\u56fe 1:\u8bba\u6587\u7f16\u7801\u5668\u3001\u5173\u7cfb\u89e3\u7801\u5668\u4e0e\u53ef\u7f16\u8f91\u8f93\u51fa\u7684\u7cfb\u7edf\u6846\u67b6\u3002", fontname="CJK", fontsize=10) + page.insert_text((45, 455), "3 \u5b9e\u9a8c", fontname="CJK", fontsize=13) + page.insert_textbox((45, 480, 555, 545), "\u5b9e\u9a8c\u8bc4\u4f30\u5b9e\u4f53\u53ec\u56de\u7387\u3001\u5173\u7cfb\u53ec\u56de\u7387\u548c\u5904\u7406\u65f6\u95f4\u3002", fontname="CJK", fontsize=10) + page.insert_text((45, 600), "4 \u7ed3\u8bba", fontname="CJK", fontsize=13) + page.insert_textbox((45, 625, 555, 690), "\u8be5\u65b9\u6cd5\u80fd\u591f\u4fdd\u6301\u8bba\u6587\u672f\u8bed\u3001\u8bc1\u636e\u9875\u7801\u548c\u6a21\u5757\u5173\u7cfb\u3002", fontname="CJK", fontsize=10) + return _save_document(document, target) + + +def _native_spanish_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Resumen", fontname="hebo", fontsize=13) + page.insert_textbox((45, 75, 555, 145), "Proponemos un codificador de documentos que transforma el artículo en evidencia estructurada y genera una figura editable.", fontsize=10) + page.insert_text((45, 185), "2 Método", fontname="hebo", fontsize=13) + page.insert_textbox((45, 210, 555, 295), "El documento de entrada pasa al codificador de documentos. El decodificador de relaciones conecta las entidades y produce una salida editable.", fontsize=10) + page.insert_textbox((45, 335, 555, 395), "Figura 1: Codificador de documentos, decodificador de relaciones y salida editable.", fontsize=10) + page.insert_text((45, 455), "3 Experimentos", fontname="hebo", fontsize=13) + page.insert_textbox((45, 480, 555, 545), "Evaluamos la recuperación de entidades, las relaciones correctas y el tiempo de procesamiento.", fontsize=10) + page.insert_text((45, 600), "4 Conclusión", fontname="hebo", fontsize=13) + page.insert_textbox((45, 625, 555, 690), "El método conserva la terminología científica y las referencias de evidencia.", fontsize=10) + return _save_document(document, target) + + +def _hyphenated_native_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 75, 555, 145), "We introduce a trans-\nformer encoder for evidence-grounded figure generation.", fontsize=10) + page.insert_text((45, 185), "2 Method", fontname="hebo", fontsize=12) + page.insert_textbox((45, 210, 555, 300), "The trans-\nformer encoder passes the input representation to a relation de-\ncoder and produces an editable output.", fontsize=10) + return _save_document(document, target) + + +def _formula_table_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 45), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 65, 555, 115), "We introduce a calibrated relation encoder and evaluate it with a dense ablation table.", fontsize=10) + page.insert_text((45, 145), "2 Method", fontname="hebo", fontsize=12) + page.insert_text((80, 185), "Q = X W_q K = X W_k V = X W_v", fontname="cour", fontsize=10) + page.insert_text((80, 215), "A = softmax(Q K^T / sqrt(d)) V", fontname="cour", fontsize=10) + page.insert_textbox((45, 245, 555, 295), "The relation encoder sends the calibrated representation to the prediction head.", fontsize=10) + page.insert_text((45, 335), "3 Experiments", fontname="hebo", fontsize=12) + page.insert_textbox((45, 360, 555, 390), "Table 1: Ablation results for encoder depth and relation accuracy.", fontsize=10) + rows = [("Model", "Depth", "Accuracy"), ("Base", "6", "81.2"), ("Large", "12", "84.7"), ("Ours", "18", "87.9")] + top = 420 + for row in rows: + for left, value in zip((65, 260, 390), row): + page.insert_text((left, top), value, fontsize=10) + top += 28 + page.insert_textbox((45, 555, 555, 600), "The ablation shows that calibrated relation encoding improves accuracy.", fontsize=10) + page.insert_text((45, 650), "4 Conclusion", fontname="hebo", fontsize=12) + page.insert_textbox((45, 675, 555, 720), "The method preserves directed evidence relations and calibrated outputs.", fontsize=10) + return _save_document(document, target) + + +def _rotated_repeated_margin_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + for page_number in range(1, 5): + page = document.new_page(width=600, height=800) + page.insert_text((45, 25), "Rotated Conference Header 2026", fontsize=8) + page.insert_text((45, 785), f"Rotated Paper 42 | Page {page_number}", fontsize=8) + page.insert_text((45, 75), f"{page_number} Method Stage", fontname="hebo", fontsize=12) + page.insert_textbox((45, 105, 555, 220), f"Stage {page_number} preserves rotated scientific evidence and semantic reading order.", fontsize=10) + page.set_rotation(90) + return _save_document(document, target) + + +def _skewed_scan_fixture(source: Path, target: Path, angle: float = 2.0) -> Path: + import fitz + from PIL import Image + + source_document = fitz.open(str(source)) + try: + pixmap = source_document[0].get_pixmap(dpi=144, alpha=False) + image = Image.open(io.BytesIO(pixmap.tobytes("png"))).convert("RGB") + finally: + source_document.close() + rotated = image.rotate(float(angle), resample=Image.Resampling.BICUBIC, expand=False, fillcolor="white") + payload = io.BytesIO() + rotated.save(payload, format="PNG") + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_image(page.rect, stream=payload.getvalue()) + return _save_document(document, target) + + +def _degraded_scan_fixture(source: Path, target: Path) -> Path: + import fitz + from PIL import Image, ImageFilter + + source_document = fitz.open(str(source)) + try: + pixmap = source_document[0].get_pixmap(dpi=120, alpha=False) + image = Image.open(io.BytesIO(pixmap.tobytes("png"))).convert("L") + finally: + source_document.close() + degraded = image.resize((420, 560), Image.Resampling.LANCZOS).filter(ImageFilter.GaussianBlur(0.3)).convert("RGB") + payload = io.BytesIO() + degraded.save(payload, format="JPEG", quality=45, optimize=True) + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_image(page.rect, stream=payload.getvalue()) + return _save_document(document, target) + + +def _fixture_ocr_adapter(image_path: Path, _lang: str) -> list[dict[str, Any]]: + if "page_002" not in image_path.name: + return [] + lines = [ + ("2 Method", 110), + ("The scanned encoder passes evidence to the scanned decoder and output module.", 220), + ("Figure 1: Scanned architecture overview with encoder decoder and output.", 420), + ] + return [ + { + "text": text, + "confidence": 0.99, + "quad": [[90, top], [1240, top], [1240, top + 55], [90, top + 55]], + } + for text, top in lines + ] + + +def _render_preview(source: Path, target: Path, page_index: int = 0) -> Path: + import fitz + + document = fitz.open(str(source)) + try: + pixmap = document[page_index].get_pixmap(dpi=110, alpha=False) + target.parent.mkdir(parents=True, exist_ok=True) + pixmap.save(str(target)) + finally: + document.close() + return target + + +def _check(name: str, passed: bool, actual: Any, expected: Any) -> dict[str, Any]: + return {"name": name, "passed": bool(passed), "actual": actual, "expected": expected} + + +def _run_case( + case_id: str, + fixture: Path, + out_dir: Path, + checks: Callable[[dict[str, Any]], list[dict[str, Any]]], + *, + ocr_engine: str = "off", + ocr_adapter: Callable | None = None, + preview_page: int = 0, +) -> dict[str, Any]: + started = time.monotonic() + parsed = parse_paper(fixture, ocr_engine=ocr_engine, ocr_adapter=ocr_adapter) + elapsed = round(time.monotonic() - started, 4) + case_dir = ensure_dir(out_dir / case_id) + write_json(case_dir / "document_model.json", parsed) + write_json(case_dir / "extraction_report.json", parsed.get("extraction_report", {})) + preview = _render_preview(fixture, case_dir / f"preview_page_{preview_page + 1}.png", preview_page) + assertions = checks(parsed) + ocr_page_durations = parsed.get("extraction_report", {}).get("ocr_page_durations", []) + result = { + "case_id": case_id, + "ok": all(item.get("passed") for item in assertions), + "fixture": str(fixture), + "preview": str(preview), + "elapsed_seconds": elapsed, + "pdf_type": parsed.get("extraction_report", {}).get("pdf_type"), + "page_count": parsed.get("page_count"), + "section_count": parsed.get("extraction_report", {}).get("section_count"), + "figure_caption_count": parsed.get("extraction_report", {}).get("figure_caption_count"), + "ocr_pages": parsed.get("extraction_report", {}).get("ocr_pages", []), + "ocr_spacing_repair_count": parsed.get("extraction_report", {}).get("ocr_spacing_repair_count", 0), + "ocr_page_durations": ocr_page_durations, + "ocr_stage_seconds": { + name: round(sum(float(item.get(name) or 0.0) for item in ocr_page_durations), 4) + for name in ("render_seconds", "detection_seconds", "classification_seconds", "recognition_seconds", "postprocess_seconds", "inference_seconds") + }, + "assertions": assertions, + } + write_json(case_dir / "benchmark_result.json", result) + return result + + +def _percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, math.ceil(len(ordered) * fraction) - 1)) + return round(ordered[index], 4) + + +def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> dict[str, Any]: + root = ensure_dir(out).resolve() + fixtures = ensure_dir(root / "fixtures") + native = _native_two_column_fixture(fixtures / "native_two_column.pdf") + bold = _bold_heading_fixture(fixtures / "unnumbered_bold_sections.pdf") + rotated = _rotated_fixture(fixtures / "rotated_native_page.pdf") + mixed = _mixed_scan_fixture(fixtures / "mixed_scan.pdf") + repeated_margin = _repeated_margin_fixture(fixtures / "repeated_margin_noise.pdf") + chinese = _native_chinese_fixture(fixtures / "native_chinese.pdf") + spanish = _native_spanish_fixture(fixtures / "native_spanish.pdf") + hyphenated = _hyphenated_native_fixture(fixtures / "hyphenated_native.pdf") + formula_table = _formula_table_fixture(fixtures / "formula_table_dense.pdf") + rotated_margin = _rotated_repeated_margin_fixture(fixtures / "rotated_repeated_margin.pdf") + + results = [ + _run_case( + "native_two_column", + native, + root, + lambda parsed: [ + _check("two_columns", parsed["extraction_report"].get("max_column_count") == 2, parsed["extraction_report"].get("max_column_count"), 2), + _check("reading_order", all(parsed["pages"][0]["text"].index(left) < parsed["pages"][0]["text"].index(right) for left, right in (("LEFT_TOP", "LEFT_BOTTOM"), ("LEFT_BOTTOM", "RIGHT_TOP"), ("RIGHT_TOP", "RIGHT_BOTTOM"))), parsed["pages"][0]["text"], "LEFT_TOP < LEFT_BOTTOM < RIGHT_TOP < RIGHT_BOTTOM"), + _check("cross_column_caption_order", parsed["pages"][0]["text"].index("RIGHT_BOTTOM") < parsed["pages"][0]["text"].index("Figure 1:") < parsed["pages"][0]["text"].index("3 Conclusion"), parsed["pages"][0]["text"], "RIGHT_BOTTOM < Figure 1 < Conclusion"), + _check("figure_caption", parsed["extraction_report"].get("figure_caption_count", 0) >= 1, parsed["extraction_report"].get("figure_caption_count"), ">=1"), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "conclusion")), parsed["extraction_report"].get("section_coverage"), "abstract/method/conclusion"), + ], + ), + _run_case( + "unnumbered_bold_sections", + bold, + root, + lambda parsed: [ + _check("model_architecture_heading", "Model Architecture" in parsed.get("headings", []), parsed.get("headings", []), "Model Architecture"), + _check("stack_heading", "Encoder and Decoder Stacks" in parsed.get("headings", []), parsed.get("headings", []), "Encoder and Decoder Stacks"), + _check("section_hint", any(item.get("section_hint") == "Model Architecture" and "image encoder" in str(item.get("text") or "").casefold() for item in parsed.get("evidence", [])), [item.get("section_hint") for item in parsed.get("evidence", []) if "image encoder" in str(item.get("text") or "").casefold()], "Model Architecture"), + ], + ), + _run_case( + "rotated_native_page", + rotated, + root, + lambda parsed: [ + _check("rotation", parsed["extraction_report"].get("rotated_pages") == [1], parsed["extraction_report"].get("rotated_pages"), [1]), + _check("display_coordinates", all(0 <= block["bbox"][0] <= block["bbox"][2] <= parsed["pages"][0]["width"] and 0 <= block["bbox"][1] <= block["bbox"][3] <= parsed["pages"][0]["height"] for block in parsed["pages"][0]["blocks"]), parsed["pages"][0]["blocks"], "all blocks inside displayed page"), + _check("semantic_reading_order", parsed["pages"][0]["text"].index("Abstract") < parsed["pages"][0]["text"].index("2 Method") < parsed["pages"][0]["text"].index("Figure 1:"), parsed["pages"][0]["text"], "Abstract < Method < Figure 1"), + _check("caption", parsed["extraction_report"].get("figure_caption_count", 0) >= 1, parsed["extraction_report"].get("figure_caption_count"), ">=1"), + ], + ), + _run_case( + "mixed_scan_fixture_adapter", + mixed, + root, + lambda parsed: [ + _check("mixed_pdf", parsed["extraction_report"].get("pdf_type") == "mixed", parsed["extraction_report"].get("pdf_type"), "mixed"), + _check("local_ocr", parsed["extraction_report"].get("ocr_pages") == [2], parsed["extraction_report"].get("ocr_pages"), [2]), + _check("method_recovered", parsed["extraction_report"].get("section_coverage", {}).get("method") is True, parsed["extraction_report"].get("section_coverage"), "method=true"), + _check("scanned_caption", any("Scanned architecture overview" in str(item.get("caption") or "") for item in parsed.get("document_index", {}).get("figures", [])), parsed.get("document_index", {}).get("figures", []), "scanned Figure 1 caption"), + ], + ocr_engine="easyocr", + ocr_adapter=_fixture_ocr_adapter, + preview_page=1, + ), + _run_case( + "repeated_margin_noise", + repeated_margin, + root, + lambda parsed: [ + _check("noise_removed", parsed["extraction_report"].get("repeated_margin_noise_removed_count") == 8, parsed["extraction_report"].get("repeated_margin_noise_removed_count"), 8), + _check("header_absent", all("Proceedings of the Example Conference" not in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no repeated header evidence"), + _check("footer_absent", all("Anonymous Paper 1234" not in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no repeated footer evidence"), + _check("scientific_content_preserved", any("relation decoder" in item.get("text", "").casefold() for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "relation decoder evidence"), + ], + ), + _run_case( + "native_chinese", + chinese, + root, + lambda parsed: [ + _check("unicode_preserved", "\u6587\u6863\u7f16\u7801\u5668" in parsed["pages"][0]["text"], parsed["pages"][0]["text"], "contains document encoder in Chinese"), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "experiments", "conclusion")), parsed["extraction_report"].get("section_coverage"), "Chinese abstract/method/experiments/conclusion"), + _check("figure_caption", any("\u8bba\u6587\u7f16\u7801\u5668" in str(item.get("caption") or "") for item in parsed.get("document_index", {}).get("figures", [])), parsed.get("document_index", {}).get("figures", []), "Chinese Figure 1 caption"), + _check("no_mojibake", parsed["extraction_report"].get("mojibake_rate") == 0.0, parsed["extraction_report"].get("mojibake_rate"), 0.0), + ], + ), + _run_case( + "native_spanish", + spanish, + root, + lambda parsed: [ + _check("unicode_preserved", "artículo" in parsed["pages"][0]["text"] and "Conclusión" in parsed["pages"][0]["text"], parsed["pages"][0]["text"], "Spanish accents preserved"), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "experiments", "conclusion")), parsed["extraction_report"].get("section_coverage"), "Spanish abstract/method/experiments/conclusion"), + _check("figure_caption", any("codificador de documentos" in str(item.get("caption") or "").casefold() for item in parsed.get("document_index", {}).get("figures", [])), parsed.get("document_index", {}).get("figures", []), "Spanish Figura 1 caption"), + _check("source_language_preserved", "document encoder" not in parsed["pages"][0]["text"].casefold() and "codificador de documentos" in parsed["pages"][0]["text"].casefold(), parsed["pages"][0]["text"], "Spanish terminology without English translation"), + ], + ), + _run_case( + "hyphenated_native", + hyphenated, + root, + lambda parsed: [ + _check("transformer_repaired", sum("transformer encoder" in item.get("text", "").casefold() for item in parsed.get("evidence", [])) >= 2, [item.get("text") for item in parsed.get("evidence", [])], "two transformer encoder mentions"), + _check("decoder_repaired", any("relation decoder" in item.get("text", "").casefold() for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "relation decoder"), + _check("repair_count", parsed["extraction_report"].get("native_hyphenation_repair_count") == 3, parsed["extraction_report"].get("native_hyphenation_repair_count"), 3), + ], + ), + _run_case( + "formula_table_dense", + formula_table, + root, + lambda parsed: [ + _check("formulas", parsed["extraction_report"].get("formula_count") == 2, parsed["extraction_report"].get("formula_count"), 2), + _check("table_caption", parsed["extraction_report"].get("table_caption_count") == 1, parsed["extraction_report"].get("table_caption_count"), 1), + _check("table_columns", parsed["document_index"]["tables"][0].get("columns") == ["Model", "Depth", "Accuracy"], parsed["document_index"]["tables"][0].get("columns"), ["Model", "Depth", "Accuracy"]), + _check("table_rows", any(row.get("values") == ["Large", "12", "84.7"] for row in parsed["document_index"]["tables"][0].get("rows", [])), parsed["document_index"]["tables"][0].get("rows"), ["Large", "12", "84.7"]), + _check("structured_table_evidence", any(item.get("kind") == "table" and "Row: Large | 12 | 84.7" in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", []) if item.get("kind") == "table"], "structured Large row"), + _check("no_orphan_cells", not any(item.get("text") == "84.7" for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no isolated table cells"), + ], + ), + _run_case( + "rotated_repeated_margin", + rotated_margin, + root, + lambda parsed: [ + _check("all_pages_rotated", parsed["extraction_report"].get("rotated_pages") == [1, 2, 3, 4], parsed["extraction_report"].get("rotated_pages"), [1, 2, 3, 4]), + _check("rotated_noise_removed", parsed["extraction_report"].get("repeated_margin_noise_removed_count") == 8, parsed["extraction_report"].get("repeated_margin_noise_removed_count"), 8), + _check("repeated_section_boundaries", parsed["extraction_report"].get("section_count") == 4, parsed["extraction_report"].get("section_count"), 4), + _check("header_absent", all("Rotated Conference Header" not in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no rotated header evidence"), + _check("body_preserved", all(any(f"Stage {page_number} preserves" in item.get("text", "") for item in parsed.get("evidence", [])) for page_number in range(1, 5)), [item.get("text") for item in parsed.get("evidence", [])], "all rotated stage evidence"), + ], + ), + ] + + if ocr_engine != "off": + scanned_two_column = rasterize_pdf_as_scan(native, fixtures / "scanned_two_column.pdf", dpi=144) + skewed_two_column = _skewed_scan_fixture(native, fixtures / "skewed_two_column.pdf") + degraded_two_column = _degraded_scan_fixture(native, fixtures / "degraded_two_column.pdf") + scanned_formula_table = rasterize_pdf_as_scan(formula_table, fixtures / "scanned_formula_table.pdf", dpi=144) + scanned_spanish = rasterize_pdf_as_scan(spanish, fixtures / "scanned_spanish.pdf", dpi=144) + results.append(_run_case( + "mixed_scan_runtime_ocr", + mixed, + root, + lambda parsed: [ + _check("ocr_completed", 2 in parsed["extraction_report"].get("ocr_pages", []), parsed["extraction_report"].get("ocr_pages", []), "contains page 2"), + _check("method_spacing", "2 method" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "contains '2 Method'"), + _check("sentence_spacing", "scanned encoder passes evidence to the scanned decoder and output" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "spaced scientific sentence"), + _check("caption_spacing", "scanned architecture overview with encoder decoder and output" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "spaced caption"), + _check("repairs_recorded", parsed["extraction_report"].get("ocr_spacing_repair_count", 0) > 0, parsed["extraction_report"].get("ocr_spacing_repair_count", 0), ">0"), + ], + ocr_engine=ocr_engine, + preview_page=1, + )) + results.append(_run_case( + "scanned_two_column_runtime_ocr", + scanned_two_column, + root, + lambda parsed: [ + _check("two_columns", parsed["extraction_report"].get("max_column_count") == 2, parsed["extraction_report"].get("max_column_count"), 2), + _check("reading_order", all(parsed["pages"][0]["text"].index(left) < parsed["pages"][0]["text"].index(right) for left, right in (("LEFT_TOP", "LEFT_BOTTOM"), ("LEFT_BOTTOM", "RIGHT_TOP"), ("RIGHT_TOP", "RIGHT_BOTTOM"))), parsed["pages"][0]["text"], "LEFT_TOP < LEFT_BOTTOM < RIGHT_TOP < RIGHT_BOTTOM"), + _check("headings", all(value in parsed.get("headings", []) for value in ("Abstract", "Method", "Conclusion")), parsed.get("headings", []), "Abstract/Method/Conclusion"), + _check("caption", parsed["extraction_report"].get("figure_caption_count") == 1, parsed["extraction_report"].get("figure_caption_count"), 1), + ], + ocr_engine=ocr_engine, + )) + results.append(_run_case( + "skewed_two_column_runtime_ocr", + skewed_two_column, + root, + lambda parsed: [ + _check("two_columns", parsed["extraction_report"].get("max_column_count") == 2, parsed["extraction_report"].get("max_column_count"), 2), + _check("reading_order", all(parsed["pages"][0]["text"].index(left) < parsed["pages"][0]["text"].index(right) for left, right in (("LEFT_TOP", "LEFT_BOTTOM"), ("LEFT_BOTTOM", "RIGHT_TOP"), ("RIGHT_TOP", "RIGHT_BOTTOM"))), parsed["pages"][0]["text"], "LEFT_TOP < LEFT_BOTTOM < RIGHT_TOP < RIGHT_BOTTOM"), + _check("mean_confidence", float(parsed["extraction_report"].get("mean_ocr_confidence") or 0.0) >= 0.9, parsed["extraction_report"].get("mean_ocr_confidence"), ">=0.9"), + _check("caption", parsed["extraction_report"].get("figure_caption_count") == 1, parsed["extraction_report"].get("figure_caption_count"), 1), + ], + ocr_engine=ocr_engine, + )) + results.append(_run_case( + "degraded_two_column_quality_gate", + degraded_two_column, + root, + lambda parsed: [ + _check("quality_gate_applied", parsed["extraction_report"].get("ocr_latin_lexical_gate_applied") is True, parsed["extraction_report"].get("ocr_latin_lexical_gate_applied"), True), + _check("low_known_word_ratio", float(parsed["extraction_report"].get("ocr_latin_known_word_ratio") or 1.0) < 0.5, parsed["extraction_report"].get("ocr_latin_known_word_ratio"), "<0.5"), + _check("extraction_failed", parsed["extraction_report"].get("status") == "fail", parsed["extraction_report"].get("status"), "fail"), + _check("no_false_cache_scope", parsed["extraction_report"].get("sampled_scan_ready") is False, parsed["extraction_report"].get("sampled_scan_ready"), False), + ], + ocr_engine=ocr_engine, + )) + results.append(_run_case( + "scanned_formula_table_runtime_ocr", + scanned_formula_table, + root, + lambda parsed: [ + _check("formula_recovered", parsed["extraction_report"].get("formula_count", 0) >= 1, parsed["extraction_report"].get("formula_count"), ">=1"), + _check("table_columns", parsed["document_index"]["tables"][0].get("columns") == ["Model", "Depth", "Accuracy"], parsed["document_index"]["tables"][0].get("columns"), ["Model", "Depth", "Accuracy"]), + _check("large_row", any(row.get("values") == ["Large", "12", "84.7"] for row in parsed["document_index"]["tables"][0].get("rows", [])), parsed["document_index"]["tables"][0].get("rows"), ["Large", "12", "84.7"]), + _check("structured_evidence", any(item.get("kind") == "table" and "Row: Large | 12 | 84.7" in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", []) if item.get("kind") == "table"], "structured Large row"), + ], + ocr_engine=ocr_engine, + )) + results.append(_run_case( + "scanned_spanish_runtime_ocr", + scanned_spanish, + root, + lambda parsed: [ + _check("ocr_completed", parsed["extraction_report"].get("ocr_pages") == [1], parsed["extraction_report"].get("ocr_pages"), [1]), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "experiments", "conclusion")), parsed["extraction_report"].get("section_coverage"), "Spanish abstract/method/experiments/conclusion"), + _check("figure_caption", parsed["extraction_report"].get("figure_caption_count") == 1, parsed["extraction_report"].get("figure_caption_count"), 1), + _check("source_language_preserved", "document encoder" not in parsed["pages"][0]["text"].casefold() and "codificador de documentos" in parsed["pages"][0]["text"].casefold(), parsed["pages"][0]["text"], "Spanish terminology without English translation"), + _check("quality_gate_passed", parsed["extraction_report"].get("status") != "fail", parsed["extraction_report"].get("status"), "pass or warning"), + ], + ocr_engine=ocr_engine, + )) + + elapsed = [float(item.get("elapsed_seconds") or 0.0) for item in results] + ocr_stage_totals = { + name: round(sum(float(item.get("ocr_stage_seconds", {}).get(name) or 0.0) for item in results), 4) + for name in ("render_seconds", "detection_seconds", "classification_seconds", "recognition_seconds", "postprocess_seconds", "inference_seconds") + } + report = { + "summary": "Deterministic multi-layout PDF extraction stress suite completed.", + "ok": all(item.get("ok") for item in results), + "out_dir": str(root), + "ocr_engine": ocr_engine, + "aggregate": { + "case_count": len(results), + "passed_case_count": sum(bool(item.get("ok")) for item in results), + "mean_elapsed_seconds": round(sum(elapsed) / max(1, len(elapsed)), 4), + "p95_elapsed_seconds": _percentile(elapsed, 0.95), + "ocr_case_count": sum(bool(item.get("ocr_pages")) for item in results), + "ocr_stage_seconds": ocr_stage_totals, + }, + "cases": results, + } + write_json(root / "pdf_extraction_stress_report.json", report) + return report diff --git a/rfs/evaluation/rebuild_visual.py b/rfs/evaluation/rebuild_visual.py new file mode 100644 index 0000000..1b10818 --- /dev/null +++ b/rfs/evaluation/rebuild_visual.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ..utils import write_json + + +def _bbox(item: dict) -> dict | None: + box = item.get("bbox_percent") + return box if isinstance(box, dict) else None + + +def _valid_bbox(box: dict | None) -> bool: + if not isinstance(box, dict): + return False + try: + x = float(box["x"]) + y = float(box["y"]) + w = float(box["w"]) + h = float(box["h"]) + except Exception: + return False + return 0 <= x <= 1 and 0 <= y <= 1 and w > 0 and h > 0 and x + w <= 1.0001 and y + h <= 1.0001 + + +def _overlap_area(a: dict, b: dict) -> float: + ax0, ay0 = float(a["x"]), float(a["y"]) + ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) + bx0, by0 = float(b["x"]), float(b["y"]) + bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + return ix * iy + + +def _overlap_ratio(a: dict, b: dict) -> float: + area = _overlap_area(a, b) + if area <= 0: + return 0.0 + a_area = float(a["w"]) * float(a["h"]) + b_area = float(b["w"]) * float(b["h"]) + return area / max(min(a_area, b_area), 0.000001) + + +def _visible_text_items(program: dict) -> list[dict]: + text_program = program.get("text_program") + if not isinstance(text_program, dict): + return [] + return [item for item in text_program.get("items", []) if isinstance(item, dict) and item.get("visible", True)] + + +def _text_alignment_group_key(item: dict) -> tuple[str, str] | None: + for key in ("paragraph_id", "text_group_id", "group_id", "source_group_id"): + value = str(item.get(key) or "").strip() + if value: + return (key, value) + return None + + +def _detect_text_issues(program: dict) -> list[dict]: + issues = [] + items = _visible_text_items(program) + for item in items: + if not _valid_bbox(_bbox(item)): + issues.append({"type": "text_bbox_out_of_bounds", "text_id": item.get("id"), "reason": "text bbox is missing or outside canvas"}) + for index, left in enumerate(items): + lbox = _bbox(left) + if not _valid_bbox(lbox): + continue + for right in items[index + 1:]: + rbox = _bbox(right) + if not _valid_bbox(rbox): + continue + ratio = _overlap_ratio(lbox, rbox) + if ratio > 0.18: + issues.append({ + "type": "text_overlap", + "text_id": left.get("id"), + "other_text_id": right.get("id"), + "overlap_ratio": round(ratio, 4), + "reason": "visible text boxes overlap substantially", + }) + grouped: dict[tuple[str, str], list[dict]] = {} + for item in items: + key = _text_alignment_group_key(item) + if key: + grouped.setdefault(key, []).append(item) + for (group_key, group_id), group in grouped.items(): + if len(group) < 3: + continue + centers = [ + float(item["bbox_percent"]["y"]) + float(item["bbox_percent"]["h"]) / 2 + for item in group + if _valid_bbox(_bbox(item)) + ] + if len(centers) >= 3 and max(centers) - min(centers) > 0.035: + issues.append({ + "type": "text_group_misaligned", + "group_key": group_key, + "group_id": group_id, + "text_ids": [item.get("id") for item in group], + "center_y_span": round(max(centers) - min(centers), 4), + "reason": "explicit text group is visually uneven", + }) + return issues + + +def _detect_object_issues(program: dict) -> list[dict]: + issues = [] + for collection_name in ("panels", "slots"): + items = [item for item in program.get(collection_name, []) if isinstance(item, dict)] + for item in items: + if not _valid_bbox(_bbox(item)): + issues.append({"type": f"{collection_name[:-1]}_bbox_out_of_bounds", "id": item.get("id"), "reason": "bbox is missing or outside canvas"}) + for index, left in enumerate(items): + lbox = _bbox(left) + if not _valid_bbox(lbox): + continue + for right in items[index + 1:]: + rbox = _bbox(right) + if not _valid_bbox(rbox): + continue + ratio = _overlap_ratio(lbox, rbox) + if ratio > (0.35 if collection_name == "slots" else 0.18): + issues.append({ + "type": f"{collection_name[:-1]}_overlap", + "id": left.get("id"), + "other_id": right.get("id"), + "overlap_ratio": round(ratio, 4), + "reason": f"{collection_name[:-1]} boxes overlap substantially", + }) + return issues + + +def _detect_arrow_issues(program: dict) -> list[dict]: + issues = [] + for arrow in program.get("arrows", []) or []: + if not isinstance(arrow, dict): + continue + path = arrow.get("path_percent") + if not isinstance(path, list) or len(path) < 2: + issues.append({"type": "arrow_missing_path", "arrow_id": arrow.get("id"), "reason": "arrow has no usable path_percent"}) + return issues + + +def _detect_ownership_issues(program: dict, ownership_report: dict | None) -> list[dict]: + issues = [] + text_ids = {str(item.get("source_reference_text_id") or item.get("id") or "") for item in _visible_text_items(program)} + report = ownership_report if isinstance(ownership_report, dict) else {} + for item in report.get("items", []) or []: + if not isinstance(item, dict): + continue + text_id = str(item.get("text_id") or "") + ownership = str(item.get("layer_ownership") or "") + included = bool(item.get("included_in_text_program")) + if ownership != "editable_text_layer" and text_id in text_ids: + issues.append({"type": "text_layer_ownership_conflict", "text_id": text_id, "layer_ownership": ownership, "reason": "non-editable-owned text is present in text_program"}) + if ownership == "editable_text_layer" and not included: + issues.append({"type": "text_layer_ownership_conflict", "text_id": text_id, "layer_ownership": ownership, "reason": "editable-owned text is missing from text_program"}) + return issues + + +def run_rebuild_visual_quality_check( + out_dir: str | Path, + program: dict, + reference_geometry: dict | None = None, + reference_controls: dict | None = None, + ownership_report: dict | None = None, + mode: str = "heuristic", +) -> dict[str, Any]: + text_issues = _detect_text_issues(program) + object_issues = _detect_object_issues(program) + arrow_issues = _detect_arrow_issues(program) + ownership_issues = _detect_ownership_issues(program, ownership_report) + issues = text_issues + object_issues + arrow_issues + ownership_issues + blocking_types = {"text_overlap", "text_bbox_out_of_bounds", "arrow_missing_path", "text_layer_ownership_conflict"} + blocking = [item for item in issues if item.get("type") in blocking_types] + report = { + "summary": "Deterministic rebuild visual quality report.", + "mode": mode, + "status": "blocked" if blocking else ("warning" if issues else "pass"), + "issue_count": len(issues), + "blocking_issue_count": len(blocking), + "text_issue_count": len(text_issues), + "object_issue_count": len(object_issues), + "arrow_issue_count": len(arrow_issues), + "ownership_issue_count": len(ownership_issues), + "issues": issues, + "reference_geometry_status": (reference_geometry or {}).get("status"), + "reference_controls_status": (reference_controls or {}).get("status"), + "policy": "deterministic check only; no program mutation", + } + write_json(Path(out_dir) / "rebuild_visual_quality_report.json", report) + return report diff --git a/rfs/evaluation/scanned_pdf.py b/rfs/evaluation/scanned_pdf.py new file mode 100644 index 0000000..8f67aed --- /dev/null +++ b/rfs/evaluation/scanned_pdf.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from pathlib import Path + + +def rasterize_pdf_as_scan(source: str | Path, target: str | Path, dpi: int = 144, jpeg_quality: int = 78) -> Path: + """Create an image-only PDF fixture while preserving displayed page sizes.""" + import fitz + + source_path = Path(source).resolve() + target_path = Path(target).resolve() + if not source_path.exists(): + raise FileNotFoundError(f"PDF does not exist: {source_path}") + if source_path.suffix.casefold() != ".pdf": + raise ValueError(f"Rasterized scan input must be a PDF: {source_path}") + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.unlink(missing_ok=True) + document = fitz.open(str(source_path)) + scanned = fitz.open() + try: + for page in document: + pixmap = page.get_pixmap(dpi=max(72, min(300, int(dpi))), alpha=False) + image = pixmap.tobytes("jpeg", jpg_quality=max(40, min(95, int(jpeg_quality)))) + output_page = scanned.new_page(width=float(page.rect.width), height=float(page.rect.height)) + output_page.insert_image(output_page.rect, stream=image) + scanned.set_metadata({ + "title": "ResearchFigureStudio rasterized scan benchmark", + "author": "ResearchFigureStudio", + "subject": "Deterministic image-only PDF benchmark fixture", + "keywords": "ResearchFigureStudio,scan,benchmark", + "creator": "ResearchFigureStudio", + "producer": "ResearchFigureStudio", + "creationDate": "", + "modDate": "", + }) + scanned.save(str(target_path), garbage=4, deflate=True, no_new_id=True, preserve_metadata=False) + finally: + scanned.close() + document.close() + return target_path diff --git a/rfs/generation/__init__.py b/rfs/generation/__init__.py new file mode 100644 index 0000000..d3a97a5 --- /dev/null +++ b/rfs/generation/__init__.py @@ -0,0 +1,6 @@ +"""Raster reference and slot-asset generation entry points.""" + +from ..asset_generator import generate_assets +from ..paper_to_image.generator import generate_and_select + +__all__ = ["generate_and_select", "generate_assets"] diff --git a/rfs/layout_locator.py b/rfs/layout_locator.py index 2eb9967..38fd6ee 100644 --- a/rfs/layout_locator.py +++ b/rfs/layout_locator.py @@ -146,6 +146,7 @@ def _default_arrows(panels: list[dict]) -> list[dict]: "path_percent": [], "label": "", "editable_in": "pptx", + "binding_source": "heuristic_default_panel_flow", }) if any(p["id"] == "shared_resource_library" for p in panels): arrows.append({ @@ -156,6 +157,7 @@ def _default_arrows(panels: list[dict]) -> list[dict]: "path_percent": [], "label": "shared resources", "editable_in": "pptx", + "binding_source": "heuristic_default_resource_bus", }) return arrows diff --git a/rfs/paper_to_editable.py b/rfs/paper_to_editable.py new file mode 100644 index 0000000..26d0c68 --- /dev/null +++ b/rfs/paper_to_editable.py @@ -0,0 +1,8 @@ +"""Compatibility import for the paper-to-editable workflow. + +New code should import from :mod:`rfs.workflows`. +""" + +from .workflows.paper_to_editable import run_paper_to_editable + +__all__ = ["run_paper_to_editable"] diff --git a/rfs/paper_to_image/__init__.py b/rfs/paper_to_image/__init__.py index 0836ed1..09cb4c7 100644 --- a/rfs/paper_to_image/__init__.py +++ b/rfs/paper_to_image/__init__.py @@ -1,5 +1,9 @@ -"""Paper-to-image workflow without PPTX generation.""" +"""Paper analysis and paper-to-image workflows.""" +from .inspection import inspect_paper +from .editable_ppt import build_semantic_figure_program, build_visual_substrate_figure_program, compile_semantic_ppt, compile_visual_substrate_ppt +from .preparation import prepare_paper_figure_contract, run_fast_framework_prompt +from .overlay_renderer import normalize_visual_substrate_geometry from .workflow import run_paper_to_image -__all__ = ["run_paper_to_image"] +__all__ = ["build_semantic_figure_program", "build_visual_substrate_figure_program", "compile_semantic_ppt", "compile_visual_substrate_ppt", "inspect_paper", "normalize_visual_substrate_geometry", "prepare_paper_figure_contract", "run_fast_framework_prompt", "run_paper_to_image"] diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 6980b77..19c98e8 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -1,167 +1,2141 @@ from __future__ import annotations +import hashlib +import json +import os import re +import shutil +import subprocess +import sys +import tempfile +import time +import unicodedata +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from difflib import SequenceMatcher from pathlib import Path -from typing import Any +from typing import Any, Callable + + +MOJIBAKE_MARKERS = ("\ufffd", "Ã", "Â", "â€", "锟", "鈥", "銆", "鏅") +SECTION_ALIASES = { + "abstract": ("abstract", "resumen", "résumé", "zusammenfassung", "resumo", "摘要", "概要", "초록"), + "introduction": ("introduction", "background", "introducción", "introduccion", "introdução", "introducao", "einleitung", "contexte", "contexto", "antecedentes", "引言", "介绍", "背景", "はじめに", "序論", "서론", "배경"), + "method": ("method", "methods", "methodology", "approach", "architecture", "system overview", "framework", "método", "metodo", "métodos", "metodos", "metodología", "metodologia", "méthode", "methode", "méthodes", "methodes", "méthodologie", "methodologie", "methoden", "methodik", "enfoque", "approche", "ansatz", "abordagem", "方法", "方法论", "架构", "系统概述", "框架", "手法", "方法論", "アーキテクチャ", "방법", "방법론", "아키텍처"), + "experiments": ("experiment", "experiments", "evaluation", "results", "experimento", "experimentos", "evaluación", "evaluacion", "resultados", "expérience", "experience", "expériences", "experiences", "évaluation", "experimente", "auswertung", "ergebnisse", "avaliação", "avaliacao", "实验", "评估", "结果", "実験", "評価", "結果", "실험", "평가", "결과"), + "conclusion": ("conclusion", "conclusions", "discussion", "conclusión", "conclusiones", "discusión", "discusion", "conclusão", "conclusao", "conclusões", "conclusoes", "discussão", "discussao", "schlussfolgerung", "schlussfolgerungen", "fazit", "diskussion", "结论", "讨论", "总结", "結論", "考察", "まとめ", "결론", "논의", "요약"), +} + +OCR_PROTECTED_WORDS = { + "acknowledgements", "architecture", "autoregressive", "bidirectional", "classification", "configuration", + "convolutional", "deterministic", "differentiable", "downstream", "embedding", "embeddings", "evaluation", + "experimental", "experiments", "finetuning", "implementation", "information", "initialization", "introduction", + "methodology", "multiframe", "multimodal", "optimization", "performance", "positional", "pretraining", + "probabilities", "representation", "representations", "retrieval", "scientific", "segmentation", "transformer", + "visualization", +} +OCR_SPLIT_ANCHORS = OCR_PROTECTED_WORDS | { + "and", "decoder", "encoder", "evidence", "for", "from", "generation", "into", "method", "model", "of", + "output", "retriever", "the", "to", "with", +} +OCR_RENDER_DPI = 84 +LATIN_CAPTION_PREFIX = re.compile( + r"^(?Pfigure|fig\.?|table|figura|abbildung|tabla|tableau|tabelle)" + r"(?=\s|\d)\s*(?P[a-z]?\d+(?:[.\-]\d+)*|[ivxlcdm]+)\b", + re.IGNORECASE, +) + + +def _caption_prefix_kind(text: str) -> str | None: + value = _clean(text) + match = LATIN_CAPTION_PREFIX.match(value) + if match: + prefix = match.group("prefix").casefold() + return "table" if prefix in {"table", "tabla", "tableau", "tabelle"} else "caption" + if re.match(r"^(图|圖|表)\s*[a-z0-9一二三四五六七八九十]+", value, re.IGNORECASE): + return "caption" if value.startswith(("图", "圖")) else "table" + return None def _clean(text: str) -> str: - return re.sub(r"[ \t]+", " ", str(text or "")).strip() + value = unicodedata.normalize("NFKC", str(text or "")).replace("\u00ad", "") + value = value.replace("\r\n", "\n").replace("\r", "\n") + previous = None + while previous != value: + previous = value + value = re.sub(r"\b([A-Z])\s+([A-Z]{2,})\b", r"\1\2", value) + value = re.sub(r"\bANIMAGE\b", "AN IMAGE", value) + value = re.sub(r"\(\s*([A-Z])\s+([A-Z])\s+([A-Z])\s*\)", r"(\1\2\3)", value) + value = re.sub(r"[ \t]+", " ", value) + value = re.sub(r"(?<=\w)-\s+(?=\w)", "", value) + value = re.sub(r"\n{3,}", "\n\n", value) + return value.strip() -def _read_pages(path: Path, max_chars: int) -> tuple[list[dict[str, Any]], str]: - suffix = path.suffix.lower() - pages: list[dict[str, Any]] = [] - loader = "plain" - if suffix == ".pdf": - loader = "pymupdf" +def _repair_ocr_spacing(text: str, splitter: Callable[[str], list[str]] | None = None) -> tuple[str, int]: + value = _clean(text) + repairs = 0 + value, count = re.subn(r"\b(figure|fig\.|table|figura|abbildung|tabla|tableau|tabelle)(?=\d)", r"\1 ", value, flags=re.IGNORECASE) + repairs += count + section_terms = ( + "abstract|introduction|background|method|methods|approach|experiments|results|conclusion|discussion|references|appendix|" + "resumen|introducción|introduccion|método|metodo|métodos|metodos|metodología|metodologia|enfoque|" + "experimento|experimentos|evaluación|evaluacion|resultados|conclusión|conclusiones|discusión|discusion|" + "résumé|méthode|méthodes|expérience|expériences|évaluation|résultats|" + "zusammenfassung|einleitung|methoden|methodik|experimente|ergebnisse|schlussfolgerung|fazit|" + "resumo|introdução|introducao|avaliação|avaliacao|conclusão|conclusao" + ) + value, count = re.subn(rf"(? str: + nonlocal repairs + token = match.group(0) + normalized = token.casefold() + if normalized in OCR_PROTECTED_WORDS or token.isupper() or (token[1:] != token[1:].lower() and token[1:] != token[1:].upper()): + return token + pieces = [str(piece) for piece in splitter(token) if str(piece)] + lowered = [piece.casefold() for piece in pieces] + anchor_count = sum(piece in OCR_SPLIT_ANCHORS for piece in lowered) + if len(pieces) < 2 or any(len(piece) < 2 for piece in pieces) or anchor_count < 2: + return token + repairs += len(pieces) - 1 + return " ".join(pieces) + + value = re.sub(r"[A-Za-z]{8,160}", split_token, value) + return _clean(value), repairs + + +def _normalized_compare_text(text: str) -> str: + return "".join(char for char in _clean(text).casefold() if char.isalnum()) + + +def _lexical_units(text: str) -> list[str]: + clean = _clean(text).casefold() + units = [f"latin:{value}" for value in re.findall(r"[a-z]{3,}", clean)] + cjk = "".join(re.findall(r"[\u3400-\u9fff\u3040-\u30ff\uac00-\ud7af]", clean)) + if len(cjk) == 1: + units.append(f"cjk:{cjk}") + else: + units.extend(f"cjk:{cjk[index:index + 2]}" for index in range(len(cjk) - 1)) + return units + + +def _cjk_character_count(text: str) -> int: + return sum( + "\u3400" <= char <= "\u9fff" + or "\u3040" <= char <= "\u30ff" + or "\uac00" <= char <= "\ud7af" + for char in str(text or "") + ) + + +def _token_overlap_agreement(left: str, right: str) -> float | None: + left_tokens = _lexical_units(left) + right_tokens = _lexical_units(right) + if not left_tokens or not right_tokens: + return None + overlap = sum((Counter(left_tokens) & Counter(right_tokens)).values()) + return round(overlap / max(1, len(left_tokens)), 4) + + +def _replacement_rate(text: str) -> float: + value = str(text or "") + return round(value.count("\ufffd") / max(1, len(value)), 6) + + +def _mojibake_rate(text: str) -> float: + value = str(text or "") + return round(sum(value.count(marker) for marker in MOJIBAKE_MARKERS) / max(1, len(value)), 6) + + +def _looks_like_typographic_heading(text: str) -> bool: + value = _clean(text) + words = re.findall(r"[A-Za-z][A-Za-z0-9+&/-]*", value) + first_alpha = next((char for char in value if char.isalpha()), "") + first_char = value[0] if value else "" + if not 2 <= len(value) <= 100 or not 1 <= len(words) <= 10: + return False + if not first_char.isalpha() or (first_char.isascii() and not re.search(r"[a-z]", value)): + return False + if re.match(r"^(?:figure|fig\.|table)\b", value, re.IGNORECASE) or re.search(r"[=<>]{1,2}|\bdoi\b|https?://", value, re.IGNORECASE): + return False + if value.endswith(":") and len(words) <= 2: + return False + if value.endswith((",", ";", "?", "!")) or (value.endswith(".") and len(words) > 5): + return False + if first_alpha and first_alpha.lower() != first_alpha.upper() and not first_alpha.isupper(): + return False + return True + + +def _block_kind(text: str, width_ratio: float = 0.0) -> str: + value = _clean(text) + caption_kind = _caption_prefix_kind(value) + if caption_kind: + return caption_kind + section_aliases = tuple(alias.casefold() for aliases in SECTION_ALIASES.values() for alias in aliases) + if len(value) <= 100 and any(value.casefold() == alias or value.casefold().startswith((f"{alias} ", f"{alias}:", f"{alias}：")) for alias in section_aliases): + return "heading" + if re.match(r"^(abstract|\d+(?:\.\d+)*\s+|[ivx]+[.)]\s+)(.{2,100})$", value, re.IGNORECASE) and len(value) <= 140: + return "heading" + if len(value) <= 180 and width_ratio >= 0.55 and not value.endswith((".", ",", ";")): + return "title" + if re.search(r"(?:[A-Za-z][A-Za-z0-9_{}^\-]*\s*=\s*[^=\n]{3,}|∑|∫|≤|≥|≈)", value): + return "formula" + return "paragraph" + + +def _column_groups(items: list[dict[str, Any]], page_width: float, max_columns: int = 3) -> list[list[dict[str, Any]]]: + if len(items) < 4: + return [items] + positioned = sorted(items, key=lambda item: float(item["bbox"][0])) + gaps = [ + (float(positioned[index + 1]["bbox"][0]) - float(positioned[index]["bbox"][0]), index) + for index in range(len(positioned) - 1) + ] + qualified_gaps = [(gap, index) for gap, index in sorted(gaps, reverse=True) if gap >= page_width * 0.12] + if not qualified_gaps: + return [items] + + def median(values: list[float]) -> float: + ordered = sorted(values) + middle = len(ordered) // 2 + return ordered[middle] if len(ordered) % 2 else (ordered[middle - 1] + ordered[middle]) / 2.0 + + def validated_groups(split_count: int) -> list[list[dict[str, Any]]] | None: + split_indexes = sorted(index for _, index in qualified_gaps[:split_count]) + groups: list[list[dict[str, Any]]] = [] + start = 0 + for split in split_indexes: + groups.append(positioned[start:split + 1]) + start = split + 1 + groups.append(positioned[start:]) + if len(groups) > max_columns or any(len(group) < 2 for group in groups): + return None + ordered_groups = sorted(groups, key=lambda group: median([float(item["bbox"][0]) for item in group])) + minimum_vertical_span = page_width * 0.12 + minimum_text = 40 + for group in ordered_groups: + vertical_span = max(float(item["bbox"][3]) for item in group) - min(float(item["bbox"][1]) for item in group) + text_count = sum(len(_normalized_compare_text(str(item.get("text") or ""))) for item in group) + if vertical_span < minimum_vertical_span or text_count < minimum_text: + return None + for left, right in zip(ordered_groups, ordered_groups[1:]): + left_right = median([float(item["bbox"][2]) for item in left]) + right_left = median([float(item["bbox"][0]) for item in right]) + if left_right > right_left + page_width * 0.04: + return None + return ordered_groups + + for split_count in range(min(max_columns - 1, len(qualified_gaps)), 0, -1): + groups = validated_groups(split_count) + if groups: + return groups + return [items] + + +def _reading_order(blocks: list[dict[str, Any]], page_width: float) -> tuple[list[dict[str, Any]], float]: + if not blocks: + return [], 0.0 + usable = [item for item in blocks if _clean(item.get("text", ""))] + if not usable: + return [], 0.0 + spanning = [item for item in usable if float(item["bbox"][2]) - float(item["bbox"][0]) >= page_width * 0.56] + non_spanning = [item for item in usable if item not in spanning] + detection_items = [ + item + for item in non_spanning + if float(item["bbox"][2]) - float(item["bbox"][0]) >= page_width * 0.20 + and len(_normalized_compare_text(str(item.get("text") or ""))) >= 20 + ] + detected_columns = _column_groups(detection_items, page_width) + if len(detected_columns) == 1: + ordered = sorted(usable, key=lambda item: (round(float(item["bbox"][1]), 1), float(item["bbox"][0]))) + confidence = 0.96 + for item in ordered: + item["column"] = 0 + else: + def median_x0(column: list[dict[str, Any]]) -> float: + values = sorted(float(item["bbox"][0]) for item in column) + middle = len(values) // 2 + return values[middle] if len(values) % 2 else (values[middle - 1] + values[middle]) / 2.0 + + anchors = [median_x0(column) for column in detected_columns] + columns = [[] for _ in anchors] + for item in non_spanning: + column_index = min(range(len(anchors)), key=lambda index: abs(float(item["bbox"][0]) - anchors[index])) + columns[column_index].append(item) + for column_index, column in enumerate(columns): + for item in column: + item["column"] = column_index + for item in spanning: + item["column"] = -1 + boundaries = sorted(spanning, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))) + ordered = [] + cursor = float("-inf") + for boundary in boundaries + [None]: + limit = float(boundary["bbox"][1]) if boundary else float("inf") + segment = [item for item in usable if item not in spanning and cursor <= float(item["bbox"][1]) < limit] + for column in columns: + ordered.extend(sorted((item for item in segment if item in column), key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0])))) + if boundary: + ordered.append(boundary) + cursor = float(boundary["bbox"][3]) + seen: set[int] = set() + ordered = [item for item in ordered if not (id(item) in seen or seen.add(id(item)))] + confidence = (0.88 if spanning else 0.82) if len(columns) == 2 else (0.84 if spanning else 0.78) + for index, item in enumerate(ordered, 1): + item["reading_order"] = index + return ordered, confidence + + +def _rotate_blocks_to_page(blocks: list[dict[str, Any]], page: Any) -> list[dict[str, Any]]: + if not int(getattr(page, "rotation", 0) or 0): + return blocks + try: + import fitz + + matrix = page.rotation_matrix + rotated = [] + for item in blocks: + value = dict(item) + rect = fitz.Rect(*item["bbox"]) * matrix + value["bbox"] = [round(float(rect.x0), 3), round(float(rect.y0), 3), round(float(rect.x1), 3), round(float(rect.y1), 3)] + rotated.append(value) + return rotated + except Exception: + return blocks + + +def _pymupdf_line_blocks(page: Any) -> list[dict[str, Any]]: + records = [] + payload = page.get_text("dict") + for parent_index, block in enumerate(payload.get("blocks", []), 1): + if int(block.get("type", 0)) != 0: + continue + for line in block.get("lines", []): + spans = [span for span in line.get("spans", []) if _clean(span.get("text", ""))] + if not spans: + continue + spans.sort(key=lambda span: float(span.get("bbox", [0, 0, 0, 0])[0])) + text = _clean(" ".join(str(span.get("text") or "") for span in spans)) + bbox_values = [span.get("bbox", [0, 0, 0, 0]) for span in spans] + bbox = [ + round(min(float(value[0]) for value in bbox_values), 3), + round(min(float(value[1]) for value in bbox_values), 3), + round(max(float(value[2]) for value in bbox_values), 3), + round(max(float(value[3]) for value in bbox_values), 3), + ] + sizes = [float(span.get("size") or 0.0) for span in spans if float(span.get("size") or 0.0) > 0] + span_bold = [ + bool(int(span.get("flags") or 0) & 16) or bool(re.search(r"(?:bold|demi|semi|medi)", str(span.get("font") or ""), re.IGNORECASE)) + for span in spans + ] + span_lengths = [max(1, len(_clean(span.get("text", "")))) for span in spans] + bold_characters = sum(length for length, bold in zip(span_lengths, span_bold) if bold) + total_characters = sum(span_lengths) + font_bold_ratio = round(bold_characters / max(1, total_characters), 4) + font_bold = font_bold_ratio >= 0.8 + records.append({ + "bbox": bbox, + "text": text, + "source": "pymupdf", + "confidence": 1.0, + "parent_block": parent_index, + "font_size": round(max(sizes), 3) if sizes else None, + "font_bold": font_bold, + "font_bold_ratio": font_bold_ratio, + }) + return records + + +def _merge_native_hyphenated_lines(records: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int]: + merged: list[dict[str, Any]] = [] + repairs = 0 + for record in records: + current = dict(record) + if not merged: + merged.append(current) + continue + previous = merged[-1] + previous_text = str(previous.get("text") or "") + current_text = str(current.get("text") or "") + match_left = re.search(r"([A-Za-z]{2,})-$", previous_text) + match_right = re.match(r"([A-Za-z]{2,})", current_text) + same_parent = previous.get("parent_block") is not None and previous.get("parent_block") == current.get("parent_block") + if not same_parent or not match_left or not match_right: + merged.append(current) + continue + combined_word = f"{match_left.group(1)}{match_right.group(1)}".casefold() + separator = "" if combined_word in OCR_PROTECTED_WORDS or len(match_left.group(1)) <= 3 else "-" + joined_text = f"{previous_text[:-1]}{separator}{current_text.lstrip()}" + previous_bbox = previous.get("bbox") or [0, 0, 0, 0] + current_bbox = current.get("bbox") or previous_bbox + previous.update({ + "text": _clean(joined_text), + "bbox": [ + round(min(float(previous_bbox[0]), float(current_bbox[0])), 3), + round(min(float(previous_bbox[1]), float(current_bbox[1])), 3), + round(max(float(previous_bbox[2]), float(current_bbox[2])), 3), + round(max(float(previous_bbox[3]), float(current_bbox[3])), 3), + ], + "confidence": min(float(previous.get("confidence") or 1.0), float(current.get("confidence") or 1.0)), + "font_size": max(float(previous.get("font_size") or 0.0), float(current.get("font_size") or 0.0)) or None, + "font_bold": bool(previous.get("font_bold")) and bool(current.get("font_bold")), + "native_hyphenation_repairs": int(previous.get("native_hyphenation_repairs") or 0) + 1, + }) + repairs += 1 + return merged, repairs + + +def _poppler_pages(path: Path, timeout: int = 30) -> tuple[list[str], str | None]: + executable = shutil.which("pdftotext") + if not executable: + return [], "pdftotext unavailable" + try: + with tempfile.TemporaryDirectory() as temp: + target = Path(temp) / "paper.txt" + subprocess.run([executable, "-layout", str(path), str(target)], check=True, timeout=timeout, capture_output=True) + return target.read_text(encoding="utf-8", errors="replace").split("\f"), None + except Exception as exc: + return [], str(exc) + + +def _pdfplumber_blocks(path: Path, page_number: int) -> list[dict[str, Any]]: + try: + import pdfplumber + + with pdfplumber.open(str(path)) as document: + page = document.pages[page_number - 1] + words = page.extract_words(use_text_flow=False, keep_blank_chars=False) or [] + rows: list[list[dict[str, Any]]] = [] + for word in sorted(words, key=lambda item: (float(item.get("top", 0.0)), float(item.get("x0", 0.0)))): + row = next((items for items in rows if abs(float(items[0].get("top", 0.0)) - float(word.get("top", 0.0))) <= 3.0), None) + if row is None: + row = [] + rows.append(row) + row.append(word) + blocks = [] + for row in rows: + row.sort(key=lambda item: float(item.get("x0", 0.0))) + text = _clean(" ".join(str(item.get("text") or "") for item in row)) + if text: + blocks.append({ + "bbox": [min(float(item["x0"]) for item in row), min(float(item["top"]) for item in row), max(float(item["x1"]) for item in row), max(float(item["bottom"]) for item in row)], + "text": text, + "source": "pdfplumber", + "confidence": 0.92, + }) + return blocks + except Exception: + return [] + + +def _pdf_metadata(path: Path) -> dict[str, Any]: + try: + from pypdf import PdfReader + + reader = PdfReader(str(path)) + if reader.is_encrypted: + try: + unlocked = reader.decrypt("") + except Exception: + unlocked = 0 + if not unlocked: + raise ValueError("PDF is encrypted and cannot be opened without a password") + return {"page_count": len(reader.pages), "encrypted": bool(reader.is_encrypted), "metadata": {str(key): str(value) for key, value in (reader.metadata or {}).items()}} + except ImportError: + return {"page_count": None, "encrypted": None, "metadata": {}, "warning": "pypdf unavailable"} + + +def _validate_pdf_container(path: Path) -> None: + size = path.stat().st_size + if size < 8: + raise ValueError("PDF is empty or too small to contain a valid document") + with path.open("rb") as handle: + header = handle.read(min(1024, size)) + handle.seek(max(0, size - 4096)) + tail = handle.read() + if b"%PDF-" not in header: + raise ValueError("File does not contain a valid PDF header") + if b"%%EOF" not in tail: + raise ValueError("PDF appears truncated because the EOF marker is missing") + + +def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | None, rapidocr_threads: int = 1) -> tuple[list[dict[str, Any]], str, dict[str, Any]]: + if adapter: + return list(adapter(image_path, lang) or []), "adapter", {} + from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr_detailed + + requested = engine + if requested == "auto": + requested = "rapidocr" try: - import fitz - - document = fitz.open(str(path)) - used = 0 - for index, page in enumerate(document, 1): - text = page.get_text("text") - remaining = max_chars - used - if remaining <= 0: + import rapidocr_onnxruntime # noqa: F401 + except Exception: + requested = "easyocr" + try: + import easyocr # noqa: F401 + except Exception: + requested = "paddle" + if requested == "rapidocr": + records, diagnostics = run_rapidocr_detailed(image_path, lang, threads=rapidocr_threads) + return records, "rapidocr", diagnostics + if requested == "easyocr": + return run_easyocr(image_path, lang), "easyocr", {} + if requested == "paddle": + return run_paddle_ocr(image_path, lang), "paddle", {} + return [], "off", {} + + +def _group_ocr_words(records: list[dict[str, Any]], image_width: float) -> list[dict[str, Any]]: + words = [] + for record in records: + quad = record.get("quad") or [] + if len(quad) < 4 or not _clean(record.get("text", "")): + continue + xs = [float(point[0]) for point in quad] + ys = [float(point[1]) for point in quad] + words.append({ + "bbox": [min(xs), min(ys), max(xs), max(ys)], + "text": _clean(record.get("text", "")), + "confidence": float(record.get("confidence") or 0.0), + }) + words.sort(key=lambda item: ((item["bbox"][1] + item["bbox"][3]) / 2.0, item["bbox"][0])) + lines: list[dict[str, Any]] = [] + for word in words: + bbox = word["bbox"] + center_y = (bbox[1] + bbox[3]) / 2.0 + height = max(1.0, bbox[3] - bbox[1]) + line = next(( + item for item in reversed(lines[-12:]) + if abs(center_y - item["center_y"]) <= max(height, item["mean_height"]) * 0.62 + and bbox[0] >= item["bbox"][0] - image_width * 0.08 + ), None) + if line is None: + lines.append({"words": [word], "bbox": list(bbox), "center_y": center_y, "mean_height": height}) + continue + line["words"].append(word) + line["bbox"] = [min(line["bbox"][0], bbox[0]), min(line["bbox"][1], bbox[1]), max(line["bbox"][2], bbox[2]), max(line["bbox"][3], bbox[3])] + line["center_y"] = sum((item["bbox"][1] + item["bbox"][3]) / 2.0 for item in line["words"]) / len(line["words"]) + line["mean_height"] = sum(max(1.0, item["bbox"][3] - item["bbox"][1]) for item in line["words"]) / len(line["words"]) + + grouped = [] + for line in lines: + line["words"].sort(key=lambda item: item["bbox"][0]) + grouped.append({ + "bbox": line["bbox"], + "text": _clean(" ".join(item["text"] for item in line["words"])), + "confidence": round(sum(item["confidence"] for item in line["words"]) / max(1, len(line["words"])), 4), + }) + grouped.sort(key=lambda item: (item["bbox"][1], item["bbox"][0])) + heights = sorted(max(1.0, item["bbox"][3] - item["bbox"][1]) for item in grouped) + median_height = heights[len(heights) // 2] if heights else 12.0 + parent = 0 + previous_by_column: dict[int, dict[str, Any]] = {} + for item in grouped: + center_x = (item["bbox"][0] + item["bbox"][2]) / 2.0 + width = item["bbox"][2] - item["bbox"][0] + column = 2 if width >= image_width * 0.62 else 0 if center_x <= image_width / 2.0 else 1 + previous = previous_by_column.get(column) + gap = item["bbox"][1] - previous["bbox"][3] if previous else float("inf") + starts_new = bool( + re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+)\b", item["text"], re.IGNORECASE) + or _caption_prefix_kind(item["text"]) + ) + if previous is None or gap > median_height * 1.8 or starts_new: + parent += 1 + item["parent_block"] = parent + previous_by_column[column] = item + return grouped + + +def _assign_ocr_parent_blocks(lines: list[dict[str, Any]], image_width: float) -> list[dict[str, Any]]: + ordered = sorted(lines, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))) + heights = sorted(max(1.0, float(item["bbox"][3]) - float(item["bbox"][1])) for item in ordered) + median_height = heights[len(heights) // 2] if heights else 12.0 + parent = 0 + previous_by_column: dict[int, dict[str, Any]] = {} + last_item: dict[str, Any] | None = None + for item in ordered: + center_x = (float(item["bbox"][0]) + float(item["bbox"][2])) / 2.0 + width = float(item["bbox"][2]) - float(item["bbox"][0]) + column = 2 if width >= image_width * 0.62 else 0 if center_x <= image_width / 2.0 else 1 + previous = previous_by_column.get(column) + if last_item and last_item.get("_caption_group"): + last_gap = float(item["bbox"][1]) - float(last_item["bbox"][3]) + last_height = max( + median_height, + float(item["bbox"][3]) - float(item["bbox"][1]), + float(last_item["bbox"][3]) - float(last_item["bbox"][1]), + ) + if last_gap <= last_height * 1.15: + previous = last_item + gap = float(item["bbox"][1]) - float(previous["bbox"][3]) if previous else float("inf") + local_height = max( + median_height, + float(item["bbox"][3]) - float(item["bbox"][1]), + (float(previous["bbox"][3]) - float(previous["bbox"][1])) if previous else 0.0, + ) + text = str(item.get("text") or "") + starts_new = bool(re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+)\b", text, re.IGNORECASE) or _caption_prefix_kind(text)) + starts_caption = bool(_caption_prefix_kind(text)) + continuing_caption = bool(previous and previous.get("_caption_group") and gap <= local_height * 1.15 and not starts_new) + paragraph_break = bool(previous and not continuing_caption and str(previous.get("text") or "").rstrip().endswith((".", ":")) and gap > local_height * 0.45) + if previous is None or gap > local_height * 1.15 or starts_new or paragraph_break: + parent += 1 + item["parent_block"] = parent + item["_caption_group"] = starts_caption or continuing_caption + previous_by_column[column] = item + last_item = item + return ordered + + +def _filter_ocr_margin_noise(records: list[dict[str, Any]], image_width: float) -> tuple[list[dict[str, Any]], int]: + anchors = [ + item + for item in records + if len(_normalized_compare_text(str(item.get("text") or ""))) >= 20 + and float(item["bbox"][2]) - float(item["bbox"][0]) >= image_width * 0.30 + ] + if len(anchors) < 3: + return records, 0 + left = sorted(float(item["bbox"][0]) for item in anchors)[max(0, len(anchors) // 10 - 1)] + right_values = sorted(float(item["bbox"][2]) for item in anchors) + right = right_values[min(len(right_values) - 1, len(right_values) - max(1, len(anchors) // 10))] + margin = image_width * 0.035 + kept = [] + removed = 0 + for item in records: + bbox = item.get("bbox") or [0, 0, 0, 0] + outside = float(bbox[2]) < left - margin or float(bbox[0]) > right + margin + width = max(1.0, float(bbox[2]) - float(bbox[0])) + height = max(1.0, float(bbox[3]) - float(bbox[1])) + short = len(_normalized_compare_text(str(item.get("text") or ""))) < 12 + vertical = height > width * 1.15 + outer_band = float(bbox[2]) < image_width * 0.12 or float(bbox[0]) > image_width * 0.88 + compact_margin_fragment = outer_band and short and (vertical or width < image_width * 0.08) + if (outside and (short or vertical)) or compact_margin_fragment: + removed += 1 + continue + kept.append(item) + return kept, removed + + +def _repeated_margin_signature(text: str) -> str: + value = _clean(text).casefold() + value = re.sub(r"\d+", "#", value) + value = re.sub(r"\s+", " ", value).strip(" |.-_") + return value + + +def _semantic_margin_region(page: dict[str, Any], bbox: list[float] | tuple[float, ...]) -> str | None: + width = float(page.get("width") or 0.0) + height = float(page.get("height") or 0.0) + if width <= 0 or height <= 0: + return None + rotation = int(page.get("rotation") or 0) % 360 + if rotation == 90: + if float(bbox[0]) >= width * 0.88: + return "header" + if float(bbox[2]) <= width * 0.12: + return "footer" + elif rotation == 180: + if float(bbox[1]) >= height * 0.88: + return "header" + if float(bbox[3]) <= height * 0.12: + return "footer" + elif rotation == 270: + if float(bbox[2]) <= width * 0.12: + return "header" + if float(bbox[0]) >= width * 0.88: + return "footer" + else: + if float(bbox[3]) <= height * 0.12: + return "header" + if float(bbox[1]) >= height * 0.88: + return "footer" + return None + + +def _remove_repeated_margin_noise(pages: list[dict[str, Any]]) -> int: + if len(pages) < 3: + return 0 + occurrences: dict[tuple[str, str], set[int]] = {} + candidates: dict[tuple[int, str], tuple[str, str]] = {} + for page in pages: + page_number = int(page.get("page") or 0) + for block in page.get("blocks", []): + bbox = block.get("bbox") + text = str(block.get("text") or "").strip() + if str(block.get("kind") or "") == "heading" or bool(block.get("font_bold")): + continue + if not isinstance(bbox, (list, tuple)) or len(bbox) != 4 or not text or len(text) > 140 or len(_lexical_units(text)) > 12 or _cjk_character_count(text) > 36: + continue + region = _semantic_margin_region(page, bbox) + if not region: + continue + signature = _repeated_margin_signature(text) + if not signature: + continue + key = (region, signature) + occurrences.setdefault(key, set()).add(page_number) + candidates[(page_number, str(block.get("id") or ""))] = key + + threshold = max(3, (len(pages) + 3) // 4) + repeated = {key for key, page_numbers in occurrences.items() if len(page_numbers) >= threshold} + if not repeated: + return 0 + + removed = 0 + for page in pages: + page_number = int(page.get("page") or 0) + kept = [] + page_removed = 0 + for block in page.get("blocks", []): + key = candidates.get((page_number, str(block.get("id") or ""))) + if key in repeated: + page_removed += 1 + continue + kept.append(block) + if not page_removed: + continue + for order, block in enumerate(kept, 1): + block["reading_order"] = order + page_text = "\n\n".join(str(item.get("text") or "") for item in kept) + page.update({ + "blocks": kept, + "text": page_text, + "char_count": len(page_text), + "replacement_character_rate": _replacement_rate(page_text), + "mojibake_rate": _mojibake_rate(page_text), + "column_count": max((int(item.get("column", 0)) for item in kept), default=0) + 1, + "repeated_margin_noise_removed": page_removed, + }) + removed += page_removed + return removed + + +def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Callable | None, rapidocr_threads: int = 1) -> tuple[list[dict[str, Any]], str, str | None, dict[str, Any]]: + try: + with tempfile.TemporaryDirectory() as temp: + target = Path(temp) / f"page_{page_number:03d}.png" + dpi = OCR_RENDER_DPI if adapter is None and engine in {"auto", "easyocr", "rapidocr"} else 160 + render_started = time.monotonic() + pixmap = page.get_pixmap(dpi=dpi, alpha=False) + pixmap.save(str(target)) + render_seconds = time.monotonic() - render_started + records, used_engine, engine_diagnostics = _ocr_records(target, engine, lang, adapter, rapidocr_threads=rapidocr_threads) + postprocess_started = time.monotonic() + spacing_repairs = 0 + for record in records: + repaired, count = _repair_ocr_spacing(record.get("text", "")) + record["text"] = repaired + spacing_repairs += count + if used_engine != "rapidocr": + records = _group_ocr_words(records, float(pixmap.width)) + else: + normalized_records = [] + for record in records: + quad = record.get("quad") or [] + if len(quad) < 4: + continue + xs = [float(point[0]) for point in quad] + ys = [float(point[1]) for point in quad] + normalized_records.append({"bbox": [min(xs), min(ys), max(xs), max(ys)], "text": _clean(record.get("text", "")), "confidence": float(record.get("confidence") or 0.0)}) + records = _assign_ocr_parent_blocks(normalized_records, float(pixmap.width)) + records, margin_noise_removed = _filter_ocr_margin_noise(records, float(pixmap.width)) + blocks = [] + scale_x = float(page.rect.width) / max(1, pixmap.width) + scale_y = float(page.rect.height) / max(1, pixmap.height) + for record in records: + raw_bbox = record.get("bbox") + if not raw_bbox: + continue + blocks.append({ + "bbox": [float(raw_bbox[0]) * scale_x, float(raw_bbox[1]) * scale_y, float(raw_bbox[2]) * scale_x, float(raw_bbox[3]) * scale_y], + "text": _clean(record.get("text", "")), + "source": used_engine, + "confidence": round(float(record.get("confidence") or 0.0), 4), + "parent_block": record.get("parent_block"), + }) + return blocks, used_engine, None, { + "margin_noise_removed": margin_noise_removed, + "spacing_repairs": spacing_repairs, + "render_seconds": round(render_seconds, 4), + "render_dpi": dpi, + "postprocess_seconds": round(time.monotonic() - postprocess_started, 4), + **engine_diagnostics, + } + except Exception as exc: + return [], engine, str(exc), {"margin_noise_removed": 0, "spacing_repairs": 0} + + +def _rapidocr_worker_count(engine: str, adapter: Callable | None, page_count: int) -> int: + if adapter is not None or page_count < 2 or engine not in {"auto", "rapidocr"}: + return 1 + if engine == "auto": + try: + import rapidocr_onnxruntime # noqa: F401 + except Exception: + return 1 + cpu_count = int(os.cpu_count() or 1) + default_workers = 4 if cpu_count >= 8 else 2 if cpu_count >= 4 else 1 + try: + configured = int(os.getenv("RFS_OCR_WORKERS") or default_workers) + except ValueError: + configured = default_workers + return max(1, min(int(page_count), configured, cpu_count)) + + +def _deadline_ocr_wave( + path: Path, + page_numbers: list[int], + engine: str, + lang: str, + deadline_at: float, +) -> list[tuple[int, tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], float]]: + """Run OCR pages in killable child processes and preserve completed results.""" + cutoff = max(time.monotonic(), float(deadline_at) - 20.0) + worker_threads = 2 if len(page_numbers) == 1 and engine in {"auto", "rapidocr"} else 1 + creation_flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) if os.name == "nt" else 0 + with tempfile.TemporaryDirectory() as temp: + processes: dict[int, dict[str, Any]] = {} + for page_number in page_numbers: + output = Path(temp) / f"page_{page_number:03d}.json" + command = [ + sys.executable, + "-m", + "rfs.paper_to_image.ocr_worker", + "--paper", + str(path), + "--page", + str(page_number), + "--engine", + engine, + "--lang", + lang, + "--threads", + str(worker_threads), + "--out", + str(output), + ] + started = time.monotonic() + try: + process = subprocess.Popen( + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + creationflags=creation_flags, + ) + processes[page_number] = {"process": process, "output": output, "started": started, "launch_error": None} + except Exception as exc: + processes[page_number] = {"process": None, "output": output, "started": started, "launch_error": str(exc)} + + pending = {page for page, item in processes.items() if item["process"] is not None} + while pending and time.monotonic() < cutoff: + finished = {page for page in pending if processes[page]["process"].poll() is not None} + pending.difference_update(finished) + if pending: + time.sleep(0.02) + + timed_out = set(pending) + for page_number in timed_out: + process = processes[page_number]["process"] + try: + process.terminate() + except Exception: + pass + for page_number in timed_out: + process = processes[page_number]["process"] + try: + process.wait(timeout=1.0) + except Exception: + try: + process.kill() + process.wait(timeout=1.0) + except Exception: + pass + + results = [] + for page_number in page_numbers: + item = processes[page_number] + process = item["process"] + elapsed = time.monotonic() - float(item["started"]) + stderr = "" + if process is not None: + try: + _stdout, stderr = process.communicate(timeout=0.2) + except Exception: + stderr = "" + if item["launch_error"]: + result = ([], engine, f"OCR worker launch failed: {item['launch_error']}", {"worker_process": True}) + elif page_number in timed_out: + result = ([], engine, "OCR exceeded extraction deadline", {"worker_process": True, "timed_out": True}) + elif item["output"].exists(): + try: + payload = json.loads(item["output"].read_text(encoding="utf-8")) + result = ( + list(payload.get("blocks") or []), + str(payload.get("engine") or engine), + str(payload.get("error")) if payload.get("error") else None, + {"worker_process": True, **dict(payload.get("diagnostics") or {})}, + ) + elapsed = float(payload.get("elapsed_seconds") or elapsed) + except Exception as exc: + result = ([], engine, f"OCR worker returned invalid output: {exc}", {"worker_process": True}) + else: + detail = stderr.strip() or f"worker exited with code {getattr(process, 'returncode', None)}" + result = ([], engine, f"OCR worker failed: {detail}", {"worker_process": True}) + results.append((page_number, result, elapsed)) + return results + + +def _prioritize_ocr_candidates( + pages: list[dict[str, Any]], + candidates: list[int], + max_pages: int, +) -> tuple[list[int], list[dict[str, Any]]]: + if max_pages <= 0 or not candidates: + return [], [] + page_count = max(1, len(pages)) + signals = { + "overview_figure": re.compile(r"\b(?:figure|fig\.)\s*[12]\b.*\b(?:overview|framework|architecture|pipeline|system|model)\b", re.IGNORECASE | re.DOTALL), + "method": re.compile(r"\b(?:methods?|methodology|approach|architecture|system overview|framework)\b", re.IGNORECASE), + "abstract": re.compile(r"\babstract\b", re.IGNORECASE), + "conclusion": re.compile(r"\b(?:conclusions?|discussion)\b", re.IGNORECASE), + "caption": re.compile(r"\b(?:figure|fig\.|table)\s*\d", re.IGNORECASE), + } + scored: list[tuple[int, int, list[str]]] = [] + for page_number in candidates: + page = pages[page_number - 1] if 0 < page_number <= len(pages) else {} + text = str(page.get("text") or "") + reasons: list[str] = [] + score = 0 + for name, weight in (("overview_figure", 14), ("method", 10), ("abstract", 9), ("conclusion", 8), ("caption", 5)): + if signals[name].search(text): + score += weight + reasons.append(name) + if page_number == 1: + score += 7 + reasons.append("first_page") + elif page_number <= 3: + score += 3 + reasons.append("early_page") + scored.append((score, page_number, reasons)) + + selected: list[int] = [] + reason_map: dict[int, list[str]] = {} + for score, page_number, reasons in sorted(scored, key=lambda item: (-item[0], item[1])): + if score <= 0 or len(selected) >= max_pages: + break + selected.append(page_number) + reason_map[page_number] = reasons + + anchors = [ + 1, + min(2, page_count), + min(3, page_count), + min(4, page_count), + max(1, round(page_count * 0.85)), + max(1, round(page_count * 0.50)), + ] + for anchor in anchors: + if len(selected) >= max_pages: + break + if anchor in selected: + continue + available = [value for value in candidates if value not in selected] + if not available: + break + page_number = min(available, key=lambda value: (abs(value - anchor), value)) + selected.append(page_number) + reason_map.setdefault(page_number, []).append(f"coverage_anchor_{anchor}") + + for page_number in candidates: + if len(selected) >= max_pages: + break + if page_number not in selected: + selected.append(page_number) + reason_map.setdefault(page_number, []).append("remaining_candidate") + + details = [ + {"page": page_number, "rank": rank, "reasons": reason_map.get(page_number, [])} + for rank, page_number in enumerate(selected, 1) + ] + return selected, details + + +def _read_pdf_pages( + path: Path, + max_chars: int, + deadline_at: float | None, + ocr_engine: str, + ocr_lang: str, + ocr_adapter: Callable | None, + max_ocr_pages: int, + ocr_rescue_min_remaining: float, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + try: + import fitz + except Exception as exc: + raise RuntimeError(f"PyMuPDF is required for PDF extraction: {exc}") from exc + + metadata = _pdf_metadata(path) + started = time.monotonic() + document = fitz.open(str(path)) + poppler_text, poppler_warning = _poppler_pages(path) + pages: list[dict[str, Any]] = [] + warnings: list[str] = [poppler_warning] if poppler_warning else [] + used_chars = 0 + ocr_candidates: list[int] = [] + try: + for index, page in enumerate(document, 1): + native_width = float(page.mediabox.width) if int(page.rotation or 0) else float(page.rect.width) + raw_blocks, hyphenation_repairs = _merge_native_hyphenated_lines(_pymupdf_line_blocks(page)) + ordered, order_confidence = _reading_order(raw_blocks, native_width) + if len(_normalized_compare_text(" ".join(item["text"] for item in ordered))) < 80: + fallback_blocks, fallback_hyphenation_repairs = _merge_native_hyphenated_lines(_pdfplumber_blocks(path, index)) + fallback_ordered, fallback_confidence = _reading_order(fallback_blocks, native_width) + if len(_normalized_compare_text(" ".join(item["text"] for item in fallback_ordered))) > len(_normalized_compare_text(" ".join(item["text"] for item in ordered))): + ordered, order_confidence = fallback_ordered, min(fallback_confidence, 0.9) + hyphenation_repairs = fallback_hyphenation_repairs + ordered = _rotate_blocks_to_page(ordered, page) + for block_index, block in enumerate(ordered, 1): + block["id"] = f"P{index:03d}_B{block_index:03d}" + block["kind"] = _block_kind(block["text"], (block["bbox"][2] - block["bbox"][0]) / max(1.0, float(page.rect.width))) + page_text = "\n\n".join(item["text"] for item in ordered) + poppler_page = poppler_text[index - 1] if index - 1 < len(poppler_text) else "" + native_norm = _normalized_compare_text(page_text) + poppler_norm = _normalized_compare_text(poppler_page) + order_agreement = round(SequenceMatcher(None, native_norm[:20000], poppler_norm[:20000]).ratio(), 4) if native_norm and poppler_norm else None + agreement = _token_overlap_agreement(page_text, poppler_page) + abnormal = len(native_norm) < 80 or _replacement_rate(page_text) > 0.005 or _mojibake_rate(page_text) > 0.005 or (agreement is not None and agreement < 0.65) + if abnormal: + ocr_candidates.append(index) + pages.append({ + "page": index, + "width": round(float(page.rect.width), 3), + "height": round(float(page.rect.height), 3), + "rotation": int(page.rotation or 0), + "column_count": max((int(item.get("column", 0)) for item in ordered), default=0) + 1, + "blocks": ordered, + "text": page_text, + "char_count": len(page_text), + "replacement_character_rate": _replacement_rate(page_text), + "mojibake_rate": _mojibake_rate(page_text), + "poppler_agreement": agreement, + "poppler_order_agreement": order_agreement, + "reading_order_confidence": order_confidence, + "native_hyphenation_repair_count": hyphenation_repairs, + "used_ocr": False, + }) + used_chars += len(page_text) + if used_chars >= max_chars and "document text exceeds evidence budget; evidence will be sampled across all pages" not in warnings: + warnings.append("document text exceeds evidence budget; evidence will be sampled across all pages") + finally: + pass + + ocr_pages: list[int] = [] + ocr_page_durations: list[dict[str, Any]] = [] + prioritized: list[int] = [] + ocr_priority: list[dict[str, Any]] = [] + ocr_worker_count = 1 + if ocr_engine != "off": + prioritized, ocr_priority = _prioritize_ocr_candidates(pages, ocr_candidates, max_ocr_pages) + ocr_worker_count = _rapidocr_worker_count(ocr_engine, ocr_adapter, len(prioritized)) + + def consume_ocr_result(page_number: int, result: tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], ocr_elapsed: float) -> None: + blocks, used_engine, error, ocr_diagnostics = result + ocr_page_durations.append({"page": page_number, "engine": used_engine, "elapsed_seconds": round(ocr_elapsed, 3), "success": not bool(error), **ocr_diagnostics}) + if error: + warnings.append(f"page {page_number} OCR unavailable: {error}") + return + current = pages[page_number - 1] + ordered, order_confidence = _reading_order(blocks, float(current["width"])) + for block_index, block in enumerate(ordered, 1): + block["id"] = f"P{page_number:03d}_B{block_index:03d}" + block["kind"] = _block_kind(block["text"], (block["bbox"][2] - block["bbox"][0]) / max(1.0, float(current["width"]))) + ocr_text = "\n\n".join(item["text"] for item in ordered) + if len(_normalized_compare_text(ocr_text)) > len(_normalized_compare_text(current["text"])): + current.update({ + "blocks": ordered, + "text": ocr_text, + "char_count": len(ocr_text), + "replacement_character_rate": _replacement_rate(ocr_text), + "mojibake_rate": _mojibake_rate(ocr_text), + "reading_order_confidence": order_confidence, + "column_count": max((int(item.get("column", 0)) for item in ordered), default=0) + 1, + "used_ocr": True, + "ocr_engine": used_engine, + "ocr_margin_noise_removed": int(ocr_diagnostics.get("margin_noise_removed") or 0), + "native_hyphenation_repair_count": 0, + }) + ocr_pages.append(page_number) + + def deadline_allows_wave(rescue_page: bool = False) -> bool: + if deadline_at is not None: + remaining = deadline_at - time.monotonic() + if rescue_page: + if remaining < max(25.0, float(ocr_rescue_min_remaining)): + warnings.append("OCR stopped to preserve deadline validation budget") + return False + return True + completed_times = [float(item["elapsed_seconds"]) for item in ocr_page_durations if isinstance(item.get("elapsed_seconds"), (int, float))] + estimated = (max(completed_times) * 1.25) if completed_times else 45.0 + if remaining < estimated + 20: + warnings.append("OCR stopped to preserve deadline validation budget") + return False + return True + + if deadline_at is not None and ocr_adapter is None: + start = 0 + while start < len(prioritized): + rescue_page = start > 0 + if not deadline_allows_wave(rescue_page=rescue_page): break - text = text[:remaining] - pages.append({"page": index, "text": text}) - used += len(text) - document.close() - except Exception as exc: - raise RuntimeError(f"PDF extraction failed: {exc}") from exc - elif suffix == ".docx": + wave_size = 1 if rescue_page else ocr_worker_count + batch = prioritized[start:start + wave_size] + for page_number, result, ocr_elapsed in _deadline_ocr_wave(path, batch, ocr_engine, ocr_lang, deadline_at): + consume_ocr_result(page_number, result, ocr_elapsed) + start += wave_size + elif ocr_worker_count > 1: + def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], float]: + ocr_started = time.monotonic() + try: + local_document = fitz.open(str(path)) + try: + result = _ocr_page(local_document[page_number - 1], page_number, ocr_engine, ocr_lang, ocr_adapter, rapidocr_threads=1) + return result, time.monotonic() - ocr_started + finally: + local_document.close() + except Exception as exc: + return ([], ocr_engine, str(exc), {"margin_noise_removed": 0}), time.monotonic() - ocr_started + + with ThreadPoolExecutor(max_workers=ocr_worker_count) as pool: + for start in range(0, len(prioritized), ocr_worker_count): + if not deadline_allows_wave(): + break + batch = prioritized[start:start + ocr_worker_count] + futures = {page_number: pool.submit(isolated_ocr, page_number) for page_number in batch} + for page_number in batch: + result, ocr_elapsed = futures[page_number].result() + consume_ocr_result(page_number, result, ocr_elapsed) + else: + rapidocr_threads = 2 if ocr_engine in {"auto", "rapidocr"} and ocr_adapter is None else 1 + for page_number in prioritized: + if not deadline_allows_wave(): + break + ocr_started = time.monotonic() + result = _ocr_page(document[page_number - 1], page_number, ocr_engine, ocr_lang, ocr_adapter, rapidocr_threads=rapidocr_threads) + consume_ocr_result(page_number, result, time.monotonic() - ocr_started) + repeated_margin_noise_removed = _remove_repeated_margin_noise(pages) + document.close() + elapsed = round(time.monotonic() - started, 3) + attempted_pages = [int(item["page"]) for item in ocr_page_durations if item.get("page")] + successful_attempt_pages = [int(item["page"]) for item in ocr_page_durations if item.get("page") and item.get("success")] + schedule_complete = set(prioritized).issubset(attempted_pages) + run_complete = schedule_complete and set(attempted_pages).issubset(successful_attempt_pages) + if not schedule_complete and "OCR schedule was not completed before the extraction deadline" not in warnings: + warnings.append("OCR schedule was not completed before the extraction deadline") + return pages, { + "metadata": metadata, + "ocr_candidate_pages": ocr_candidates, + "ocr_priority_pages": prioritized, + "ocr_priority": ocr_priority, + "ocr_pages": ocr_pages, + "ocr_page_durations": ocr_page_durations, + "ocr_attempted_pages": attempted_pages, + "ocr_successful_attempt_pages": successful_attempt_pages, + "ocr_schedule_complete": schedule_complete, + "ocr_run_complete": run_complete, + "ocr_worker_count": ocr_worker_count, + "ocr_rescue_min_remaining_seconds": float(ocr_rescue_min_remaining), + "repeated_margin_noise_removed_count": repeated_margin_noise_removed, + "native_hyphenation_repair_count": sum(int(page.get("native_hyphenation_repair_count") or 0) for page in pages), + "warnings": warnings, + "poppler_available": bool(poppler_text), + "elapsed_seconds": elapsed, + } + + +def _read_non_pdf_pages(path: Path, max_chars: int) -> tuple[list[dict[str, Any]], str]: + suffix = path.suffix.lower() + if suffix == ".docx": loader = "python-docx" try: import docx document = docx.Document(str(path)) text = "\n".join(paragraph.text for paragraph in document.paragraphs)[:max_chars] - pages.append({"page": 1, "text": text}) except Exception as exc: raise RuntimeError(f"DOCX extraction failed: {exc}") from exc else: - text = path.read_text(encoding="utf-8", errors="ignore")[:max_chars] - pages.append({"page": 1, "text": text}) - return pages, loader - - -def _chunk_page(text: str, target_chars: int = 2200) -> list[str]: - paragraphs = [_clean(item) for item in re.split(r"\n\s*\n", text) if _clean(item)] - if not paragraphs: - paragraphs = [_clean(item) for item in text.splitlines() if _clean(item)] - chunks: list[str] = [] - current: list[str] = [] - size = 0 + loader = "plain" + text = path.read_text(encoding="utf-8", errors="replace")[:max_chars] + clean = _clean(text) + paragraphs = [_clean(value) for value in re.split(r"\n\s*\n", clean) if _clean(value)] + chunks = [] for paragraph in paragraphs: - if current and size + len(paragraph) > target_chars: - chunks.append("\n".join(current)) - current = [] - size = 0 - current.append(paragraph) - size += len(paragraph) + 1 - if current: - chunks.append("\n".join(current)) - return chunks - - -def _heading_candidates(text: str) -> list[str]: - headings: list[str] = [] - for raw in text.splitlines(): - line = _clean(raw) - if not 3 <= len(line) <= 100: - continue - if re.match(r"^(?:\d+(?:\.\d+)*[.)]?\s+|[IVX]+[.)]\s+|第[一二三四五六七八九十]+[章节])", line): - value = re.sub(r"^(?:\d+(?:\.\d+)*[.)]?\s+|[IVX]+[.)]\s+)", "", line).strip() - if value and value not in headings: - headings.append(value) - return headings[:30] - - -def _document_index(pages: list[dict[str, Any]], headings: list[str]) -> dict: - sections = [] - formulas = [] - tables = [] + lines = paragraph.splitlines() + if len(lines) > 1 and re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+|[ivx]+[.)]\s+)", lines[0], re.IGNORECASE): + chunks.extend([_clean(lines[0]), _clean("\n".join(lines[1:]))]) + else: + chunks.append(paragraph) + blocks = [] + for index, chunk in enumerate(chunks or [clean], 1): + blocks.append({"id": f"P001_B{index:03d}", "bbox": None, "text": chunk, "kind": _block_kind(chunk), "source": loader, "confidence": 1.0, "reading_order": index}) + return [{"page": 1, "width": None, "height": None, "blocks": blocks, "text": clean, "char_count": len(clean), "replacement_character_rate": _replacement_rate(clean), "mojibake_rate": _mojibake_rate(clean), "poppler_agreement": None, "reading_order_confidence": 1.0, "used_ocr": False}], loader + + +def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: + headings = [] + seen = set() + body_started = False + body_anchor: tuple[int, float] | None = None + for page in pages: + font_sizes = sorted(float(block.get("font_size")) for block in page.get("blocks", []) if isinstance(block.get("font_size"), (int, float)) and len(_normalized_compare_text(str(block.get("text") or ""))) >= 20) + middle = len(font_sizes) // 2 + body_font_size = (font_sizes[middle] if len(font_sizes) % 2 else (font_sizes[middle - 1] + font_sizes[middle]) / 2.0) if font_sizes else None + for block in page.get("blocks", []): + lines = [_clean(value) for value in str(block.get("text", "")).splitlines() if _clean(value)] + for line_index, line in enumerate(lines): + if not 2 <= len(line) <= 140: + continue + explicit = re.match(r"^(abstract|references|acknowledg(?:e)?ments?|appendix)$", line, re.IGNORECASE) + numbered = re.match(r"^\s*((?:\d+(?:\.\d+)*)|(?:[ivx]+))[.)]?\s+(.{2,120})$", line, re.IGNORECASE) + compact_line = _normalized_compare_text(line) + block_bold = bool(block.get("font_bold")) + font_size = float(block.get("font_size") or 0.0) + first_line_alpha = next((char for char in line if char.isalpha()), "") + heading_case = not first_line_alpha or not first_line_alpha.isascii() or first_line_alpha.isupper() + known = len(line) <= 60 and not line.endswith((".", ",", ";")) and any( + compact_alias and heading_case and (compact_line == compact_alias or (block_bold and compact_line.startswith(compact_alias))) + for aliases in SECTION_ALIASES.values() + for alias in aliases + for compact_alias in [_normalized_compare_text(alias)] + ) + block_y = float((block.get("bbox") or [0, 0, 0, 0])[1]) + after_anchor = body_anchor is not None and (int(page["page"]) > body_anchor[0] or block_y >= body_anchor[1]) + position_ok = after_anchor if body_anchor is not None else body_started + size_ok = body_font_size is None or font_size >= body_font_size * 0.95 + typographic = position_ok and block_bold and size_ok and _looks_like_typographic_heading(line) + if numbered: + title_candidate = numbered.group(2).strip() + first_alpha = next((char for char in title_candidate if char.isalpha()), "") + numbered_style = block.get("source") != "pymupdf" or body_font_size is None or block_bold or font_size >= body_font_size * 1.05 + if not numbered_style: + first_alpha = "" + if not title_candidate or not title_candidate[0].isalpha(): + first_alpha = "" + has_case = bool(first_alpha and first_alpha.lower() != first_alpha.upper()) + if not first_alpha or (has_case and not first_alpha.isupper()) or re.search(r"[=±∑∫]", title_candidate) or title_candidate.endswith(".") or len(title_candidate) > 80 or len(title_candidate.split()) > 10: + numbered = None + if not (explicit or numbered or known or typographic): + continue + normalized = numbered.group(2).strip() if numbered else line.strip().rstrip(".:") + key = (normalized.casefold(), int(page.get("page") or 0), str(block.get("id") or ""), line_index) + if normalized and key not in seen: + seen.add(key) + headings.append({ + "id": f"section_{len(headings) + 1:03d}", + "title": normalized, + "page": page["page"], + "block_id": block["id"], + "line_index": line_index, + "_candidate_type": "explicit" if explicit else "numbered" if numbered else "known" if known else "typographic", + "_bbox": block.get("bbox"), + "_font_size": font_size, + "_reading_order": int(block.get("reading_order") or 0), + }) + if body_anchor is None and (explicit or numbered or known): + body_anchor = (int(page["page"]), block_y) + body_started = body_started or bool(explicit or numbered or known) + merged: list[dict[str, Any]] = [] + for heading in headings: + previous = merged[-1] if merged else None + previous_bbox = previous.get("_bbox") if previous else None + current_bbox = heading.get("_bbox") + same_multiline_heading = bool( + previous + and previous.get("_candidate_type") == "typographic" + and heading.get("_candidate_type") == "typographic" + and int(previous.get("page") or 0) == int(heading.get("page") or 0) + and int(heading.get("_reading_order") or 0) == int(previous.get("_reading_order") or 0) + 1 + and previous_bbox + and current_bbox + and float(current_bbox[1]) - float(previous_bbox[3]) <= max(float(previous.get("_font_size") or 0), float(heading.get("_font_size") or 0), 1.0) * 0.8 + and len(f"{previous.get('title', '')} {heading.get('title', '')}") <= 140 + ) + if same_multiline_heading: + previous["title"] = _clean(f"{previous.get('title', '')} {heading.get('title', '')}").rstrip(".:") + previous.setdefault("continuation_block_ids", []).append(str(heading.get("block_id") or "")) + previous["_bbox"] = [ + min(float(previous_bbox[0]), float(current_bbox[0])), + min(float(previous_bbox[1]), float(current_bbox[1])), + max(float(previous_bbox[2]), float(current_bbox[2])), + max(float(previous_bbox[3]), float(current_bbox[3])), + ] + previous["_reading_order"] = int(heading.get("_reading_order") or 0) + continue + merged.append(heading) + for index, heading in enumerate(merged, 1): + heading["id"] = f"section_{index:03d}" + return [{key: value for key, value in heading.items() if not key.startswith("_")} for heading in merged[:80]] + + +def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) -> dict[str, Any]: figures = [] - seen_sections = set() - formula_pattern = re.compile(r"(?:[A-Za-zΑ-Ωα-ω][A-Za-z0-9_{}^\-]*\s*=\s*[^=\n]{3,}|[∑∫∂∇≈≤≥]\s*[^\n]{3,}|\([^\n]{2,80}\)\s*\(\d+\)\s*$)") - table_pattern = re.compile(r"^(?:Table|TABLE|表)\s*[A-Za-z0-9一二三四五六七八九十.：: -]+", re.IGNORECASE) - figure_pattern = re.compile(r"^(?:Figure|FIGURE|Fig\.|图)\s*[A-Za-z0-9一二三四五六七八九十.：: -]+", re.IGNORECASE) + tables = [] + formulas = [] for page in pages: - for raw in page.get("text", "").splitlines(): - line = _clean(raw) - if not line: - continue - for heading in headings: - if heading.lower() in line.lower() and heading not in seen_sections: - sections.append({"id": f"section_{len(sections) + 1:03d}", "title": heading, "page": page["page"]}) - seen_sections.add(heading) - break - if len(formulas) < 120 and 4 <= len(line) <= 240 and formula_pattern.search(line): - formulas.append({"id": f"formula_{len(formulas) + 1:03d}", "page": page["page"], "text": line}) - if len(tables) < 80 and table_pattern.match(line): - tables.append({"id": f"table_{len(tables) + 1:03d}", "page": page["page"], "caption": line}) - if len(figures) < 80 and figure_pattern.match(line): - figures.append({"id": f"figure_{len(figures) + 1:03d}", "page": page["page"], "caption": line}) + page_blocks = page.get("blocks", []) + for block_index, block in enumerate(page_blocks): + record = {"page": page["page"], "block_id": block["id"], "bbox": block.get("bbox"), "caption": block.get("text", "")} + if block.get("kind") == "caption": + parent = block.get("parent_block") + continuation_blocks = [] + for following in page_blocks[block_index + 1:]: + if parent is None or following.get("parent_block") != parent or _caption_prefix_kind(str(following.get("text") or "")): + break + continuation_blocks.append(following) + + # A caption may continue in the next column at the same vertical + # position while PyMuPDF assigns a different parent block. Recover + # those lines by geometry and font, then keep column-major order. + start_bbox = block.get("bbox") if isinstance(block.get("bbox"), list) else [0, 0, 0, 0] + start_y = float(start_bbox[1] or 0) + start_font = float(block.get("font_size") or 0) + page_height = max(1.0, float(page.get("height") or 1)) + max_y = start_y + max(42.0, page_height * 0.09) + included_ids = {str(block.get("id") or ""), *(str(item.get("id") or "") for item in continuation_blocks)} + for candidate in page_blocks: + candidate_id = str(candidate.get("id") or "") + if not candidate_id or candidate_id in included_ids: + continue + if int(candidate.get("column") if candidate.get("column") is not None else -99) == int(block.get("column") if block.get("column") is not None else -98): + continue + bbox = candidate.get("bbox") if isinstance(candidate.get("bbox"), list) else [0, 0, 0, 0] + candidate_y = float(bbox[1] or 0) + candidate_font = float(candidate.get("font_size") or 0) + if not (start_y - 2.0 <= candidate_y <= max_y): + continue + if start_font and candidate_font and abs(candidate_font - start_font) > 0.8: + continue + if _caption_prefix_kind(str(candidate.get("text") or "")): + continue + continuation_blocks.append(candidate) + included_ids.add(candidate_id) + if continuation_blocks: + continuation_blocks.sort(key=lambda item: (int(item.get("column") or 0), float((item.get("bbox") or [0, 0, 0, 0])[1]), float((item.get("bbox") or [0, 0, 0, 0])[0]))) + record["caption"] = _clean(record["caption"] + " " + " ".join(str(item.get("text") or "") for item in continuation_blocks)) + figures.append({"id": f"figure_{len(figures) + 1:03d}", **record}) + elif block.get("kind") == "table": + tables.append({"id": f"table_{len(tables) + 1:03d}", **record}) + elif block.get("kind") == "formula": + formulas.append({"id": f"formula_{len(formulas) + 1:03d}", "page": page["page"], "block_id": block["id"], "bbox": block.get("bbox"), "text": block.get("text", "")}) + page_map = {int(page["page"]): page for page in pages} + for table in tables: + page = page_map.get(int(table.get("page") or 0)) + if not page: + continue + blocks = page.get("blocks", []) + caption_index = next((index for index, block in enumerate(blocks) if str(block.get("id") or "") == str(table.get("block_id") or "")), None) + if caption_index is None: + continue + caption_bbox = table.get("bbox") or [0, 0, 0, 0] + candidates = [] + for block in blocks[caption_index + 1:]: + bbox = block.get("bbox") + text = _clean(block.get("text", "")) + if not bbox or not text: + continue + if str(block.get("kind") or "") in {"heading", "caption", "table", "formula"}: + break + if float(bbox[1]) > float(caption_bbox[3]) + min(300.0, float(page.get("height") or 800.0) * 0.35): + break + if len(text) > 60: + break + candidates.append(block) + if len(candidates) < 4: + continue + heights = sorted(max(1.0, float(block["bbox"][3]) - float(block["bbox"][1])) for block in candidates) + tolerance = max(5.0, heights[len(heights) // 2] * 0.75) + row_groups: list[list[dict[str, Any]]] = [] + for block in sorted(candidates, key=lambda item: ((float(item["bbox"][1]) + float(item["bbox"][3])) / 2.0, float(item["bbox"][0]))): + center = (float(block["bbox"][1]) + float(block["bbox"][3])) / 2.0 + if not row_groups: + row_groups.append([block]) + continue + previous_center = sum((float(item["bbox"][1]) + float(item["bbox"][3])) / 2.0 for item in row_groups[-1]) / len(row_groups[-1]) + if abs(center - previous_center) <= tolerance: + row_groups[-1].append(block) + else: + row_groups.append([block]) + if len(row_groups) < 2 or len(row_groups[0]) < 2: + continue + header = sorted(row_groups[0], key=lambda item: float(item["bbox"][0])) + anchors = [(float(item["bbox"][0]) + float(item["bbox"][2])) / 2.0 for item in header] + columns = [_clean(item.get("text", "")) for item in header] + rows = [] + cell_records = [] + for row_index, group in enumerate(row_groups): + values: list[str | None] = [None] * len(columns) + block_ids: list[str | None] = [None] * len(columns) + for block in sorted(group, key=lambda item: float(item["bbox"][0])): + center = (float(block["bbox"][0]) + float(block["bbox"][2])) / 2.0 + column_index = min(range(len(anchors)), key=lambda index: abs(center - anchors[index])) + value = _clean(block.get("text", "")) + values[column_index] = f"{values[column_index]} {value}".strip() if values[column_index] else value + block_ids[column_index] = str(block.get("id") or "") + cell_records.append({"row": row_index, "column": column_index, "text": value, "block_id": block.get("id"), "bbox": block.get("bbox")}) + if row_index: + rows.append({"values": values, "block_ids": block_ids}) + header_signal = any( + _cjk_character_count(value) > 0 + or (value and value[0].isalpha() and value[0].isupper()) + or value.isupper() + or bool(re.search(r"(?:[A-Za-z].*\d|\d.*[A-Za-z])", value)) + for value in columns + ) + if len(rows) < 2 or not header_signal: + continue + cell_block_ids = {str(record.get("block_id") or "") for record in cell_records} + for block in candidates: + if str(block.get("id") or "") in cell_block_ids: + block["kind"] = "table_cell" + all_boxes = [caption_bbox] + [block.get("bbox") for block in candidates if block.get("bbox")] + table.update({ + "columns": columns, + "rows": rows, + "cells": cell_records, + "bbox": [ + min(float(box[0]) for box in all_boxes), + min(float(box[1]) for box in all_boxes), + max(float(box[2]) for box in all_boxes), + max(float(box[3]) for box in all_boxes), + ], + }) + return {"summary": "Page-aware section, formula, table-caption, and figure-caption index.", "sections": sections, "formulas": formulas[:120], "tables": tables[:80], "figures": figures[:80]} + + +def _section_coverage(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) -> dict[str, bool]: + candidates = ("\n".join(item["title"] for item in sections) + "\n" + "\n".join(page.get("text", "") for page in pages)).casefold() + compact = _normalized_compare_text(candidates) + coverage = { + name: any(re.search(rf"\b{re.escape(alias)}\b", candidates) or (_normalized_compare_text(alias) and _normalized_compare_text(alias) in compact) for alias in aliases) + for name, aliases in SECTION_ALIASES.items() + } + # Nature-style articles and several journals print a lead abstract without + # an explicit "Abstract" heading. Treat a substantial first-page research + # summary as abstract coverage when it contains an authorial contribution + # statement, instead of emitting a false extraction warning. + first_page = str(pages[0].get("text") or "") if pages else "" + first_page_compact = _normalized_compare_text(first_page) + implicit_abstract = bool( + len(first_page_compact) >= 700 + and re.search( + r"\b(?:here|in this (?:work|paper|study))\s+(?:we\s+)?(?:present|introduce|propose|develop|show|demonstrate)\b" + r"|\bwe\s+(?:present|introduce|propose|develop|show|demonstrate)\b", + first_page, + re.IGNORECASE, + ) + ) + coverage["abstract"] = bool(coverage.get("abstract") or implicit_abstract) + return coverage + + +def _ocr_latin_lexical_quality(pages: list[dict[str, Any]]) -> dict[str, Any]: + text = " ".join(str(page.get("text") or "") for page in pages if page.get("used_ocr")) + anchors = re.findall(r"\b(?:abstract|introduction|method|methods|experiments|results|conclusion|discussion)\b", text, re.IGNORECASE) + tokens = [token for token in re.findall(r"[A-Za-z]{3,}", text) if not token.isupper()] + if len(tokens) < 20 or len(anchors) < 2: + return {"applied": False, "token_count": len(tokens), "known_word_ratio": None, "english_anchor_count": len(anchors)} + try: + import wordninja + + known_words = wordninja.DEFAULT_LANGUAGE_MODEL._wordcost + except Exception: + return {"applied": False, "token_count": len(tokens), "known_word_ratio": None, "english_anchor_count": len(anchors)} + known = sum(token.casefold() in known_words or token.casefold() in OCR_PROTECTED_WORDS for token in tokens) return { - "summary": "Page-aware section, formula, table-caption, and figure-caption index.", - "sections": sections, - "formulas": formulas, - "tables": tables, - "figures": figures, + "applied": True, + "token_count": len(tokens), + "known_word_ratio": round(known / max(1, len(tokens)), 4), + "english_anchor_count": len(anchors), } -def parse_paper(path: str | Path, max_chars: int = 90000) -> dict: - source = Path(path).resolve() - if not source.exists(): - raise FileNotFoundError(f"Paper does not exist: {source}") - pages, loader = _read_pages(source, max_chars=max_chars) +def _evidence_id(page: int, text: str, bbox: Any, kind: str) -> str: + payload = json.dumps({"page": page, "text": _clean(text), "bbox": bbox, "kind": kind}, ensure_ascii=False, sort_keys=True).encode("utf-8") + return f"E_P{int(page):03d}_{hashlib.sha1(payload).hexdigest()[:12]}" + + +def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], document_index: dict[str, Any], max_chars: int) -> list[dict[str, Any]]: + section_positions = sorted(sections, key=lambda item: (item["page"], item.get("block_id", ""))) evidence: list[dict[str, Any]] = [] + used = 0 + for figure in document_index.get("figures", []): + text = _clean(figure.get("caption", "")) + if not text or used + len(text) > max_chars: + continue + evidence.append({ + "id": _evidence_id(int(figure.get("page") or 0), text, figure.get("bbox"), "caption"), + "page": figure.get("page"), + "block_id": figure.get("block_id"), + "bbox": figure.get("bbox"), + "section_hint": "Figure Captions", + "kind": "caption", + "source": "pymupdf", + "confidence": 1.0, + "text": text, + "char_count": len(text), + }) + used += len(text) + + for table in document_index.get("tables", []): + lines = [_clean(table.get("caption", ""))] + columns = [str(value or "") for value in table.get("columns", [])] + if columns: + lines.append("Columns: " + " | ".join(columns)) + for row in table.get("rows", []): + values = [str(value or "") for value in row.get("values", [])] + lines.append("Row: " + " | ".join(values)) + text = _clean("\n".join(value for value in lines if value)) + if not text or used + len(text) > max_chars: + continue + evidence.append({ + "id": _evidence_id(int(table.get("page") or 0), text, table.get("bbox"), "table"), + "page": table.get("page"), + "block_id": table.get("block_id"), + "bbox": table.get("bbox"), + "section_hint": "Table Captions", + "kind": "table", + "source": "structured-table", + "confidence": 1.0, + "text": text, + "char_count": len(text), + }) + used += len(text) + + block_candidates: list[dict[str, Any]] = [] for page in pages: - for chunk in _chunk_page(page["text"]): - if not chunk: + for block in page.get("blocks", []): + text = _clean(block.get("text", "")) + if not text or block.get("kind") in {"caption", "table", "table_cell"}: continue - evidence_id = f"E{len(evidence) + 1:04d}" - evidence.append({ - "id": evidence_id, + block_id = str(block.get("id") or "") + section = next( + ( + item["title"] + for item in reversed(section_positions) + if (int(item["page"]), str(item.get("block_id") or "")) <= (int(page["page"]), block_id) + ), + None, + ) + block_candidates.append({ + "id": _evidence_id(int(page["page"]), text, block.get("bbox"), str(block.get("kind") or "paragraph")), "page": page["page"], - "section_hint": None, - "text": chunk, - "char_count": len(chunk), + "block_id": block_id, + "bbox": block.get("bbox"), + "section_hint": section, + "kind": block.get("kind"), + "source": block.get("source"), + "confidence": block.get("confidence", 1.0), + "text": text, + "char_count": len(text), }) - all_text = "\n".join(page["text"] for page in pages) - headings = _heading_candidates(all_text) - document_index = _document_index(pages, headings) - for item in evidence: - low = item["text"].lower() - item["section_hint"] = next((heading for heading in headings if heading.lower() in low), None) + selected_ids: set[str] = set() + page_groups: dict[int, list[dict[str, Any]]] = {} + for item in block_candidates: + page_groups.setdefault(int(item["page"]), []).append(item) + + def add(item: dict[str, Any], max_length: int | None = None) -> bool: + nonlocal used + item_id = str(item["id"]) + if item_id in selected_ids or used >= max_chars: + return False + remaining = max_chars - used + if remaining <= 0: + return False + value = dict(item) + if max_length is not None and len(value["text"]) > max_length: + value["text"] = value["text"][:max_length].rstrip() + value["char_count"] = len(value["text"]) + value["id"] = _evidence_id(int(value["page"]), value["text"], value.get("bbox"), str(value.get("kind") or "paragraph")) + if len(value["text"]) > remaining: + if remaining < 40: + return False + value["text"] = value["text"][:remaining].rstrip() + value["char_count"] = len(value["text"]) + value["id"] = _evidence_id(int(value["page"]), value["text"], value.get("bbox"), str(value.get("kind") or "paragraph")) + item_id = str(value["id"]) + evidence.append(value) + selected_ids.add(str(item["id"])) + selected_ids.add(item_id) + used += len(value["text"]) + return True + + # Reserve a representative block for every page before spending the budget + # on long early sections. This keeps conclusions and appendices discoverable. + representative_quota = max(80, min(600, max(1, max_chars - used) // max(1, len(page_groups)))) + for page_number in sorted(page_groups): + candidates = page_groups[page_number] + representative = max( + candidates, + key=lambda item: ( + 1 if item.get("kind") in {"paragraph", "title"} else 0, + min(len(str(item.get("text") or "")), 600), + ), + ) + add(representative, max_length=representative_quota) - if not evidence: - raise ValueError("No readable paper text was extracted") + section_priority = { + "abstract": 0, + "conclusion": 1, + "conclusions": 1, + "discussion": 1, + "method": 2, + "methods": 2, + "methodology": 2, + "approach": 2, + "architecture": 2, + "framework": 2, + "system overview": 2, + "introduction": 3, + "background": 3, + "experiment": 4, + "experiments": 4, + "evaluation": 4, + "results": 4, + } + + def priority(item: dict[str, Any]) -> tuple[int, int, str]: + section = str(item.get("section_hint") or "").casefold() + text = str(item.get("text") or "") + topology_definition = bool(re.search( + r"\b(?:has|have|consists? of|comprises?|contains?)\b.{0,80}\b(?:stages?|components?|modules?|steps?)\b" + r"|\b(?:assisted[ -]manual|semi[ -]automatic|fully automatic)\b" + r"|\b(?:input|image|text|query|prompt)\b.{0,100}\b(?:encoder|decoder|retriever|generator)\b" + r"|\b(?:encoder|decoder|retriever|generator)\b.{0,100}\b(?:output|prediction|representation|embedding)\b", + text, + re.IGNORECASE, + )) + rank = min((value for name, value in section_priority.items() if name in section), default=5) + if topology_definition: + rank = -1 + if item.get("kind") == "heading": + rank = min(rank, 2) + return rank, int(item.get("page") or 0), str(item.get("block_id") or "") + + for item in sorted(block_candidates, key=priority): + add(item) + + document_order = { + str(item["id"]): index + for index, item in enumerate(block_candidates) + } + caption_count = sum(1 for item in evidence if item.get("kind") == "caption") + caption_items = evidence[:caption_count] + body_items = sorted(evidence[caption_count:], key=lambda item: (int(item.get("page") or 0), document_order.get(str(item.get("id")), 10**9), str(item.get("block_id") or ""))) + evidence = caption_items + body_items + for index, item in enumerate(evidence, 1): + item["legacy_id"] = f"E{index:04d}" + return evidence + + +def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[str, Any], details: dict[str, Any]) -> dict[str, Any]: + readable = [page for page in pages if len(_normalized_compare_text(page.get("text", ""))) >= 80] + readable_ratio = round(len(readable) / max(1, len(pages)), 4) + ocr_count = sum(1 for page in pages if page.get("used_ocr")) + candidate_count = len(details.get("ocr_candidate_pages", [])) + fully_scanned_source = bool(pages and candidate_count == len(pages)) + partial_scan = bool(fully_scanned_source and 0 < ocr_count < len(pages)) + pdf_type = "scanned" if fully_scanned_source else "mixed" if ocr_count and ocr_count < len(pages) else "scanned" if ocr_count else "born_digital" + coverage = _section_coverage(pages, index.get("sections", [])) + bold_block_ids = {(int(page["page"]), str(block.get("id") or "")) for page in pages for block in page.get("blocks", []) if block.get("font_bold")} + warnings = list(details.get("warnings", [])) + ocr_blocks = [block for page in pages if page.get("used_ocr") for block in page.get("blocks", []) if isinstance(block, dict)] + ocr_confidences = [float(block.get("confidence") or 0.0) for block in ocr_blocks] + mean_ocr_confidence = round(sum(ocr_confidences) / max(1, len(ocr_confidences)), 4) if ocr_blocks else None + short_ocr_blocks = [block for block in ocr_blocks if len(_normalized_compare_text(str(block.get("text") or ""))) < 3] + short_ocr_block_ratio = round(len(short_ocr_blocks) / max(1, len(ocr_blocks)), 4) if ocr_blocks else None + lexical_quality = _ocr_latin_lexical_quality(pages) + sampled_scan_ready = bool( + partial_scan + and len(readable) >= min(3, len(pages)) + and coverage.get("abstract") + and coverage.get("method") + and mean_ocr_confidence is not None + and mean_ocr_confidence >= 0.75 + ) + if readable_ratio < 0.6 and not sampled_scan_ready: + warnings.append("fewer than 60% of pages contain reliable text") + if sampled_scan_ready: + warnings.append("fully scanned document was only partially OCRed; semantic scope is limited to sampled pages") + if mean_ocr_confidence is not None and mean_ocr_confidence < 0.5: + warnings.append("mean OCR confidence is below 0.50") + if short_ocr_block_ratio is not None and short_ocr_block_ratio > 0.3: + warnings.append("OCR output is highly fragmented") + if lexical_quality.get("applied") and float(lexical_quality.get("known_word_ratio") or 0.0) < 0.72: + warnings.append("OCR English lexical plausibility is below 0.72") + if not coverage.get("abstract") or not coverage.get("method"): + warnings.append("abstract or method-like section was not confidently located") + agreements = [page["poppler_agreement"] for page in pages if isinstance(page.get("poppler_agreement"), (int, float))] + lexical_failure = bool(lexical_quality.get("applied") and float(lexical_quality.get("known_word_ratio") or 0.0) < 0.5) + report_status = "fail" if (readable_ratio < 0.6 and not sampled_scan_ready) or (mean_ocr_confidence is not None and mean_ocr_confidence < 0.35) or lexical_failure else "warning" if warnings else "pass" + return { + "summary": "PDF extraction quality and fallback report.", + "status": report_status, + "source_path": str(source), + "pdf_type": pdf_type, + "page_count": len(pages), + "readable_page_count": len(readable), + "readable_page_ratio": readable_ratio, + "semantic_scope": "sampled_pages_only" if partial_scan else "full_document", + "scientific_scope_complete": not partial_scan, + "sampled_scan_ready": sampled_scan_ready, + "empty_or_low_text_pages": [page["page"] for page in pages if page not in readable], + "ocr_candidate_pages": details.get("ocr_candidate_pages", []), + "ocr_priority_pages": details.get("ocr_priority_pages", []), + "ocr_priority": details.get("ocr_priority", []), + "ocr_pages": details.get("ocr_pages", []), + "ocr_page_durations": details.get("ocr_page_durations", []), + "ocr_attempted_pages": details.get("ocr_attempted_pages", []), + "ocr_successful_attempt_pages": details.get("ocr_successful_attempt_pages", []), + "ocr_schedule_complete": bool(details.get("ocr_schedule_complete", True)), + "ocr_run_complete": bool(details.get("ocr_run_complete", True)), + "ocr_worker_count": int(details.get("ocr_worker_count") or 1), + "ocr_rescue_min_remaining_seconds": float(details.get("ocr_rescue_min_remaining_seconds") or 45.0), + "repeated_margin_noise_removed_count": int(details.get("repeated_margin_noise_removed_count") or 0), + "native_hyphenation_repair_count": int(details.get("native_hyphenation_repair_count") or 0), + "ocr_margin_noise_removed_count": sum(int(item.get("margin_noise_removed") or 0) for item in details.get("ocr_page_durations", [])), + "ocr_spacing_repair_count": sum(int(item.get("spacing_repairs") or 0) for item in details.get("ocr_page_durations", [])), + "mean_ocr_confidence": mean_ocr_confidence, + "short_ocr_block_ratio": short_ocr_block_ratio, + "ocr_latin_lexical_gate_applied": bool(lexical_quality.get("applied")), + "ocr_latin_token_count": int(lexical_quality.get("token_count") or 0), + "ocr_latin_known_word_ratio": lexical_quality.get("known_word_ratio"), + "ocr_english_anchor_count": int(lexical_quality.get("english_anchor_count") or 0), + "replacement_character_rate": round(sum(page.get("replacement_character_rate", 0.0) for page in pages) / max(1, len(pages)), 6), + "mojibake_rate": round(sum(page.get("mojibake_rate", 0.0) for page in pages) / max(1, len(pages)), 6), + "cross_extractor_agreement": round(sum(agreements) / len(agreements), 4) if agreements else None, + "reading_order_confidence": round(sum(page.get("reading_order_confidence", 0.0) for page in pages) / max(1, len(pages)), 4), + "max_column_count": max((int(page.get("column_count") or 1) for page in pages), default=1), + "multi_column_page_count": sum(int(page.get("column_count") or 1) > 1 for page in pages), + "rotated_pages": [int(page["page"]) for page in pages if int(page.get("rotation") or 0) % 360 != 0], + "section_coverage": coverage, + "figure_caption_count": len(index.get("figures", [])), + "table_caption_count": len(index.get("tables", [])), + "formula_count": len(index.get("formulas", [])), + "section_count": len(index.get("sections", [])), + "typographic_heading_count": sum((int(item.get("page") or 0), str(item.get("block_id") or "")) in bold_block_ids for item in index.get("sections", [])), + "merged_heading_line_count": sum(len(item.get("continuation_block_ids", [])) for item in index.get("sections", [])), + "poppler_available": details.get("poppler_available", False), + "warnings": warnings, + "elapsed_seconds": details.get("elapsed_seconds", 0.0), + } + + +def parse_paper( + path: str | Path, + max_chars: int = 90000, + deadline_at: float | None = None, + ocr_engine: str = "off", + ocr_lang: str = "en_ch", + ocr_adapter: Callable | None = None, + max_ocr_pages: int = 6, + ocr_rescue_min_remaining: float = 45.0, +) -> dict[str, Any]: + source = Path(path).resolve() + if not source.exists(): + raise FileNotFoundError(f"Paper does not exist: {source}") + source_hash = hashlib.sha256(source.read_bytes()).hexdigest() + if source.suffix.lower() == ".pdf": + _validate_pdf_container(source) + pages, details = _read_pdf_pages(source, max_chars, deadline_at, ocr_engine, ocr_lang, ocr_adapter, max_ocr_pages, ocr_rescue_min_remaining) + loader = "structured-pdf" + else: + pages, loader = _read_non_pdf_pages(source, max_chars) + details = {"warnings": [], "ocr_candidate_pages": [], "ocr_pages": [], "poppler_available": False, "elapsed_seconds": 0.0} + sections = _heading_candidates(pages) + section_blocks = { + (int(item.get("page") or 0), block_id) + for item in sections + for block_id in [str(item.get("block_id") or ""), *(str(value) for value in item.get("continuation_block_ids", []))] + if block_id + } + for page in pages: + for block in page.get("blocks", []): + if (int(page["page"]), str(block.get("id") or "")) in section_blocks: + block["kind"] = "heading" + document_index = _document_index(pages, sections) + evidence = _build_evidence(pages, sections, document_index, max_chars) + all_text = "\n\n".join(page.get("text", "") for page in pages) + extraction_report = _extraction_report(source, pages, document_index, details) + evidence_pages = {int(item.get("page") or 0) for item in evidence if item.get("page")} + extraction_report["evidence_char_count"] = sum(int(item.get("char_count") or 0) for item in evidence) + extraction_report["evidence_page_count"] = len(evidence_pages) + extraction_report["evidence_page_coverage_ratio"] = round(len(evidence_pages) / max(1, len(pages)), 4) return { - "summary": "Paper parsed into page-aware evidence chunks.", + "summary": "Paper parsed into a structured, page-aware evidence model.", "source_path": str(source), "source_name": source.name, "source_type": source.suffix.lower(), + "source_sha256": source_hash, "loader": loader, "page_count": len(pages), "char_count": len(all_text), - "headings": headings, + "headings": [item["title"] for item in sections], + "sections": sections, + "pages": pages, "document_index": document_index, "evidence": evidence, + "extraction_report": extraction_report, } +def paper_markdown(parsed: dict[str, Any]) -> str: + parts = [] + for page in parsed.get("pages", []): + parts.append(f"\n\n{page.get('text', '')}") + return "\n\n".join(parts).strip() + "\n" + + +def _overview_figure_evidence_priority(parsed: dict[str, Any], limit: int = 2) -> dict[str, tuple[int, int]]: + """Prioritize labels spatially inside likely overview figures. + + Figure-local labels are often short text blocks with no section heading, so + a section-first evidence sort can push them past the fast planner's character + budget. Recover the visual region around the strongest overview captions + and rank those blocks immediately after their caption. + """ + positive = re.compile(r"\b(overview|framework|architecture|pipeline|procedure|approach|method|model|system|workflow)\b", re.IGNORECASE) + negative = re.compile(r"\b(comparison|performance|result|ablation|visualization|qualitative|attention map)\b", re.IGNORECASE) + ranked: list[tuple[int, int, dict[str, Any]]] = [] + for index, figure in enumerate(parsed.get("document_index", {}).get("figures", [])): + if not isinstance(figure, dict): + continue + caption = str(figure.get("caption") or "").strip() + if not caption or not positive.search(caption): + continue + score = 8 + if re.match(r"^(?:figure|fig\.)\s*[12]\b", caption, re.IGNORECASE): + score += 5 + if 160 <= len(caption) <= 1600: + score += 4 + if negative.search(caption): + score -= 7 + ranked.append((score, -index, figure)) + candidates = [item for _, _, item in sorted(ranked, reverse=True)[: max(1, int(limit))]] + if not candidates: + return {} + + pages = { + int(page.get("page") or 0): page + for page in parsed.get("pages", []) + if isinstance(page, dict) and page.get("page") + } + evidence = [item for item in parsed.get("evidence", []) if isinstance(item, dict)] + priority: dict[str, tuple[int, int]] = {} + for candidate_index, figure in enumerate(candidates): + page_number = int(figure.get("page") or 0) + page = pages.get(page_number, {}) + page_height = float(page.get("height") or 0.0) + caption = str(figure.get("caption") or "") + caption_record = next( + ( + item + for item in evidence + if int(item.get("page") or 0) == page_number + and str(item.get("kind") or "").casefold() == "caption" + and str(item.get("text") or "").startswith(caption[:60]) + ), + None, + ) + if not caption_record: + continue + caption_id = str(caption_record.get("id") or "") + if caption_id: + priority[caption_id] = (candidate_index * 2, -1) + caption_bbox = caption_record.get("bbox") + if not (isinstance(caption_bbox, list) and len(caption_bbox) == 4 and page_height > 0): + continue + caption_top = float(caption_bbox[1]) + caption_bottom = float(caption_bbox[3]) + caption_center_ratio = ((caption_top + caption_bottom) / 2.0) / page_height + for item in evidence: + if int(item.get("page") or 0) != page_number: + continue + item_id = str(item.get("id") or "") + item_bbox = item.get("bbox") + if not item_id or not (isinstance(item_bbox, list) and len(item_bbox) == 4): + continue + item_top = float(item_bbox[1]) + item_bottom = float(item_bbox[3]) + if caption_center_ratio >= 0.38: + in_visual_region = item_bottom <= caption_bottom + 6.0 and item_top >= max(0.0, caption_top - 0.62 * page_height) + elif caption_center_ratio <= 0.25: + in_visual_region = item_top >= caption_top - 6.0 and item_bottom <= min(page_height, caption_bottom + 0.62 * page_height) + else: + in_visual_region = item_bottom >= caption_top - 0.42 * page_height and item_top <= caption_bottom + 0.42 * page_height + if not in_visual_region: + continue + block_id = str(item.get("block_id") or "") + reading_order = next( + ( + int(block.get("reading_order") or 0) + for block in page.get("blocks", []) + if isinstance(block, dict) and str(block.get("id") or "") == block_id + ), + 0, + ) + value = (candidate_index * 2 + 1, reading_order) + if item_id not in priority or value < priority[item_id]: + priority[item_id] = value + return priority + + +def overview_figure_spatial_excerpt(parsed: dict[str, Any], max_items: int = 180) -> str: + """Return compact bbox-aware evidence for likely overview figure regions.""" + priority = _overview_figure_evidence_priority(parsed) + if not priority: + return "[]" + pages = { + int(page.get("page") or 0): page + for page in parsed.get("pages", []) + if isinstance(page, dict) and page.get("page") + } + records = [] + for item in parsed.get("evidence", []): + if not isinstance(item, dict): + continue + item_id = str(item.get("id") or "") + if item_id not in priority: + continue + page_number = int(item.get("page") or 0) + page = pages.get(page_number, {}) + width = max(1.0, float(page.get("width") or 1.0)) + height = max(1.0, float(page.get("height") or 1.0)) + bbox = item.get("bbox") if isinstance(item.get("bbox"), list) and len(item.get("bbox")) == 4 else None + bbox_percent = None + if bbox: + bbox_percent = [ + round(float(bbox[0]) / width, 4), + round(float(bbox[1]) / height, 4), + round(float(bbox[2]) / width, 4), + round(float(bbox[3]) / height, 4), + ] + records.append({ + "id": item_id, + "page": page_number, + "block_id": str(item.get("block_id") or ""), + "kind": str(item.get("kind") or "paragraph"), + "reading_order": priority[item_id][1], + "bbox_percent": bbox_percent, + "text": str(item.get("text") or ""), + "_priority": priority[item_id], + }) + records.sort(key=lambda item: (item["_priority"], item["page"], item["block_id"])) + for item in records: + item.pop("_priority", None) + return json.dumps(records[: max(1, int(max_items))], ensure_ascii=False, separators=(",", ":")) + + +def overview_figure_panel_excerpt(parsed: dict[str, Any], max_items_per_panel: int = 48) -> str: + """Group concise target-figure labels by explicit panel markers.""" + def role_hint(text_value: str) -> str | None: + low = text_value.casefold().strip(" .:") + words = text_value.split() + if len(text_value) > 72 or len(words) > 8 or low.startswith(("we ", "with ", "the vector ", "each node ")): + return None + if re.search(r"\b(loss|objective|optimization)\b", low): + return "objective" + if re.search(r"\b(predictions?|predicted .*distribution|outputs?|results?)\b", low) or re.fullmatch(r"y\s*test", low): + return "output" + # Figure authors often use a determiner-led dataset phrase as a heading + # for a whole visual region (for example, "A synthetic dataset"), not + # as a separate tensor/table node in the data-flow graph. Keep explicit + # operational labels such as "Input dataset" and X_train/Y_test as + # inputs, while treating descriptive noun phrases as group containers. + if re.fullmatch(r"(?:a|an|the)\s+(?:[\w-]+\s+){1,6}datasets?", low): + return "group" + if re.search(r"\b(dataset|input data|input dataset|source data)\b", low) or re.fullmatch(r"[xy]\s*(?:train|test)", low): + return "input" + if re.search(r"\b(layer|block|stack)\b", low) and re.search(r"\b\d+\s*[×x]\b|\(\d+", low): + return "group" + if len(words) <= 5 and re.search(r"\b(model|network|encoder|decoder|attention|mlp|transformer|retriever|generator|classifier|backbone|module|tabpfn)\b", low): + return "module" + if re.fullmatch(r"[A-Z][A-Z0-9-]{2,}", text_value.strip()): + return "module" + return None + + priority = _overview_figure_evidence_priority(parsed, limit=1) + if not priority: + return "{}" + candidate_ids = {item_id for item_id, value in priority.items() if value[0] <= 1} + evidence = [ + item for item in parsed.get("evidence", []) + if isinstance(item, dict) and str(item.get("id") or "") in candidate_ids + ] + caption_record = next((item for item in evidence if str(item.get("kind") or "").casefold() == "caption"), None) + if not caption_record: + return "{}" + caption = str(caption_record.get("text") or "") + panel_matches = list(re.finditer(r"(?:^|[.;])\s*([a-h])\s*[,.:]\s*", caption, re.IGNORECASE)) + panel_labels = [match.group(1).casefold() for match in panel_matches] + if len(panel_labels) < 2: + return "{}" + descriptions: dict[str, str] = {} + for index, match in enumerate(panel_matches): + end = panel_matches[index + 1].start() if index + 1 < len(panel_matches) else len(caption) + descriptions[match.group(1).casefold()] = re.sub(r"\s+", " ", caption[match.end():end]).strip(" .") + + page_number = int(caption_record.get("page") or 0) + page = next((item for item in parsed.get("pages", []) if isinstance(item, dict) and int(item.get("page") or 0) == page_number), {}) + width = max(1.0, float(page.get("width") or 1.0)) + height = max(1.0, float(page.get("height") or 1.0)) + caption_bbox = caption_record.get("bbox") if isinstance(caption_record.get("bbox"), list) and len(caption_record.get("bbox")) == 4 else [0.0, height, width, height] + visual_records = [ + item for item in evidence + if item is not caption_record + and isinstance(item.get("bbox"), list) + and len(item.get("bbox")) == 4 + and float(item["bbox"][3]) <= float(caption_bbox[3]) + 6.0 + ] + markers: list[tuple[str, dict[str, Any]]] = [] + for panel_label in panel_labels: + candidates = [item for item in visual_records if str(item.get("text") or "").strip().casefold() == panel_label] + if candidates: + markers.append((panel_label, min(candidates, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))))) + if len(markers) < 2: + return "{}" + markers.sort(key=lambda item: (float(item[1]["bbox"][1]), float(item[1]["bbox"][0]))) + x_centers = [(float(item[1]["bbox"][0]) + float(item[1]["bbox"][2])) / 2.0 for item in markers] + y_centers = [(float(item[1]["bbox"][1]) + float(item[1]["bbox"][3])) / 2.0 for item in markers] + orientation = "vertical" if (max(y_centers) - min(y_centers)) >= (max(x_centers) - min(x_centers)) else "horizontal" + visual_top = min(float(item["bbox"][1]) for item in visual_records) + visual_left = min(float(item["bbox"][0]) for item in visual_records) + visual_right = max(float(item["bbox"][2]) for item in visual_records) + visual_bottom = min(float(caption_bbox[1]), max(float(item["bbox"][3]) for item in visual_records)) + marker_positions = y_centers if orientation == "vertical" else x_centers + panels = [] + for index, (panel_label, marker) in enumerate(markers): + if orientation == "vertical": + start = max(visual_top, float(marker["bbox"][1]) - 0.01 * height) + end = (float(markers[index + 1][1]["bbox"][1]) - 0.01 * height) if index + 1 < len(markers) else visual_bottom + panel_bbox = [visual_left, start, visual_right, end] + else: + start = max(visual_left, float(marker["bbox"][0]) - 0.01 * width) + end = (float(markers[index + 1][1]["bbox"][0]) - 0.01 * width) if index + 1 < len(markers) else visual_right + panel_bbox = [start, visual_top, end, visual_bottom] + labels = [] + for item in visual_records: + text_value = re.sub(r"\s+", " ", str(item.get("text") or "")).strip() + if not text_value or text_value.casefold() in panel_labels or len(text_value) > 140: + continue + if sum(char.isalpha() for char in text_value) < 2: + continue + bbox = item["bbox"] + center_x = (float(bbox[0]) + float(bbox[2])) / 2.0 + center_y = (float(bbox[1]) + float(bbox[3])) / 2.0 + if not (panel_bbox[0] <= center_x <= panel_bbox[2] and panel_bbox[1] <= center_y <= panel_bbox[3]): + continue + hint = role_hint(text_value) + labels.append({ + "evidence_id": str(item.get("id") or ""), + "text": text_value, + "role_hint": hint, + "required_candidate": bool(hint), + "bbox_percent": [ + round(float(bbox[0]) / width, 4), + round(float(bbox[1]) / height, 4), + round(float(bbox[2]) / width, 4), + round(float(bbox[3]) / height, 4), + ], + }) + generic_module_labels = {"model", "network", "neuralnetwork", "encoder", "decoder"} + for item in labels: + if item.get("role_hint") != "module" or _normalized_compare_text(str(item.get("text") or "")) not in generic_module_labels: + continue + bbox = item.get("bbox_percent") or [0.0, 0.0, 0.0, 0.0] + for other in labels: + if other is item or other.get("role_hint") != "module": + continue + other_key = _normalized_compare_text(str(other.get("text") or "")) + if not other_key or other_key in generic_module_labels: + continue + other_bbox = other.get("bbox_percent") or [0.0, 0.0, 0.0, 0.0] + overlap = max(0.0, min(float(bbox[2]), float(other_bbox[2])) - max(float(bbox[0]), float(other_bbox[0]))) + min_width = max(1e-6, min(float(bbox[2]) - float(bbox[0]), float(other_bbox[2]) - float(other_bbox[0]))) + vertical_gap = float(bbox[1]) - float(other_bbox[3]) + if overlap / min_width >= 0.6 and -0.004 <= vertical_gap <= 0.018: + item["role_hint"] = "supporting" + item["required_candidate"] = False + break + panels.append({ + "panel_label": panel_label, + "description": descriptions.get(panel_label, ""), + "bbox_percent": [ + round(panel_bbox[0] / width, 4), + round(panel_bbox[1] / height, 4), + round(panel_bbox[2] / width, 4), + round(panel_bbox[3] / height, 4), + ], + "label_inventory": labels[: max(1, int(max_items_per_panel))], + }) + return json.dumps({ + "summary": "Concise panel-local label inventory recovered from explicit figure panel markers.", + "page": page_number, + "orientation": orientation, + "panels": panels, + }, ensure_ascii=False, separators=(",", ":")) + + def evidence_excerpt(parsed: dict, max_chars: int = 58000) -> str: + overview_priority = _overview_figure_evidence_priority(parsed) + + def priority_key(item: dict[str, Any]) -> tuple[int, int, int, str]: + item_id = str(item.get("id") or "") + if item_id in overview_priority: + local_tier, reading_order = overview_priority[item_id] + return local_tier, reading_order, int(item.get("page") or 0), str(item.get("block_id") or "") + tier = ( + 10 if item.get("kind") == "caption" and re.match(r"^(figure|fig\.)\s*1\b", str(item.get("text") or ""), re.IGNORECASE) and re.search(r"\b(components?|overview|framework|architecture|pipeline)\b", str(item.get("text") or ""), re.IGNORECASE) else + 11 if item.get("kind") == "caption" and re.search(r"\b(overview|framework|architecture|pipeline)\b", str(item.get("text") or ""), re.IGNORECASE) else + 12 if item.get("kind") == "caption" and re.match(r"^(figure|fig\.)\s*1\b", str(item.get("text") or ""), re.IGNORECASE) else + 13 if item.get("kind") == "caption" else + 14 if any(term in str(item.get("section_hint") or "").casefold() for term in ("abstract", "introduction", "method", "approach", "architecture", "framework", "conclusion")) else 15 + ) + return tier, 0, int(item.get("page") or 0), str(item.get("block_id") or "") + + prioritized = sorted( + parsed.get("evidence", []), + key=priority_key, + ) lines: list[str] = [] used = 0 - for item in parsed.get("evidence", []): - block = f"[{item['id']} | page {item['page']} | {item.get('section_hint') or 'unknown section'}]\n{item['text']}" + for item in prioritized: + block = f"[{item['id']} | page {item['page']} | {item.get('section_hint') or 'unknown section'} | {item.get('block_id') or 'no block'}]\n{item['text']}" if used + len(block) > max_chars: - break + continue lines.append(block) used += len(block) return "\n\n".join(lines) diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py new file mode 100644 index 0000000..b5dfad3 --- /dev/null +++ b/rfs/paper_to_image/contract_completion.py @@ -0,0 +1,785 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + + +def _normalized(value: object) -> str: + return re.sub(r"[^a-z0-9]+", "", str(value or "").casefold()) + + +def _label(item: dict[str, Any]) -> str: + return str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("label") or item.get("id") or "").strip() + + +def _stable_id(prefix: str, label: str) -> str: + slug = "_".join(part for part in re.sub(r"[^a-z0-9]+", " ", label.casefold()).split() if part)[:48] + return f"{prefix}_{slug or 'item'}" + + +@dataclass(frozen=True) +class ConceptRule: + key: str + label: str + field: str + role: str + pattern: str + aliases: tuple[str, ...] = () + requires_overview: bool = False + context_pattern: str | None = None + + +EMBEDDING_SUM_CONTEXT = r"\btoken(?: embeddings?)?\s*,\s*(?:the\s+)?(?:segment|segmentation)(?: embeddings?)?\s*,?\s+and\s+(?:the\s+)?position embeddings?\b" + + +CONCEPT_RULES = ( + ConceptRule("input_image", "Input Image", "inputs", "input", r"\binput images?\b|\bsplit an image\b|\braw image\b|\btest images?\b|\bimages? are shown\b", ("image", "2d image")), + ConceptRule("images", "Images", "inputs", "input modality", r"\b(?:batch of\s+)?\(?images?\s*[,)]", ("training images", "image batch"), True), + ConceptRule("image", "Image", "inputs", "input modality", r"\bimages?\s*\+\s*text\b|\bimage encoder\b", ("images", "input image"), True), + ConceptRule("text", "Text", "inputs", "input modality", r"\b(?:batch of\s+)?\(?image\s*,\s*text\)?|\bimages?\s*\+\s*text\b|\btext training examples?\b", ("captions", "training text", "text batch")), + ConceptRule("audio", "Audio", "inputs", "input modality", r"\baudio\b", (), True), + ConceptRule("depth", "Depth", "inputs", "input modality", r"\bdepth\b", (), True), + ConceptRule("thermal", "Thermal", "inputs", "input modality", r"\bthermal\b", (), True), + ConceptRule("imu", "IMU", "inputs", "input modality", r"\bimu\b", (), True), + ConceptRule("prompt", "Prompt", "inputs", "input", r"\b(?:input|segmentation|point|box|mask) prompts?\b|\bprompt encoder\b", ("sparse prompts", "dense prompts", "segmentation prompt"), True), + ConceptRule("task_input", "Input", "inputs", "input", r"\bgiven an input\b|\btask (?:prompt|instruction)\b", ("task prompt", "task instruction"), True), + ConceptRule("camera_ray", "Camera Rays", "inputs", "input", r"\bcamera rays?\b", ("camera ray", "ray", "rays")), + ConceptRule("viewing_direction", "Viewing Direction", "inputs", "conditioning", r"\bview(?:ing)? directions?\b", ("view direction",)), + ConceptRule("object_queries", "Object Queries", "inputs", "conditioning", r"\bobject quer(?:y|ies)\b", ("learned object queries",)), + ConceptRule("class_descriptions", "Class Descriptions", "inputs", "conditioning", r"\b(?:names or descriptions|class names|class descriptions|text prompts)\b", ("class names", "text prompts")), + ConceptRule("cnn_backbone", "CNN Backbone", "modules", "feature extractor", r"\b(?:conventional\s+|common\s+)?cnn backbone\b|\bconvolutional backbone\b", ("backbone", "convolutional backbone")), + ConceptRule("backbone", "Backbone", "modules", "feature extractor", r"\bbackbones?\b", ("backbone architecture",), True), + ConceptRule("image_encoder", "Image Encoder", "modules", "encoder", r"\bimage encoder\b"), + ConceptRule("text_encoder", "Text Encoder", "modules", "encoder", r"\btext encoder\b"), + ConceptRule("transformer_encoder", "Transformer Encoder", "modules", "encoder", r"\btransformer encoder\b|^encoder$", (), True), + ConceptRule("transformer_decoder", "Transformer Decoder", "modules", "decoder", r"\btransformer decoder\b|\b(?:the\s+)?decoder receives\b|^decoder$", (), True), + ConceptRule("position_embedding", "Position Embedding", "modules", "conditioning", r"\bposition embeddings?\b", ("positional embedding",), True), + ConceptRule("positional_encoding", "Positional Encoding", "modules", "conditioning", r"\bposition(?:al)? encod(?:ing|ings)\b"), + ConceptRule("image_patches", "Image Patches", "modules", "intermediate", r"\bimage patches?\b|\bfixed-size patches\b", ("patches",), True), + ConceptRule("linear_projection", "Linear Projection of Flattened Patches", "modules", "projection", r"\blinear projection(?: of flattened patches)?\b|\blinearly embed\b", ("linear projection",), True), + ConceptRule("class_token", "Class Token", "modules", "conditioning", r"\bclass(?:ification)? token\b|\bclass embedding\b", ("classification token", "class embedding"), True), + ConceptRule("mlp_head", "MLP Head", "modules", "prediction head", r"\bmlp head\b|\bclassification head[^.]{0,100}\bmlp\b", ("mlp",)), + ConceptRule("sampled_points", "Sampled 3D Points", "modules", "intermediate", r"\bsampl(?:e|ed|ing)\s+(?:the\s+)?(?:5d coordinates|3d points)\b|\b5d coordinates\b", ("5d coordinates", "spatial locations")), + ConceptRule("prediction_ffn", "Feed Forward Network", "modules", "prediction head", r"\bfeed[ -]?forward network\b|\bprediction ffn\b|\bffn\b", ("ffn", "prediction feed-forward network")), + ConceptRule("region_proposals", "Region Proposals", "modules", "proposal artifact", r"\bregion proposals?\b|\bregion proposal network\b|\brpn\b", ("proposals", "region proposal network (rpn)")), + ConceptRule("roi_align", "RoIAlign", "modules", "alignment", r"\broialign\b"), + ConceptRule("generate", "Generate", "modules", "generator", r"\bstarts? by generating\b|\bgenerate(?:s|d| an output)?\b", ("init", "generator"), True), + ConceptRule("initial_output", "Initial Output", "modules", "artifact", r"\binitial output\b|\bpreviously generated output\b", (), True), + ConceptRule("feedback", "Feedback", "modules", "feedback provider", r"\bget feedback\b|\breceive feedback\b|\bfeedback provider\b", ("feedback provider",), True), + ConceptRule("self_feedback", "Self-Feedback", "modules", "feedback artifact", r"\bself[ -]?feedback\b|\bfeedback is passed back\b", ("self-feedback (nl)",), True), + ConceptRule("refine", "Refine", "modules", "refiner", r"\brefines? the previously generated output\b|\biterate\s*/\s*refine\b", ("refiner",), True), + ConceptRule("modality_encoders", "Modality Encoders", "modules", "module group", r"\bmodality encoders?\b|\bencoders for each modality\b", ("transformer-based encoders for each modality",)), + ConceptRule("joint_embedding", "Joint Embedding Space", "modules", "shared representation", r"\b(?:single shared )?(?:joint|common) embedding space\b|\bshared representation space\b", ("joint embedding", "single shared joint embedding space")), + ConceptRule("prompt_encoder", "Prompt Encoder", "modules", "encoder", r"\bprompt encoder\b"), + ConceptRule("mask_decoder", "Mask Decoder", "modules", "decoder", r"\bmask decoder\b"), + ConceptRule("mlp", "MLP", "modules", "neural field", r"\bneural radiance field\b|\bfeeding[^.]{0,100}\bmlp\b|\bmlp to produce\b", ("neural radiance field", "neural network")), + ConceptRule("image_embeddings", "Image Embeddings", "modules", "representation", r"\bimage embeddings?\b|\bimage features?\b", ("image embedding", "image features"), True), + ConceptRule("text_embeddings", "Text Embeddings", "modules", "representation", r"\btext embeddings?\b|\btext features?\b", ("text embedding", "text features")), + ConceptRule("contrastive_objective", "Contrastive Learning", "modules", "training objective", r"\bcontrastive (?:pre-?training|learning|objective|loss)\b|\bcorrect pairings?\b", ("contrastive pre-training", "image-text contrastive loss", "correct pairings")), + ConceptRule("bipartite_matching", "Bipartite Matching", "modules", "training objective", r"\bbipartite matching\b|\bhungarian matching\b", ("hungarian matching", "set prediction loss")), + ConceptRule("volume_density", "Volume Density", "modules", "field output", r"\bvolume density\b", ("density", "sigma")), + ConceptRule("color", "Color", "modules", "field output", r"\b(?:rgb|emitted) colou?r\b|\bradiance\b|\bcolou?r and volume density\b", ("rgb color", "radiance"), True), + ConceptRule("volume_rendering", "Volume Rendering", "modules", "renderer", r"\b(?:differentiable\s+)?volume rendering\b", ("differentiable volume rendering",)), + ConceptRule("class_predictions", "Class Predictions", "outputs", "output", r"\bclass (?:prediction|predictions|labels?)\b|\bpredicts?[^.]{0,80}\bclass\b", ("classes", "class labels")), + ConceptRule("class_prediction", "Class Prediction", "outputs", "output", r"\bimage classification predictions?\b|\bclassification predictions?\b|\bclass label\b", ("image classification predictions", "classification predictions", "class label", "class")), + ConceptRule("box_predictions", "Bounding Box Predictions", "outputs", "output", r"\bbounding box(?:es| predictions?)?\b|\bbox predictions?\b", ("bounding boxes", "box predictions")), + ConceptRule("classification", "Classification", "outputs", "output head", r"\bclassification head\b|\bpredicts? the class label\b", ("class label",)), + ConceptRule("box_regression", "Bounding-box Regression", "outputs", "output head", r"\bbounding[ -]?box regression\b|\bbox regression\b", ("bounding box", "box branch")), + ConceptRule("mask_branch", "Mask Branch", "outputs", "output head", r"\bmask branch\b"), + ConceptRule("refined_output", "Refined Output", "outputs", "output", r"\brefined output\b|\brefines? (?:the )?(?:previously generated )?output\b|\bquality of the output improves\b"), + ConceptRule("emergent_alignment", "Emergent Cross-modal Alignment", "outputs", "output", r"\bemergent (?:cross-modal )?alignments?\b|\bbinding (?:agent|property)\b", ("emergent alignment", "binding agent", "binding property")), + ConceptRule("valid_mask", "Valid Segmentation Mask", "outputs", "output", r"\bvalid masks?\b|\bobject masks?\b", ("valid masks", "valid mask"), True), + ConceptRule("zero_shot_classifier", "Zero-Shot Classifier", "outputs", "inference output", r"\bzero[ -]?shot (?:linear )?classifier\b", ("zero-shot linear classifier",)), + ConceptRule("zero_shot_prediction", "Zero-shot Prediction", "outputs", "inference output", r"\bzero[ -]?shot prediction\b", ("zero-shot prediction",)), + ConceptRule("rendered_image", "Rendered Image", "outputs", "output", r"\b(?:rendered|synthesi[sz]ed) (?:images?|views?)\b|\bsynthesi[sz]e images?\b|\bnovel views?\b", ("synthesized image", "synthesized view", "novel view")), + ConceptRule("source_tokens", "Inputs", "inputs", "source sequence", r"\bencoder maps an input sequence(?: of symbol representations)?\b|\blearned embeddings to convert the input\s+tokens\b", ("input sequence", "source tokens")), + ConceptRule("input_embedding", "Input Embedding", "modules", "embedding", r"\blearned embeddings to convert the input\s+tokens\b|\binput embeddings?\b", ("input embeddings",), context_pattern=r"\bencoder\b.*\bdecoder\b|\bencoder and decoder stacks\b"), + ConceptRule("encoder_stack", "Encoder Stack", "modules", "encoder", r"\bencoder is composed of a stack\b|\bencoder stack\b", ("encoder", "transformer encoder")), + ConceptRule("target_tokens", "Outputs (shifted right)", "inputs", "target sequence", r"\boutput embeddings? (?:are )?offset by one position\b|\binput\s+tokens and output tokens\b", ("target tokens", "target sequence", "output sequence", "outputs shifted right")), + ConceptRule("output_embedding", "Output Embedding", "modules", "embedding", r"\boutput embeddings?\b|\binput\s+tokens and output tokens\b", ("output embeddings",), context_pattern=r"\bencoder\b.*\bdecoder\b|\bencoder and decoder stacks\b"), + ConceptRule("decoder_stack", "Decoder Stack", "modules", "decoder", r"\bdecoder is also composed of a stack\b|\bdecoder stack\b", ("decoder", "transformer decoder")), + ConceptRule("output_linear", "Linear", "modules", "output projection", r"\blearned linear transfor(?:mation|\-\s*mation) and softmax function to convert the decoder output\b", ("linear layer", "output projection")), + ConceptRule("output_softmax", "Softmax", "modules", "output normalization", r"\blinear transfor(?:mation|\-\s*mation) and softmax function to convert the decoder output\b", ("softmax layer",)), + ConceptRule("output_probabilities", "Output Probabilities", "outputs", "output", r"\bpredicted next-token probabilities\b", ("predicted next-token probabilities", "next-token probabilities", "outputs")), + ConceptRule("input_tokens", "Input Sequence", "inputs", "input sequence", r"\brepresent the input\s+sequence\b|\bfor a given token, its input representation\b", ("input tokens", "token sequence"), context_pattern=EMBEDDING_SUM_CONTEXT), + ConceptRule("token_embeddings", "Token Embeddings", "modules", "embedding", rf"\btoken embeddings?\b|{EMBEDDING_SUM_CONTEXT}", (), context_pattern=EMBEDDING_SUM_CONTEXT), + ConceptRule("segment_embeddings", "Segment Embeddings", "modules", "embedding", rf"\b(?:segment|segmentation) embeddings?\b|{EMBEDDING_SUM_CONTEXT}", ("segmentation embeddings",), context_pattern=EMBEDDING_SUM_CONTEXT), + ConceptRule("bert_position_embeddings", "Position Embeddings", "modules", "embedding", r"\bposition embeddings?\b", (), context_pattern=EMBEDDING_SUM_CONTEXT), + ConceptRule("input_representation", "Input Representation", "modules", "representation", r"\binput representation\b|\binput embeddings are the sum of\b", ("bert input representation",), context_pattern=EMBEDDING_SUM_CONTEXT), + ConceptRule("bidirectional_encoder", "Bidirectional Transformer Encoder", "modules", "encoder", r"\bmulti-layer bidirectional transformer en-?\s*coder\b|\bbidirectional transformer encoder\b", ("bert", "bert encoder")), + ConceptRule("masked_lm", "Masked LM", "modules", "training objective", r"\bmasked (?:language model(?:ing)?|lm)\b", ("masked language model", "masked language modeling", "mlm objective"), context_pattern=r"\bnext sentence prediction\b|\bNSP\b"), + ConceptRule("next_sentence", "Next Sentence Prediction", "modules", "training objective", r"\bnext sentence prediction\b|\bNSP\b", ("nsp",), context_pattern=r"\bmasked (?:language model(?:ing)?|lm)\b"), + ConceptRule("fine_tuning", "Fine-tuning", "modules", "adaptation", r"\bframework:\s*pre-training and fine-tuning\b|\bpre-trained model parameters are used to initialize models for different down-stream tasks\b", ("fine-tune", "fine tuning")), + ConceptRule("downstream_tasks", "Downstream Tasks", "outputs", "task outputs", r"\bmodels for different down-stream tasks\b|\blabeled data from the downstream tasks\b|\bmodel many downstream tasks\b", ("task-specific outputs", "fine-tuning tasks")), + ConceptRule("input_query", "Input Query", "inputs", "query", r"\bfor query x\b|\bgiven a query x\b|\bquery representation produced by a query encoder\b", ("query", "input x")), + ConceptRule("query_encoder", "Query Encoder", "modules", "encoder", r"\bquery encoder\b"), + ConceptRule("document_index", "Document Index", "modules", "non-parametric memory", r"\bdocument index\b|\bdense vector index\b", ("non-parametric memory", "wikipedia index")), + ConceptRule("retriever", "Retriever", "modules", "retriever", r"\b(?:pre-trained |neural )?retriever\b"), + ConceptRule("top_k_documents", "Top-K Documents", "modules", "retrieved context", r"\btop-?k documents\b|\btop k documents are retrieved\b", ("retrieved documents", "retrieved passages")), + ConceptRule("generator_module", "Generator", "modules", "generator", r"\bpre-trained seq2seq model\s*\(\s*generator\s*\)|\bgenerator p|\bgenerator component\b", ("generate", "seq2seq generator", "pre-trained generator")), + ConceptRule("output_sequence", "Output Sequence", "outputs", "generated output", r"\bgenerator produces the output sequence\b|\bgenerating the target sequence y\b|\bfinal prediction y\b", ("prediction y", "final prediction", "generated output", "output y")), +) + + +RELATION_RULES = ( + ("input_image", "image_patches", "data_flow"), + ("image_patches", "linear_projection", "data_flow"), + ("linear_projection", "transformer_encoder", "data_flow"), + ("position_embedding", "transformer_encoder", "conditioning"), + ("class_token", "transformer_encoder", "conditioning"), + ("transformer_encoder", "mlp_head", "data_flow"), + ("mlp_head", "class_prediction", "data_flow"), + ("mlp_head", "classification", "data_flow"), + ("input_image", "cnn_backbone", "data_flow"), + ("input_image", "backbone", "data_flow"), + ("cnn_backbone", "transformer_encoder", "data_flow"), + ("positional_encoding", "transformer_encoder", "conditioning"), + ("transformer_encoder", "transformer_decoder", "data_flow"), + ("object_queries", "transformer_decoder", "conditioning"), + ("transformer_decoder", "prediction_ffn", "data_flow"), + ("prediction_ffn", "class_predictions", "branch"), + ("prediction_ffn", "box_predictions", "branch"), + ("class_predictions", "bipartite_matching", "training_objective"), + ("box_predictions", "bipartite_matching", "training_objective"), + ("backbone", "region_proposals", "data_flow"), + ("region_proposals", "roi_align", "data_flow"), + ("backbone", "roi_align", "feature_flow"), + ("roi_align", "classification", "branch"), + ("roi_align", "class_predictions", "branch"), + ("roi_align", "box_regression", "branch"), + ("roi_align", "box_predictions", "branch"), + ("roi_align", "mask_branch", "branch"), + ("task_input", "generate", "data_flow"), + ("generate", "initial_output", "data_flow"), + ("initial_output", "feedback", "evaluation"), + ("feedback", "self_feedback", "feedback"), + ("initial_output", "refine", "revision_input"), + ("self_feedback", "refine", "feedback"), + ("refine", "refined_output", "data_flow"), + ("refined_output", "feedback", "feedback_loop"), + ("image", "modality_encoders", "encoding"), + ("text", "modality_encoders", "encoding"), + ("audio", "modality_encoders", "encoding"), + ("depth", "modality_encoders", "encoding"), + ("thermal", "modality_encoders", "encoding"), + ("imu", "modality_encoders", "encoding"), + ("modality_encoders", "joint_embedding", "alignment"), + ("joint_embedding", "emergent_alignment", "enables"), + ("image", "image_encoder", "data_flow"), + ("prompt", "prompt_encoder", "data_flow"), + ("image_encoder", "mask_decoder", "feature_flow"), + ("prompt_encoder", "mask_decoder", "conditioning"), + ("mask_decoder", "valid_mask", "data_flow"), + ("images", "image_encoder", "encoding"), + ("text", "text_encoder", "encoding"), + ("image_encoder", "image_embeddings", "data_flow"), + ("text_encoder", "text_embeddings", "data_flow"), + ("image_embeddings", "contrastive_objective", "alignment"), + ("text_embeddings", "contrastive_objective", "alignment"), + ("class_descriptions", "text_encoder", "encoding"), + ("image_embeddings", "zero_shot_classifier", "classification"), + ("text_embeddings", "zero_shot_classifier", "classification"), + ("camera_ray", "sampled_points", "sampling"), + ("sampled_points", "positional_encoding", "encoding"), + ("positional_encoding", "mlp", "data_flow"), + ("viewing_direction", "mlp", "conditioning"), + ("mlp", "volume_density", "prediction"), + ("mlp", "color", "prediction"), + ("volume_density", "volume_rendering", "rendering_input"), + ("color", "volume_rendering", "rendering_input"), + ("volume_rendering", "rendered_image", "data_flow"), + ("source_tokens", "input_embedding", "embedding"), + ("input_embedding", "encoder_stack", "data_flow"), + ("positional_encoding", "encoder_stack", "conditioning"), + ("target_tokens", "output_embedding", "embedding"), + ("output_embedding", "decoder_stack", "data_flow"), + ("positional_encoding", "decoder_stack", "conditioning"), + ("encoder_stack", "decoder_stack", "cross_attention"), + ("decoder_stack", "output_linear", "data_flow"), + ("output_linear", "output_softmax", "data_flow"), + ("output_softmax", "output_probabilities", "data_flow"), + ("input_tokens", "token_embeddings", "embedding"), + ("token_embeddings", "input_representation", "sum"), + ("segment_embeddings", "input_representation", "sum"), + ("bert_position_embeddings", "input_representation", "sum"), + ("input_representation", "bidirectional_encoder", "data_flow"), + ("bidirectional_encoder", "masked_lm", "training_objective"), + ("bidirectional_encoder", "next_sentence", "training_objective"), + ("bidirectional_encoder", "fine_tuning", "initialization"), + ("fine_tuning", "downstream_tasks", "data_flow"), + ("input_query", "query_encoder", "encoding"), + ("query_encoder", "retriever", "retrieval_query"), + ("document_index", "retriever", "memory_lookup"), + ("retriever", "top_k_documents", "retrieval"), + ("input_query", "generator_module", "generation_input"), + ("top_k_documents", "generator_module", "conditioning"), + ("generator_module", "output_sequence", "generation"), +) + + +def _overview_candidates(parsed: dict[str, Any], limit: int = 2) -> list[dict[str, Any]]: + positive = re.compile(r"\b(overview|framework|architecture|pipeline|procedure|approach|model|system|workflow|representation|encoder|decoder|directly predicts)\b", re.IGNORECASE) + negative = re.compile(r"\b(comparison|differences?|versus|performance|result|ablation|visualization|qualitative|attention|distribution|interface|scaling|dimensions?|timings?|compute|memory)\b", re.IGNORECASE) + ranked = [] + for index, figure in enumerate(parsed.get("document_index", {}).get("figures", [])): + caption = str(figure.get("caption") or "") + positive_match = positive.search(caption) + if not positive_match: + continue + score = 8 + (4 if re.match(r"^(figure|fig\.)\s*[12]\b", caption, re.IGNORECASE) else 0) + score += 3 if 160 <= len(caption) <= 1600 else 0 + score -= 14 if negative.search(caption) else 0 + ranked.append((score, -index, figure)) + ranked.sort(reverse=True) + if not ranked: + return [] + best_score = ranked[0][0] + # Keep a second, explicitly architectural figure when it is not a result or + # analysis figure. Early Figure 1/2 overviews still rank first, while the + # negative-caption penalty excludes late scaling, timing, attention, and + # visualization figures. The second page often contains the readable + # subsystem labels that a short overview caption omits. + minimum_score = max(8, best_score - 4) + return [item for score, _, item in ranked if score >= minimum_score][:limit] + + +def _evidence_for_figure(parsed: dict[str, Any], figure: dict[str, Any]) -> dict[str, Any] | None: + caption = str(figure.get("caption") or "") + return next( + ( + item + for item in parsed.get("evidence", []) + if item.get("kind") == "caption" + and item.get("page") == figure.get("page") + and str(item.get("text") or "").startswith(caption[:60]) + ), + None, + ) + + +def _relevant_evidence(parsed: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + candidates = _overview_candidates(parsed) + caption_evidence = [item for item in (_evidence_for_figure(parsed, figure) for figure in candidates) if item] + selected_pages = {int(item.get("page") or 0) for item in caption_evidence if item.get("page")} + selected = list(caption_evidence) + for item in parsed.get("evidence", []): + if int(item.get("page") or 0) not in selected_pages or item in selected: + continue + section = str(item.get("section_hint") or "").casefold() + if any(term in section for term in ("reference", "acknowledg")): + continue + item_bbox = item.get("bbox") if isinstance(item.get("bbox"), list) and len(item.get("bbox")) == 4 else None + captions = [value for value in caption_evidence if value.get("page") == item.get("page")] + near_caption = False + for caption in captions: + caption_bbox = caption.get("bbox") if isinstance(caption.get("bbox"), list) and len(caption.get("bbox")) == 4 else None + if item_bbox is None or caption_bbox is None: + near_caption = True + break + if float(item_bbox[1]) <= float(caption_bbox[3]) + 72.0 and float(item_bbox[3]) >= float(caption_bbox[1]) - 540.0: + near_caption = True + break + if near_caption: + selected.append(item) + selected_ids = {item["id"] for item in selected} + page_count = max(1, int(parsed.get("page_count") or 1)) + early_page_limit = max(8, int(page_count * 0.45)) + useful_sections = ("abstract", "introduction", "method", "approach", "architecture", "framework", "model", "system", "figure captions") + relevant = list(selected) + for item in parsed.get("evidence", []): + if item.get("id") in selected_ids: + continue + section = str(item.get("section_hint") or "").casefold() + if any(term in section for term in ("reference", "acknowledg")): + continue + if int(item.get("page") or page_count + 1) <= early_page_limit or any(term in section for term in useful_sections): + relevant.append(item) + return selected, relevant + + +def _window_matches(pattern: re.Pattern[str], items: list[dict[str, Any]], max_window: int = 3) -> list[dict[str, Any]]: + for start in range(len(items)): + page = items[start].get("page") + for size in range(2, max_window + 1): + window = items[start:start + size] + if len(window) != size or any(item.get("page") != page for item in window): + break + if pattern.search(" ".join(str(item.get("text") or "") for item in window)): + return window + return [] + + +def _find_matches(rule: ConceptRule, selected: list[dict[str, Any]], relevant: list[dict[str, Any]]) -> list[dict[str, Any]]: + pattern = re.compile(rule.pattern, re.IGNORECASE) + if rule.context_pattern: + context = re.compile(rule.context_pattern, re.IGNORECASE | re.DOTALL) + context_text = " ".join(str(item.get("text") or "") for item in [*selected, *relevant]) + if not context.search(context_text): + return [] + primary = [item for item in selected if pattern.search(str(item.get("text") or ""))] + if primary: + return primary[:2] + primary_window = _window_matches(pattern, selected) + if primary_window: + return primary_window + if rule.requires_overview: + return [] + secondary = [item for item in relevant if pattern.search(str(item.get("text") or ""))] + secondary_window = _window_matches(pattern, relevant) + if rule.context_pattern and secondary_window: + return secondary_window + if secondary: + return secondary[:2] + return secondary_window + + +def _find_existing(spec: dict[str, Any], rule: ConceptRule) -> tuple[str, dict[str, Any]] | None: + accepted = {_normalized(rule.key), _normalized(rule.label), *(_normalized(value) for value in rule.aliases)} + for field in ("inputs", "modules", "outputs", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + raw_value = _label(item) + value = _normalized(raw_value) + item_id = _normalized(item.get("id")) + if item_id in accepted or value in accepted: + return field, item + if rule.label.isupper() and re.match(rf"^\s*{re.escape(rule.label)}\s*(?:\(|$)", raw_value, re.IGNORECASE): + return field, item + if rule.key != "self_feedback" and field == rule.field and len(value) >= 8 and any(len(candidate) >= 8 and (value in candidate or candidate in value) for candidate in accepted if candidate): + return field, item + return None + + +def _relation_bridge(source: dict[str, Any], target: dict[str, Any], relevant: list[dict[str, Any]]) -> dict[str, Any] | None: + def terms(item: dict[str, Any]) -> set[str]: + values = set() + for word in re.findall(r"[a-z0-9]+", _label(item).casefold()): + if len(word) < 4 or word in {"shifted", "right", "stack", "module", "layer"}: + continue + values.add(word[:-1] if word.endswith("s") and len(word) > 4 else word) + return values + + source_terms = terms(source) + target_terms = terms(target) + distinct_target_terms = target_terms - source_terms or target_terms + if not source_terms or not distinct_target_terms: + return None + return next( + ( + item + for item in relevant + if any(term in _normalized(item.get("text")) for term in source_terms) + and any(term in _normalized(item.get("text")) for term in distinct_target_terms) + ), + None, + ) + + +def _ground_statement(item: object, relevant: list[dict[str, Any]]) -> list[str]: + if not isinstance(item, dict) or item.get("evidence_ids"): + return [] + statement = str(item.get("text") or item.get("statement") or "").strip() + if not statement or statement.casefold() == "unknown": + return [] + stop_words = {"a", "an", "and", "are", "as", "for", "in", "is", "of", "on", "or", "the", "to", "we", "with"} + target_terms = {word for word in re.findall(r"[a-z0-9]+", statement.casefold()) if len(word) >= 4 and word not in stop_words} + if not target_terms: + return [] + best_score = 0.0 + best_window: list[dict[str, Any]] = [] + for start in range(len(relevant)): + page = relevant[start].get("page") + for size in range(1, 4): + window = relevant[start:start + size] + if len(window) != size or any(value.get("page") != page for value in window): + break + text_terms = set(re.findall(r"[a-z0-9]+", " ".join(str(value.get("text") or "") for value in window).casefold())) + score = len(target_terms & text_terms) / len(target_terms) + if score > best_score: + best_score = score + best_window = window + if best_score < 0.6: + item["text"] = "unknown" + item["status"] = "unknown" + return [] + evidence_ids = [str(value.get("id")) for value in best_window if value.get("id")] + item["evidence_ids"] = evidence_ids + return evidence_ids + + +def _ground_declared_entities(spec: dict[str, Any], relevant: list[dict[str, Any]]) -> list[str]: + grounded: list[str] = [] + for field in ("inputs", "modules", "outputs", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict) or item.get("evidence_ids"): + continue + label = _label(item) + normalized_label = _normalized(label) + acronym = bool(label.isupper() and 3 <= len(normalized_label) <= 12) + if not acronym and (len(normalized_label) < 8 or len(re.findall(r"[a-z0-9]+", label.casefold())) < 2): + continue + match: list[dict[str, Any]] = [] + for start in range(len(relevant)): + page = relevant[start].get("page") + for size in range(1, 4): + window = relevant[start:start + size] + if len(window) != size or any(value.get("page") != page for value in window): + break + combined_text = " ".join(str(value.get("text") or "") for value in window) + combined = _normalized(combined_text) + exact_acronym = acronym and re.search(rf"(? list[str]: + evidence_by_id = { + str(item.get("id")): item + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + corrected: list[str] = [] + for item in spec.get("outputs", []) if isinstance(spec.get("outputs"), list) else []: + if not isinstance(item, dict) or _normalized(_label(item)) != "zeroshotclassifier": + continue + evidence_text = " ".join( + str(evidence_by_id[evidence_id].get("text") or "") + for evidence_id in item.get("evidence_ids", []) + if evidence_id in evidence_by_id + ) + if re.search(r"\bzero[ -]?shot (?:linear )?classifier\b", evidence_text, re.IGNORECASE): + continue + if re.search(r"\bzero[ -]?shot prediction\b", evidence_text, re.IGNORECASE): + item["name"] = "Zero-shot Prediction" + corrected.append(str(item.get("id") or "zero_shot_prediction")) + return corrected + + +def _deduplicate_entities(spec: dict[str, Any]) -> list[str]: + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} + visible_aliases = { + _normalized(source): str(visible).strip() + for source, visible in terminology.items() + if _normalized(source) and str(visible).strip() + } + id_remap: dict[str, str] = {} + removed: list[str] = [] + seen: dict[str, dict[str, Any]] = {} + for field in ("inputs", "modules", "outputs", "innovations"): + items = spec.get(field, []) if isinstance(spec.get(field), list) else [] + field_seen = {} if field == "innovations" else seen + unique: list[dict[str, Any]] = [] + for item in items: + if not isinstance(item, dict): + continue + raw_key = _normalized(_label(item)) + if raw_key in visible_aliases: + item["name"] = visible_aliases[raw_key] + key = _normalized(_label(item)) + if field == "inputs" and (key == "input" or re.fullmatch(r"input[a-z0-9]{1,4}", key)): + key = "input" + if not key or key not in field_seen: + if key: + field_seen[key] = item + unique.append(item) + continue + kept = field_seen[key] + kept["evidence_ids"] = list(dict.fromkeys([*kept.get("evidence_ids", []), *item.get("evidence_ids", [])])) + duplicate_id = str(item.get("id") or "") + kept_id = str(kept.get("id") or "") + if duplicate_id and kept_id and duplicate_id != kept_id: + id_remap[duplicate_id] = kept_id + removed.append(duplicate_id or _label(item)) + spec[field] = unique + if id_remap: + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + relation["source"] = id_remap.get(str(relation.get("source")), relation.get("source")) + relation["target"] = id_remap.get(str(relation.get("target")), relation.get("target")) + return removed + + +def _heuristic_rule_scope( + selected: list[dict[str, Any]], + relevant: list[dict[str, Any]], + declared_rule_keys: set[str], +) -> tuple[set[str], dict[str, list[dict[str, Any]]]]: + selected_ids = {str(item.get("id")) for item in selected if item.get("id")} + scoped_relevant = relevant + matches = { + rule.key: found + for rule in CONCEPT_RULES + if (found := _find_matches(rule, selected, scoped_relevant)) + } + candidates = set(matches) + if not candidates: + return set(), matches + seeds = declared_rule_keys | { + rule.key + for rule in CONCEPT_RULES + if rule.key in candidates and _find_matches(rule, selected, []) + } + adjacency = {key: set() for key in candidates} + + def locally_connected(left: str, right: str) -> bool: + left_items, right_items = matches.get(left, []), matches.get(right, []) + if not left_items or not right_items: + return True + for left_item in left_items: + for right_item in right_items: + left_page, right_page = int(left_item.get("page") or 0), int(right_item.get("page") or 0) + if not left_page or not right_page: + return True + page_gap = abs(left_page - right_page) + if 0 < page_gap <= 3: + return True + if page_gap != 0: + continue + left_bbox, right_bbox = left_item.get("bbox"), right_item.get("bbox") + if not (isinstance(left_bbox, list) and len(left_bbox) == 4 and isinstance(right_bbox, list) and len(right_bbox) == 4): + return True + left_center = (float(left_bbox[1]) + float(left_bbox[3])) / 2.0 + right_center = (float(right_bbox[1]) + float(right_bbox[3])) / 2.0 + if abs(left_center - right_center) <= 360.0: + return True + left_selected = any(str(item.get("id")) in selected_ids for item in left_items) + right_selected = any(str(item.get("id")) in selected_ids for item in right_items) + + def method_grounded(items: list[dict[str, Any]]) -> bool: + blocked = ("introduction", "background", "related work", "reference", "acknowledg", "experiment", "result") + return any(not any(term in str(item.get("section_hint") or "").casefold() for term in blocked) for item in items) + + if left_selected and method_grounded(right_items): + return True + if right_selected and method_grounded(left_items): + return True + return False + + for source_key, target_key, _ in RELATION_RULES: + if source_key in candidates and target_key in candidates and locally_connected(source_key, target_key): + adjacency[source_key].add(target_key) + adjacency[target_key].add(source_key) + + def component(start: str) -> set[str]: + reached: set[str] = set() + pending = [start] + while pending: + current = pending.pop() + if current in reached: + continue + reached.add(current) + pending.extend(adjacency.get(current, set()) - reached) + return reached + + if seeds & candidates: + allowed: set[str] = set() + for seed in seeds & candidates: + allowed.update(component(seed)) + return allowed, matches + + components: list[set[str]] = [] + remaining = set(candidates) + while remaining: + group = component(next(iter(remaining))) + components.append(group) + remaining -= group + components.sort(key=lambda group: (len(group), sum(len(matches[key]) for key in group)), reverse=True) + return components[0], matches + + +def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: + selected, relevant = _relevant_evidence(parsed) + corrected_entities = _correct_unsupported_entity_names(spec, parsed) + deduplicated_entities = _deduplicate_entities(spec) + declared_entities = [ + item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + fallback_declared = any(str(item.get("role") or "") == "paper-derived stage requiring VLM verification" for item in declared_entities) + conservative_expansion = len(declared_entities) >= 3 and len([item for item in spec.get("relations", []) if isinstance(item, dict)]) >= 1 and not fallback_declared + selected_pages = {int(item.get("page") or 0) for item in selected if item.get("page")} + relevant_non_background = [ + item + for item in relevant + if not any(term in str(item.get("section_hint") or "").casefold() for term in ("related work", "reference", "acknowledg")) + and not ( + any(term in str(item.get("section_hint") or "").casefold() for term in ("introduction", "background")) + and re.search(r"\b(?:for example|e\.g\.|such as)\b[^\n]{0,160}\[\s*\d+", str(item.get("text") or ""), re.IGNORECASE) + ) + ] + initially_declared_rule_keys = { + rule.key + for rule in CONCEPT_RULES + if _find_existing(spec, rule) + } + heuristic_rule_keys: set[str] | None = None + heuristic_matches: dict[str, list[dict[str, Any]]] = {} + if not conservative_expansion: + heuristic_rule_keys, heuristic_matches = _heuristic_rule_scope(selected, relevant_non_background, initially_declared_rule_keys) + grounded_statements = { + field: _ground_statement(spec.get(field), relevant) + for field in ("research_problem", "central_claim") + } + grounded_entities = _ground_declared_entities(spec, relevant) + found: dict[str, dict[str, Any]] = {} + added_entities: list[str] = [] + upgraded_entities: list[str] = [] + adopted_entities: list[str] = [] + for rule in CONCEPT_RULES: + existing = _find_existing(spec, rule) + if existing: + matches = _find_matches(rule, selected, relevant) + elif not conservative_expansion: + matches = heuristic_matches.get(rule.key, []) if rule.key in (heuristic_rule_keys or set()) else [] + else: + matches = _find_matches(rule, selected, []) + if not matches: + connected_to_declared = any( + (source_key == rule.key and target_key in initially_declared_rule_keys) + or (target_key == rule.key and source_key in initially_declared_rule_keys) + for source_key, target_key, _ in RELATION_RULES + ) + if connected_to_declared: + matches = _find_matches(rule, [], relevant_non_background) + if not matches: + continue + evidence_ids = list(dict.fromkeys(str(item.get("id")) for item in matches if item.get("id"))) + if existing: + _, item = existing + current = _label(item) + if str(item.get("role") or "") == "paper-derived stage requiring VLM verification": + item["role"] = rule.role + adopted_entities.append(str(item.get("id") or rule.key)) + accepted_names = {_normalized(rule.label), *(_normalized(value) for value in rule.aliases)} + exact_rule_id = _normalized(item.get("id")) == _normalized(rule.key) + if _normalized(current) != _normalized(rule.label) and (exact_rule_id or _normalized(current) in accepted_names or _normalized(current) in _normalized(rule.label) or current.casefold().startswith(("neural network module", "learned "))): + item["name"] = rule.label + upgraded_entities.append(str(item.get("id") or rule.key)) + item["evidence_ids"] = list(dict.fromkeys(list(item.get("evidence_ids", [])) + evidence_ids)) + found[rule.key] = item + continue + item = {"id": _stable_id(rule.field.rstrip("s"), rule.label), "name": rule.label, "role": rule.role, "evidence_ids": evidence_ids} + spec.setdefault(rule.field, []).append(item) + found[rule.key] = item + added_entities.append(item["id"]) + + if conservative_expansion: + for _pass in range(3): + progress = False + connected_keys = set(found) + for rule in CONCEPT_RULES: + if rule.key in found or _find_existing(spec, rule): + continue + connected_to_found = any( + (source_key == rule.key and target_key in connected_keys) + or (target_key == rule.key and source_key in connected_keys) + for source_key, target_key, _ in RELATION_RULES + ) + if not connected_to_found: + continue + matches = _find_matches(rule, selected, []) or _find_matches(rule, [], relevant_non_background) + if not matches: + continue + evidence_ids = list(dict.fromkeys(str(item.get("id")) for item in matches if item.get("id"))) + item = {"id": _stable_id(rule.field.rstrip("s"), rule.label), "name": rule.label, "role": rule.role, "evidence_ids": evidence_ids} + spec.setdefault(rule.field, []).append(item) + found[rule.key] = item + added_entities.append(item["id"]) + progress = True + if not progress: + break + + deduplicated_entities.extend(_deduplicate_entities(spec)) + found = {} + for rule in CONCEPT_RULES: + existing = _find_existing(spec, rule) + if existing: + found[rule.key] = existing[1] + + relations = spec.setdefault("relations", []) + existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + selected_ids = [str(item.get("id")) for item in selected if item.get("id")] + evidence_by_id = {str(item.get("id")): item for item in relevant if item.get("id")} + added_relations: list[str] = [] + repaired_relations: list[str] = [] + for source_key, target_key, relation_type in RELATION_RULES: + source = found.get(source_key) + target = found.get(target_key) + if not source or not target: + continue + pair = (str(source.get("id")), str(target.get("id"))) + if pair in existing_pairs: + continue + evidence_ids = list(dict.fromkeys(list(source.get("evidence_ids", [])) + list(target.get("evidence_ids", [])))) + if selected_ids and not any(value in selected_ids for value in evidence_ids): + bridge = _relation_bridge(source, target, relevant) + source_pages = [int(evidence_by_id[value].get("page") or 0) for value in source.get("evidence_ids", []) if value in evidence_by_id] + target_pages = [int(evidence_by_id[value].get("page") or 0) for value in target.get("evidence_ids", []) if value in evidence_by_id] + nearby = bool(source_pages and target_pages and min(abs(left - right) for left in source_pages for right in target_pages) <= 3) + if not bridge and not nearby: + continue + if bridge: + evidence_ids.append(str(bridge.get("id"))) + relations.append({"source": pair[0], "target": pair[1], "type": relation_type, "label": "", "evidence_ids": list(dict.fromkeys(evidence_ids))}) + existing_pairs.add(pair) + added_relations.append(f"{pair[0]}->{pair[1]}") + + endpoints = { + str(item.get("id")): item + for field in ("inputs", "modules", "outputs", "innovations") + for item in spec.get(field, []) if isinstance(item, dict) and item.get("id") + } + for relation in relations: + if not isinstance(relation, dict) or relation.get("evidence_ids"): + continue + source = endpoints.get(str(relation.get("source"))) + target = endpoints.get(str(relation.get("target"))) + if not source or not target: + continue + candidate_ids = list(dict.fromkeys(list(source.get("evidence_ids", [])) + list(target.get("evidence_ids", [])))) + source_pages = [int(evidence_by_id[value].get("page") or 0) for value in source.get("evidence_ids", []) if value in evidence_by_id] + target_pages = [int(evidence_by_id[value].get("page") or 0) for value in target.get("evidence_ids", []) if value in evidence_by_id] + nearby = bool(source_pages and target_pages and min(abs(left - right) for left in source_pages for right in target_pages) <= 3) + bridge = _relation_bridge(source, target, relevant) + if not candidate_ids or not (any(value in selected_ids for value in candidate_ids) or nearby or bridge): + continue + if bridge and bridge.get("id"): + candidate_ids.append(str(bridge["id"])) + relation["evidence_ids"] = list(dict.fromkeys(candidate_ids)) + repaired_relations.append(f"{relation.get('source')}->{relation.get('target')}") + + overview_terms = [rule.key for rule in CONCEPT_RULES if _find_matches(rule, selected, [])] + covered_terms = [key for key in overview_terms if key in found] + return { + "summary": "Conservative evidence-driven contract completion report.", + "selected_overview_evidence_ids": selected_ids, + "overview_term_count": len(overview_terms), + "covered_overview_term_count": len(covered_terms), + "overview_term_coverage": round(len(covered_terms) / max(1, len(overview_terms)), 4), + "added_entities": added_entities, + "upgraded_entities": upgraded_entities, + "adopted_entities": adopted_entities, + "added_relations": added_relations, + "repaired_relations": repaired_relations, + "grounded_statements": {field: ids for field, ids in grounded_statements.items() if ids}, + "grounded_entities": grounded_entities, + "corrected_entities": corrected_entities, + "deduplicated_entities": deduplicated_entities, + "conservative_expansion": conservative_expansion, + "expansion_page_scope": sorted(selected_pages), + "new_entities_require_declared_neighbor": conservative_expansion, + "rules_are_evidence_gated": True, + } diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 95f7e05..bf2358f 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -1,37 +1,107 @@ from __future__ import annotations +import os import json import re +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any, Callable -from PIL import Image +from PIL import Image, ImageChops, ImageStat -from ..reference_text_extractor import run_easyocr, run_paddle_ocr +from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available +from .planner import collect_visible_labels, collect_visual_relations, collect_visual_roles def _normalize_text(value: str) -> str: return re.sub(r"[^\w\u4e00-\u9fff]+", "", str(value or "").casefold()) +def _issue_text(value: object) -> str: + if not isinstance(value, dict): + return str(value).strip() + source = str(value.get("source") or value.get("source_label") or "").strip() + target = str(value.get("target") or value.get("target_label") or "").strip() + status = str(value.get("status") or value.get("type") or "").strip() + evidence = str(value.get("visible_evidence") or value.get("reason") or "").strip() + relation = f"{source} -> {target}" if source or target else "connector issue" + details = ": ".join(part for part in (status, evidence) if part) + return f"{relation}: {details}" if details else relation + + +def _is_connector_issue(value: object) -> bool: + text = _issue_text(value).casefold() + return any(token in text for token in ( + "arrow", "connector", "connection", "direct edge", "shortcut", "bypass", + "relation", "reversed", "source ->", "target ->", "连线", "箭头", "直连", + )) + + +def _reconcile_with_focused_topology(vlm_raw: dict, topology_review: dict, ground_truth: dict) -> dict: + """Let the focused connector verifier arbitrate connector-only disagreements. + + The general critic remains authoritative for paper semantics, roles, labels, + and aesthetics. This only suppresses connector claims when the independent + topology pass inspected all declared relations and found no connector issue. + """ + + if not topology_review.get("passed") or any(topology_review.get(field) for field in ("missing_relations", "reversed_relations", "bypassed_relations", "invented_relations")): + return {"applied": False, "reason": "focused topology did not pass cleanly", "suppressed": []} + + suppressed: list[dict] = [] + scientific = vlm_raw.get("scientific") if isinstance(vlm_raw.get("scientific"), dict) else {} + for field in ("missing_relations", "reversed_relations"): + for issue in scientific.get(field, []) if isinstance(scientific.get(field), list) else []: + suppressed.append({"section": "scientific", "field": field, "issue": issue}) + scientific[field] = [] + invented = scientific.get("invented_items", []) if isinstance(scientific.get("invented_items"), list) else [] + scientific["invented_items"] = [issue for issue in invented if not _is_connector_issue(issue)] + for issue in invented: + if _is_connector_issue(issue): + suppressed.append({"section": "scientific", "field": "invented_items", "issue": issue}) + vlm_raw["scientific"] = scientific + + if isinstance(vlm_raw.get("paper_alignment"), dict): + alignment = vlm_raw["paper_alignment"] + for field in ("evidence_conflicts", "unsupported_visual_claims", "repair"): + values = alignment.get(field, []) if isinstance(alignment.get(field), list) else [] + alignment[field] = [issue for issue in values if not _is_connector_issue(issue)] + for issue in values: + if _is_connector_issue(issue): + suppressed.append({"section": "paper_alignment", "field": field, "issue": issue}) + + priority_kinds = { + str(item.get("id") or "").strip(): str(item.get("kind") or "").strip() + for item in ground_truth.get("priorities", []) if isinstance(item, dict) + } + for priority in alignment.get("priorities", []) if isinstance(alignment.get("priorities"), list) else []: + if not isinstance(priority, dict): + continue + status = str(priority.get("status") or "").strip().casefold() + issue = priority.get("visible_evidence") or priority.get("reason") or "" + priority_id = str(priority.get("id") or "").strip() + if status not in {"supported", "present", "correct"} and _is_connector_issue(issue): + priority["status"] = "supported" + priority["reconciled_by"] = "focused_topology" + suppressed.append({"section": "paper_alignment", "field": "priorities", "priority_id": priority_id, "kind": priority_kinds.get(priority_id), "issue": issue}) + if not any(alignment.get(field) for field in ("missing_priorities", "evidence_conflicts", "unsupported_visual_claims")): + alignment["passed"] = True + for field in ("repair", "remove"): + values = vlm_raw.get(field, []) if isinstance(vlm_raw.get(field), list) else [] + vlm_raw[field] = [issue for issue in values if not _is_connector_issue(issue)] + for issue in values: + if _is_connector_issue(issue): + suppressed.append({"section": "critic", "field": field, "issue": issue}) + return { + "applied": bool(suppressed), + "reason": "focused topology is authoritative for visible connector geometry", + "suppressed": suppressed, + } + + def required_labels(plan: dict) -> list[str]: - labels: list[str] = [] - spec = plan.get("figure_specification", {}) - terminology = spec.get("terminology", {}) - if isinstance(terminology, dict): - labels.extend(str(value) for value in terminology.values()) - elif isinstance(terminology, list): - labels.extend(str(item.get("visible_label") or item.get("statement") or "") for item in terminology if isinstance(item, dict)) - for module in spec.get("modules", []): - if isinstance(module, dict): - labels.append(str(module.get("name") or module.get("statement") or "")) - result = [] - for label in labels: - label = label.strip() - if label and len(label) <= 48 and label not in result: - result.append(label) - return result[:24] + return collect_visible_labels(plan.get("figure_specification", {})) def _local_ocr(path: Path, engine: str, lang: str, adapter: Callable | None = None) -> tuple[list[dict], str, str | None]: @@ -39,6 +109,8 @@ def _local_ocr(path: Path, engine: str, lang: str, adapter: Callable | None = No if adapter: records = adapter(path, lang) return records, "adapter", None + if engine == "rapidocr": + return run_rapidocr(path, lang), "rapidocr", None if engine == "easyocr": return run_easyocr(path, lang), "easyocr", None if engine == "paddle": @@ -48,7 +120,8 @@ def _local_ocr(path: Path, engine: str, lang: str, adapter: Callable | None = No return [], engine, "local OCR disabled" -def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: list[str], forbidden_labels: list[str], model: str | None, adapter: Callable | None = None) -> dict: +def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: list[str], repeatable_labels: list[str], optional_visible_labels: list[str], forbidden_labels: list[str], model: str | None, adapter: Callable | None = None) -> dict: + review_ground_truth = plan.get("review_ground_truth") if isinstance(plan.get("review_ground_truth"), dict) else {} prompt = f""" # Summary @@ -57,22 +130,42 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: Required exact labels: {json.dumps(labels, ensure_ascii=False)} +Evidence-supported labels that may repeat to show reuse of the same component: +{json.dumps(repeatable_labels, ensure_ascii=False)} + +Evidence-supported optional labels that may appear but must not be reported missing: +{json.dumps(optional_visible_labels, ensure_ascii=False)} + Forbidden copied reference labels: {json.dumps(forbidden_labels, ensure_ascii=False)} Scientific specification: {json.dumps(plan.get('figure_specification', {}), ensure_ascii=False, indent=2)} +Paper-evidence review ground truth. Treat the quoted source excerpts and page numbers as higher authority than visual plausibility. Evaluate every required priority by ID. Do not infer a common architecture when the evidence does not state it: +{json.dumps(review_ground_truth, ensure_ascii=False, indent=2)} + +Mandatory visual connector checklist: +{json.dumps(collect_visual_relations(plan.get('figure_specification', {})), ensure_ascii=False, indent=2)} + +Mandatory entity role and containment checklist: +{json.dumps(collect_visual_roles(plan.get('figure_specification', {})), ensure_ascii=False, indent=2)} + +Judge the visible flow against both checklists. Relations involving an evidence-supported repeatable shared component may be represented by repeated badges rather than extra implementation-level arrows. A shortcut that bypasses a checklist module is a scientific error. Merely showing every word is insufficient: a dataset/source shown as a peer modality, an output shown as an internal module, or a modality shown as a processing stage is a hard scientific role mismatch. + Selected template: {json.dumps({key: template.get(key) for key in ['template_id', 'panels', 'connectors', 'visual_density', 'style']}, ensure_ascii=False, indent=2)} +Template interpretation rule: panels are macro spatial regions and role guides, not an exact required count of scientific content nodes. Do not report a template mismatch merely because multiple paper-grounded nodes appear inside one macro region. Judge reading order, spatial roles, connector rhythm, and the required feedback topology. + Return: {{ "summary": "Production candidate review.", "ocr": {{"detected_labels": [], "missing_labels": [], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "score": 0.0, "passed": false}}, - "scientific": {{"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "innovation_visible": true, "score": 0.0, "passed": false}}, + "scientific": {{"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "role_mismatches": [], "containment_mismatches": [], "innovation_visible": true, "score": 0.0, "passed": false}}, + "paper_alignment": {{"priorities": [{{"id": "...", "status": "supported|missing|conflicting|uncertain", "visible_evidence": "...", "evidence_ids": []}}], "missing_priorities": [], "evidence_conflicts": [], "unsupported_visual_claims": [], "repair": [], "score": 0.0, "passed": false}}, "template": {{"macro_panel_match": 0.0, "reading_order_match": 0.0, "connector_rhythm_match": 0.0, "visual_density_match": 0.0, "copied_reference_content": [], "score": 0.0, "passed": false}}, - "aesthetic": {{"hierarchy": 0.0, "balance": 0.0, "whitespace": 0.0, "color": 0.0, "icon_consistency": 0.0, "readability": 0.0, "score": 0.0, "passed": false}}, + "aesthetic": {{"hierarchy": 0.0, "balance": 0.0, "whitespace": 0.0, "color": 0.0, "icon_consistency": 0.0, "readability": 0.0, "visual_information_density": 0.0, "mechanism_visualization": 0.0, "publication_polish": 0.0, "score": 0.0, "passed": false}}, "preserve": [], "repair": [], "remove": [], @@ -82,12 +175,27 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: "overall_score": 0.0 }} -Hard failures: any missing/misspelled critical label, copied reference term, missing core module, reversed relation, invented mechanism, or template score below 0.72. Aesthetic quality cannot compensate for scientific or OCR failure. +Hard failures: any missing/misspelled critical label, any duplicate label not explicitly allowed above, copied reference term, missing paper priority, conflict with a quoted paper passage, unsupported visual claim, missing core module, reversed relation, invented mechanism, semantic role/containment mismatch, or template score below 0.72. Aesthetic quality cannot compensate for scientific, evidence-alignment, topology, or OCR failure. + +Paper-alignment rules: +- Return exactly one priorities record for every required priority ID in review_ground_truth. +- A priority is supported only when the visible image agrees with both its label/relationship and the quoted evidence. +- Put visually asserted content with no support in the specification/evidence into unsupported_visual_claims. +- Put any contradiction with a quoted passage into evidence_conflicts and cite evidence_ids. +- Unknown items must remain unknown; resolving them through a plausible-looking visual is unsupported invention. + +Aesthetic scoring rules: +- A neat text-only flowchart is not a complete Image-2 scientific figure. Empty rounded rectangles containing only labels must score below 0.70 for mechanism_visualization and visual_information_density. +- Inspect whether inputs, core methods, innovations, and outputs contain paper-grounded visual cues, examples, mechanism details, or icons rather than unused interior space. +- icon_consistency cannot receive full credit when no icons or scientific visual metaphors are present. +- visual_density_match must reflect actual information inside nodes, not merely the number and alignment of boxes. +- Reserve scores above 0.90 for figures that are both structurally correct and visually explanatory at publication quality. """.strip() if adapter: return adapter(path, blueprint, prompt) resolved = resolve_vlm_model("RFS_PAPER_TO_IMAGE_REVIEW_MODEL", "RFS_CRITIC_MODEL", explicit_model=model) - raw = call_vlm_json(prompt, [path, blueprint], model=resolved, timeout=240, retries=1) + timeout = max(15, int(os.getenv("RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT", "90"))) + raw = call_vlm_json(prompt, [path, blueprint], model=resolved, timeout=timeout, retries=0) raw.setdefault("summary", "Production candidate review.") raw["model"] = resolved return raw @@ -97,13 +205,130 @@ def _normalize_section(raw: Any, name: str) -> dict: section = raw if isinstance(raw, dict) else {} section.setdefault("summary", f"{name} review.") try: - section["score"] = max(0.0, min(1.0, float(section.get("score", 0.0)))) + score = float(section.get("score", 0.0)) + if 1.0 < score <= 10.0: + score /= 10.0 + elif 10.0 < score <= 100.0: + score /= 100.0 + section["score"] = max(0.0, min(1.0, score)) except Exception: section["score"] = 0.0 section["passed"] = bool(section.get("passed", False)) return section +def _normalize_paper_alignment(raw: Any, ground_truth: dict, scientific: dict, adapter_used: bool) -> dict: + required = [item for item in ground_truth.get("priorities", []) if isinstance(item, dict) and item.get("required")] + strict = bool(ground_truth.get("strict_evidence_required")) + if not isinstance(raw, dict): + if adapter_used or not strict: + return { + "summary": "Paper alignment inherited from a trusted adapter or legacy contract without source excerpts.", + "priorities": [], + "missing_priorities": [], + "evidence_conflicts": [], + "unsupported_visual_claims": [], + "score": float(scientific.get("score") or 0.0), + "passed": bool(scientific.get("passed")), + "legacy_fallback": True, + } + return { + "summary": "Paper-evidence alignment was not returned by the production critic.", + "priorities": [], + "missing_priorities": [ + {"id": item.get("id"), "label": item.get("label"), "evidence_ids": item.get("evidence_ids", []), "reason": "not evaluated by critic"} + for item in required + ], + "evidence_conflicts": [], + "unsupported_visual_claims": [], + "score": 0.0, + "passed": False, + } + alignment = _normalize_section(raw, "Paper alignment") + for field in ("priorities", "missing_priorities", "evidence_conflicts", "unsupported_visual_claims", "repair"): + if not isinstance(alignment.get(field), list): + alignment[field] = [] + evaluated = { + str(item.get("id") or "").strip(): str(item.get("status") or "").strip().casefold() + for item in alignment["priorities"] if isinstance(item, dict) and str(item.get("id") or "").strip() + } + missing_ids = { + str(item.get("id") or "").strip() if isinstance(item, dict) else str(item).strip() + for item in alignment["missing_priorities"] + } + if strict: + for item in required: + item_id = str(item.get("id") or "").strip() + status = evaluated.get(item_id) + if status not in {"supported", "present", "correct"} and item_id not in missing_ids: + alignment["missing_priorities"].append({ + "id": item_id, + "label": item.get("label"), + "evidence_ids": item.get("evidence_ids", []), + "reason": "required priority was not explicitly verified as supported", + }) + missing_ids.add(item_id) + issue_count = sum(len(alignment[field]) for field in ("missing_priorities", "evidence_conflicts", "unsupported_visual_claims")) + if strict and required and not issue_count and all(evaluated.get(str(item.get("id") or "").strip()) in {"supported", "present", "correct"} for item in required): + alignment["score"] = 1.0 + alignment["passed"] = bool(alignment.get("passed")) and alignment["score"] >= 0.95 and issue_count == 0 + return alignment + + +def _vlm_topology_critic(path: Path, plan: dict, model: str | None, adapter: Callable | None = None) -> dict: + specification = plan.get("figure_specification", {}) + relations = collect_visual_relations(specification) + repeatable_labels = [str(value).strip() for value in specification.get("repeatable_labels", []) if str(value).strip()] + prompt = f""" +# Summary + +Act as a focused topology verifier for one scientific framework image. Inspect visible connector paths and arrowheads rather than inferring the intended method. Return JSON only. + +Mandatory directed connectors: +{json.dumps(relations, ensure_ascii=False, indent=2)} + +Mandatory entity roles: +{json.dumps(collect_visual_roles(specification), ensure_ascii=False, indent=2)} + +Evidence-supported repeatable shared-component labels: +{json.dumps(repeatable_labels, ensure_ascii=False)} + +Rules: +- Verify every source, target, and arrowhead direction from visible geometry. +- A line that joins the outgoing side of a target or a downstream junction does not count as entering that target. +- A shortcut that bypasses an intermediate module is an invented relation. +- Verify that source, modality, module, and output layers preserve their declared roles. A source -> modality -> method chain must not be flattened into peer inputs. +- For refinement loops, Initial Output and Self-Feedback must both enter Refine or its explicit input-side shared-model node before Refined Output. +- Refined Output must return to Feedback to close the loop. +- Do not penalize repeated labels explicitly allowed by the scientific contract. +- Respect explicit containment semantics. When a mandatory source is a clearly labeled operation/container and an allowed repeatable shared-component badge inside that container visibly produces the target, count the container-to-target relation as present. The nested badge explains who implements the container operation; it is not an invented extra scientific edge. +- In particular, a FEEDBACK container holding an allowed shared Model M badge with an arrow to Self-Feedback satisfies FEEDBACK -> Self-Feedback. Do not require an unnatural arrow starting at the container border or title text. +- Apply the containment exception only when the shared badge is visibly inside the declared source container and is listed above as repeatable. Do not use it to excuse missing cross-container arrows or shortcuts. + +Return: +{{ + "summary": "Focused visible-connector verification.", + "relations": [{{"source": "...", "target": "...", "status": "present|missing|reversed|bypassed", "visible_evidence": "..."}}], + "missing_relations": [], + "reversed_relations": [], + "bypassed_relations": [], + "invented_relations": [], + "repair": [], + "repair_regions": [], + "score": 0.0, + "passed": false +}} +""".strip() + if adapter: + return adapter(path, prompt) + resolved = resolve_vlm_model("RFS_PAPER_TO_IMAGE_TOPOLOGY_MODEL", "RFS_FROZEN_JUDGE_MODEL", explicit_model=model) + timeout = max(15, int(os.getenv("RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT", "90"))) + result = call_vlm_json(prompt, [path], model=resolved, timeout=timeout, retries=0) + result.setdefault("summary", "Focused visible-connector verification.") + result["model"] = resolved + return result + + def review_candidate( path: str | Path, blueprint: str | Path, @@ -115,17 +340,95 @@ def review_candidate( ocr_lang: str = "en_ch", ocr_adapter: Callable | None = None, critic_adapter: Callable | None = None, + topology_adapter: Callable | None = None, + acceptable_aspect_ratios: list[str] | None = None, + require_visual_enrichment: bool = False, ) -> dict: candidate = Path(path) blueprint_path = Path(blueprint) labels = required_labels(plan) + repeatable_labels = [str(value).strip() for value in plan.get("figure_specification", {}).get("repeatable_labels", []) if str(value).strip()] + optional_visible_labels = [str(value).strip() for value in plan.get("figure_specification", {}).get("optional_visible_labels", []) if str(value).strip()] forbidden = [str(value) for value in template.get("forbidden_copy_terms", []) if str(value).strip()] with Image.open(candidate) as image: width, height = image.size with Image.open(blueprint_path) as image: bw, bh = image.size - ratio_error = abs((width / max(height, 1)) - (bw / max(bh, 1))) / max(bw / max(bh, 1), 0.01) - basic = {"summary": "Deterministic candidate checks.", "valid_image": width > 0 and height > 0, "width": width, "height": height, "blueprint_width": bw, "blueprint_height": bh, "aspect_ratio_error": round(ratio_error, 5), "passed": ratio_error <= 0.08 and min(width, height) >= 512} + with Image.open(candidate) as candidate_image, Image.open(blueprint_path) as blueprint_image: + candidate_rgb = candidate_image.convert("RGB") + blueprint_rgb = blueprint_image.convert("RGB").resize(candidate_rgb.size) + visual_diff = ImageChops.difference(candidate_rgb, blueprint_rgb) + threshold_mask = visual_diff.point(lambda value: 255 if value > 35 else 0).convert("L") + mask_histogram = threshold_mask.histogram() + changed_pixels = sum(mask_histogram[1:]) + total_pixels = max(1, candidate_rgb.size[0] * candidate_rgb.size[1]) + blueprint_global_enrichment_ratio = changed_pixels / total_pixels + corner_pixels = [ + blueprint_rgb.getpixel((0, 0)), + blueprint_rgb.getpixel((blueprint_rgb.width - 1, 0)), + blueprint_rgb.getpixel((0, blueprint_rgb.height - 1)), + blueprint_rgb.getpixel((blueprint_rgb.width - 1, blueprint_rgb.height - 1)), + ] + background_color = tuple(sorted(pixel[channel] for pixel in corner_pixels)[len(corner_pixels) // 2] for channel in range(3)) + background = Image.new("RGB", blueprint_rgb.size, background_color) + foreground_mask = ImageChops.difference(blueprint_rgb, background).point(lambda value: 255 if value > 8 else 0).convert("L") + active_bbox = foreground_mask.getbbox() or (0, 0, blueprint_rgb.width, blueprint_rgb.height) + padding = 24 + active_bbox = ( + max(0, active_bbox[0] - padding), + max(0, active_bbox[1] - padding), + min(blueprint_rgb.width, active_bbox[2] + padding), + min(blueprint_rgb.height, active_bbox[3] + padding), + ) + active_diff = threshold_mask.crop(active_bbox) + active_pixels = max(1, active_diff.width * active_diff.height) + blueprint_active_enrichment_ratio = sum(active_diff.histogram()[1:]) / active_pixels + blueprint_enrichment_ratio = blueprint_active_enrichment_ratio + blueprint_mean_abs_difference = sum(ImageStat.Stat(visual_diff).mean) / 3.0 + try: + minimum_enrichment_ratio = max(0.0, min(1.0, float(os.getenv("RFS_MIN_BLUEPRINT_ENRICHMENT_RATIO", "0.08")))) + except ValueError: + minimum_enrichment_ratio = 0.08 + visual_enrichment_passed = not require_visual_enrichment or blueprint_enrichment_ratio >= minimum_enrichment_ratio + candidate_ratio = width / max(height, 1) + ratio_targets: list[tuple[str, float]] = [("blueprint", bw / max(bh, 1))] + for value in acceptable_aspect_ratios or []: + try: + left, right = str(value).split(":", 1) + parsed = float(left) / float(right) + except Exception: + continue + if parsed <= 0 or any(abs(parsed - target) <= 0.001 for _, target in ratio_targets): + continue + ratio_targets.append((str(value), parsed)) + ratio_errors = { + label: round(abs(candidate_ratio - target) / max(target, 0.01), 5) + for label, target in ratio_targets + } + matched_ratio, ratio_error = min(ratio_errors.items(), key=lambda item: item[1]) + basic = { + "summary": "Deterministic candidate checks.", + "valid_image": width > 0 and height > 0, + "width": width, + "height": height, + "blueprint_width": bw, + "blueprint_height": bh, + "candidate_aspect_ratio": round(candidate_ratio, 5), + "acceptable_aspect_ratios": [label for label, _ in ratio_targets], + "aspect_ratio_errors": ratio_errors, + "matched_aspect_ratio": matched_ratio, + "aspect_ratio_error": ratio_error, + "visual_enrichment_required": bool(require_visual_enrichment), + "blueprint_enrichment_ratio": round(blueprint_enrichment_ratio, 5), + "blueprint_global_enrichment_ratio": round(blueprint_global_enrichment_ratio, 5), + "blueprint_active_enrichment_ratio": round(blueprint_active_enrichment_ratio, 5), + "blueprint_active_region_bbox": list(active_bbox), + "blueprint_active_region_fraction": round(active_pixels / total_pixels, 5), + "blueprint_mean_abs_difference": round(blueprint_mean_abs_difference, 3), + "minimum_blueprint_enrichment_ratio": minimum_enrichment_ratio, + "visual_enrichment_passed": visual_enrichment_passed, + "passed": ratio_error <= 0.08 and min(width, height) >= 512 and visual_enrichment_passed, + } local_records: list[dict] = [] local_engine = "not_run" @@ -136,15 +439,47 @@ def review_candidate( if requested_engine in {"paddle", "easyocr"}: local_records, local_engine, local_warning = _local_ocr(candidate, requested_engine, ocr_lang, adapter=ocr_adapter) + topology_name = str(plan.get("figure_specification", {}).get("topology") or "unknown") + declared_relations = collect_visual_relations(plan.get("figure_specification", {})) + visual_roles = collect_visual_roles(plan.get("figure_specification", {})) + has_structured_provenance = any(str(item.get("role") or "").casefold() == "data_source" for item in visual_roles) + topology_required = bool(declared_relations) or topology_name in {"feedback", "branch", "multimodal", "dense_multiframe"} or str(template.get("template_id") or "") in {"feedback", "arbor"} or has_structured_provenance + focused_topology_available = bool(topology_adapter or (critic_adapter is None and vlm_credentials_available())) + focused_topology_required = topology_required and focused_topology_available vlm_raw: dict = {} + topology_raw: dict = {} vlm_warning = None - if mode == "vlm" and (vlm_credentials_available() or critic_adapter): - try: - vlm_raw = _vlm_critic(candidate, blueprint_path, plan, template, labels, forbidden, model, adapter=critic_adapter) - except Exception as exc: - vlm_warning = str(exc) + topology_warning = None + review_available = vlm_credentials_available() or critic_adapter or topology_adapter + if mode == "vlm" and review_available: + with ThreadPoolExecutor(max_workers=2 if focused_topology_required else 1) as executor: + critic_future = executor.submit( + _vlm_critic, + candidate, + blueprint_path, + plan, + template, + labels, + repeatable_labels, + optional_visible_labels, + forbidden, + model, + critic_adapter, + ) + topology_future = executor.submit(_vlm_topology_critic, candidate, plan, model, topology_adapter) if focused_topology_required else None + try: + vlm_raw = critic_future.result() + except Exception as exc: + vlm_warning = str(exc) + if topology_future is not None: + try: + topology_raw = topology_future.result() + except Exception as exc: + topology_warning = str(exc) elif mode == "vlm": vlm_warning = "VLM review credentials unavailable" + if focused_topology_required: + topology_warning = vlm_warning detected_local = [str(item.get("text") or "").strip() for item in local_records if str(item.get("text") or "").strip()] ocr = _normalize_section(vlm_raw.get("ocr"), "OCR") @@ -163,42 +498,142 @@ def review_candidate( else: ocr.setdefault("local_engine", local_engine) ocr.setdefault("local_detected_text", []) + repeatable_normalized = {_normalize_text(value) for value in repeatable_labels if _normalize_text(value)} + optional_normalized = {_normalize_text(value) for value in optional_visible_labels if _normalize_text(value)} + required_normalized = {_normalize_text(value) for value in labels if _normalize_text(value)} + duplicate_labels = [str(value) for value in ocr.get("duplicate_labels", [])] + allowed_duplicate_labels = [value for value in duplicate_labels if _normalize_text(value) in repeatable_normalized] + noncritical_duplicate_labels = [ + value for value in duplicate_labels + if _normalize_text(value) not in repeatable_normalized and _normalize_text(value) not in required_normalized + ] + ocr["allowed_duplicate_labels"] = allowed_duplicate_labels + ocr["noncritical_duplicate_labels"] = noncritical_duplicate_labels + ocr["duplicate_labels"] = [ + value for value in duplicate_labels + if _normalize_text(value) not in repeatable_normalized and _normalize_text(value) in required_normalized + ] + detected_labels = [str(value).strip() for value in ocr.get("detected_labels", []) if str(value).strip()] + allowed_visible_labels = required_normalized | repeatable_normalized | optional_normalized + ocr["unexpected_labels"] = [ + value for value in detected_labels + if _normalize_text(value) and _normalize_text(value) not in allowed_visible_labels + ] + topology_review = _normalize_section(topology_raw, "Topology") if focused_topology_required else {"summary": "Focused topology review not available or not required for this run.", "score": 1.0, "passed": True, "skipped": True} + topology_issues = sum(len(topology_review.get(field, [])) for field in ["missing_relations", "reversed_relations", "bypassed_relations", "invented_relations"]) + if focused_topology_required: + topology_review["passed"] = bool(topology_review.get("passed")) and topology_review["score"] >= 0.9 and topology_issues == 0 and not topology_warning + ground_truth = plan.get("review_ground_truth") if isinstance(plan.get("review_ground_truth"), dict) else {} + cross_critic_reconciliation = _reconcile_with_focused_topology(vlm_raw, topology_review, ground_truth) if focused_topology_required else {"applied": False, "reason": "focused topology was not run", "suppressed": []} scientific = _normalize_section(vlm_raw.get("scientific"), "Scientific") + unexpected_visible_labels = [str(value) for value in ocr.get("unexpected_labels", []) if str(value).strip()] + scientific["unexpected_visible_labels"] = unexpected_visible_labels + invented_items = [str(value) for value in scientific.get("invented_items", []) if str(value).strip()] template_review = _normalize_section(vlm_raw.get("template"), "Template") aesthetic = _normalize_section(vlm_raw.get("aesthetic"), "Aesthetic") - ocr_issues = sum(len(ocr.get(field, [])) for field in ["missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found"]) - ocr["passed"] = bool(ocr.get("passed")) and ocr["score"] >= 0.999 and ocr_issues == 0 - scientific_issues = sum(len(scientific.get(field, [])) for field in ["missing_modules", "missing_relations", "reversed_relations", "invented_items"]) + for field in ("visual_information_density", "mechanism_visualization", "publication_polish"): + try: + aesthetic[field] = max(0.0, min(1.0, float(aesthetic.get(field, aesthetic["score"])))) + except Exception: + aesthetic[field] = aesthetic["score"] + scientific_issues = sum(len(scientific.get(field, [])) for field in ["missing_modules", "missing_relations", "reversed_relations", "invented_items", "role_mismatches", "containment_mismatches"]) has_innovations = bool(plan.get("figure_specification", {}).get("innovations")) innovation_ok = not has_innovations or bool(scientific.get("innovation_visible", False)) - scientific["passed"] = bool(scientific.get("passed")) and scientific["score"] >= 0.95 and scientific_issues == 0 and innovation_ok + if scientific_issues == 0 and innovation_ok: + scientific["score"] = 1.0 + scientific["passed"] = scientific["score"] >= 0.95 and scientific_issues == 0 and innovation_ok + paper_alignment = _normalize_paper_alignment(vlm_raw.get("paper_alignment"), ground_truth, scientific, critic_adapter is not None) + unsupported_sources = [*invented_items, *(_issue_text(value) for value in paper_alignment.get("unsupported_visual_claims", []))] + unsupported_unexpected_labels = [ + value for value in unexpected_visible_labels + if any( + _normalize_text(value) + and (_normalize_text(value) in _normalize_text(source) or _normalize_text(source) in _normalize_text(value)) + for source in unsupported_sources if _normalize_text(source) + ) + ] + ocr["unsupported_unexpected_labels"] = unsupported_unexpected_labels + ocr_issues = sum(len(ocr.get(field, [])) for field in ["missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found", "unsupported_unexpected_labels"]) + if ocr_issues == 0: + ocr["score"] = 1.0 + ocr["passed"] = ocr["score"] >= 0.999 and ocr_issues == 0 copied = template_review.get("copied_reference_content", []) - template_review["passed"] = bool(template_review.get("passed")) and template_review["score"] >= 0.72 and not copied - aesthetic["passed"] = bool(aesthetic.get("passed")) and aesthetic["score"] >= 0.75 + template_review["passed"] = template_review["score"] >= 0.70 and not copied + aesthetic_floor = min(aesthetic["visual_information_density"], aesthetic["mechanism_visualization"], aesthetic["publication_polish"]) + aesthetic["passed"] = aesthetic["score"] >= 0.78 and aesthetic_floor >= 0.70 and visual_enrichment_passed if mode != "vlm": template_review.update({"score": 1.0 if basic["passed"] else 0.0, "passed": basic["passed"], "engineering_only": True}) aesthetic.update({"score": 0.0, "passed": False, "engineering_only": True}) scientific.update({"score": 0.0, "passed": False, "engineering_only": True}) + paper_alignment.update({"score": 0.0, "passed": False, "engineering_only": True}) ocr.update({"score": 0.0, "passed": False, "engineering_only": True}) - hard_errors = list(vlm_raw.get("hard_errors", [])) - production_pass = bool(basic["passed"] and ocr["passed"] and scientific["passed"] and template_review["passed"] and aesthetic["passed"] and not hard_errors) - score = 0.30 * scientific["score"] + 0.25 * ocr["score"] + 0.25 * template_review["score"] + 0.20 * aesthetic["score"] + hard_errors = [] + for value in vlm_raw.get("hard_errors", []): + text = str(value) + normalized_error = _normalize_text(text) + allowed_duplicate_error = "duplicate" in text.casefold() and any(label in normalized_error for label in repeatable_normalized) + noncritical_duplicate_error = "duplicate" in text.casefold() and any(_normalize_text(label) in normalized_error for label in noncritical_duplicate_labels) + reconciled_connector_error = bool(cross_critic_reconciliation.get("applied")) and _is_connector_issue(text) + benign_unexpected_error = ( + any(token in text.casefold() for token in ("unexpected", "whitelist", "outside the contract", "not in the contract")) + and not unsupported_unexpected_labels + ) or ( + any(token in text.casefold() for token in ("unexpected", "whitelist", "outside the contract", "not in the contract")) + and any(_normalize_text(label) in normalized_error for label in unexpected_visible_labels) + and not any(_normalize_text(label) in normalized_error for label in unsupported_unexpected_labels) + ) + contradicted_template_error = "template" in text.casefold() and template_review["passed"] and template_review["score"] >= 0.70 + if not allowed_duplicate_error and not noncritical_duplicate_error and not reconciled_connector_error and not benign_unexpected_error and not contradicted_template_error: + hard_errors.append(text) + if focused_topology_required and not topology_review["passed"]: + hard_errors.extend(_issue_text(value) for value in topology_review.get("missing_relations", [])) + hard_errors.extend(_issue_text(value) for value in topology_review.get("reversed_relations", [])) + hard_errors.extend(_issue_text(value) for value in topology_review.get("bypassed_relations", [])) + hard_errors.extend(_issue_text(value) for value in topology_review.get("invented_relations", [])) + if topology_warning: + hard_errors.append(f"Focused topology review unavailable: {topology_warning}") + if not paper_alignment["passed"]: + hard_errors.extend(_issue_text(value) for value in paper_alignment.get("missing_priorities", []) if _issue_text(value)) + hard_errors.extend(_issue_text(value) for value in paper_alignment.get("evidence_conflicts", []) if _issue_text(value)) + hard_errors.extend(_issue_text(value) for value in paper_alignment.get("unsupported_visual_claims", []) if _issue_text(value)) + if ground_truth.get("strict_evidence_required") and not isinstance(vlm_raw.get("paper_alignment"), dict) and critic_adapter is None: + hard_errors.append("Production critic did not return paper-evidence alignment") + hard_errors = list(dict.fromkeys(value for value in hard_errors if value)) + repair = list(vlm_raw.get("repair", [])) + repair.extend(_issue_text(value) for value in paper_alignment.get("repair", []) if _issue_text(value)) + repair.extend(_issue_text(value) for value in topology_review.get("repair", []) if _issue_text(value)) + repair_regions = list(vlm_raw.get("repair_regions", [])) + repair_regions.extend(_issue_text(value) for value in topology_review.get("repair_regions", []) if _issue_text(value)) + unexpected_labels = [str(value) for value in ocr.get("unsupported_unexpected_labels", []) if str(value).strip()] + repair.extend(f"Remove unsupported visible label: {value}" for value in unexpected_labels) + remove = list(vlm_raw.get("remove", [])) + remove.extend(unexpected_labels) + if require_visual_enrichment and not visual_enrichment_passed: + repair.append("Enrich node interiors with paper-grounded visual examples, mechanism cues, and consistent scientific icons while preserving all node boxes and connectors.") + hard_errors.append(f"Image-2 visual enrichment ratio {blueprint_enrichment_ratio:.3f} is below required {minimum_enrichment_ratio:.3f}; candidate is too close to the bare layout blueprint") + hard_errors = list(dict.fromkeys(hard_errors)) + production_pass = bool(basic["passed"] and ocr["passed"] and scientific["passed"] and paper_alignment["passed"] and topology_review["passed"] and template_review["passed"] and aesthetic["passed"] and not hard_errors) + score = 0.22 * scientific["score"] + 0.18 * paper_alignment["score"] + 0.17 * ocr["score"] + 0.17 * topology_review["score"] + 0.14 * template_review["score"] + 0.12 * aesthetic["score"] return { - "summary": "Four-layer production candidate review.", + "summary": "Evidence-grounded production candidate review.", "path": str(candidate), "mode": mode, "basic": basic, "ocr": ocr, "scientific": scientific, + "paper_alignment": paper_alignment, + "topology": topology_review, + "cross_critic_reconciliation": cross_critic_reconciliation, "template": template_review, "aesthetic": aesthetic, "hard_errors": hard_errors, "preserve": list(vlm_raw.get("preserve", [])), - "repair": list(vlm_raw.get("repair", [])), - "remove": list(vlm_raw.get("remove", [])), - "repair_regions": list(vlm_raw.get("repair_regions", [])), + "repair": list(dict.fromkeys(repair)), + "remove": list(dict.fromkeys(remove)), + "repair_regions": list(dict.fromkeys(repair_regions)), "local_ocr_warning": local_warning, "vlm_warning": vlm_warning, + "topology_warning": topology_warning, "production_pass": production_pass, "overall_score": round(score, 4), } @@ -210,6 +645,8 @@ def collect(section: str) -> dict: return { "ocr_review": collect("ocr"), "scientific_critic_report": collect("scientific"), + "paper_alignment_report": collect("paper_alignment"), + "topology_critic_report": collect("topology"), "template_alignment_report": collect("template"), "aesthetic_critic_report": collect("aesthetic"), } diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py new file mode 100644 index 0000000..f9dc990 --- /dev/null +++ b/rfs/paper_to_image/document_cache.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +from ..utils import read_json, write_json + + +DOCUMENT_CACHE_VERSION = 32 + + +def document_model_cacheable(parsed: dict[str, Any]) -> bool: + report = parsed.get("extraction_report", {}) + return bool(report.get("status") != "fail" and report.get("ocr_run_complete", True)) + + +def _cache_root() -> Path: + return Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) + + +def document_cache_path( + source: str | Path, + *, + ocr_engine: str, + ocr_lang: str, + max_chars: int, + max_ocr_pages: int, +) -> Path: + path = Path(source).resolve() + digest = hashlib.sha256(path.read_bytes()).hexdigest() + signature = json.dumps({ + "version": DOCUMENT_CACHE_VERSION, + "suffix": path.suffix.lower(), + "ocr_engine": str(ocr_engine), + "ocr_lang": str(ocr_lang), + "max_chars": int(max_chars), + "max_ocr_pages": int(max_ocr_pages), + }, sort_keys=True).encode("utf-8") + variant = hashlib.sha256(signature).hexdigest()[:16] + return _cache_root() / "documents" / digest / variant / "document_model.json" + + +def read_document_cache( + source: str | Path, + *, + ocr_engine: str, + ocr_lang: str, + max_chars: int = 90000, + max_ocr_pages: int = 6, +) -> dict[str, Any] | None: + path = document_cache_path(source, ocr_engine=ocr_engine, ocr_lang=ocr_lang, max_chars=max_chars, max_ocr_pages=max_ocr_pages) + if not path.exists(): + return None + cached = read_json(path) + if ( + not isinstance(cached, dict) + or cached.get("document_cache_version") != DOCUMENT_CACHE_VERSION + or not document_model_cacheable(cached) + ): + return None + active = Path(source).resolve() + cached["source_path"] = str(active) + cached["source_name"] = active.name + if isinstance(cached.get("extraction_report"), dict): + cached["extraction_report"]["source_path"] = str(active) + return cached + + +def write_document_cache( + source: str | Path, + parsed: dict[str, Any], + *, + ocr_engine: str, + ocr_lang: str, + max_chars: int = 90000, + max_ocr_pages: int = 6, +) -> Path | None: + if not document_model_cacheable(parsed): + return None + path = document_cache_path(source, ocr_engine=ocr_engine, ocr_lang=ocr_lang, max_chars=max_chars, max_ocr_pages=max_ocr_pages) + payload = dict(parsed) + payload["document_cache_version"] = DOCUMENT_CACHE_VERSION + write_json(path, payload) + return path diff --git a/rfs/paper_to_image/editable_ppt.py b/rfs/paper_to_image/editable_ppt.py new file mode 100644 index 0000000..75a5f98 --- /dev/null +++ b/rfs/paper_to_image/editable_ppt.py @@ -0,0 +1,358 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ..composition.pptx import compile_ppt +from ..utils import write_json +from .semantic_blueprint import compile_semantic_blueprint + + +_NODE_STYLES = { + "inputs": {"fill_color": "#EDF5FC", "stroke_color": "#2A63A1", "text_color": "#244B72"}, + "modules": {"fill_color": "#EDF9F7", "stroke_color": "#1F756F", "text_color": "#195B57"}, + "outputs": {"fill_color": "#F2F8F3", "stroke_color": "#427E50", "text_color": "#315F3B"}, + "innovations": {"fill_color": "#FCF4EB", "stroke_color": "#B15C21", "text_color": "#8C481A"}, +} + +_RELATION_STYLES = { + "feedback_loop": {"stroke_color": "#B15C21", "line_pattern": "dash", "stroke_width_pt": 2.0}, + "iteration": {"stroke_color": "#B15C21", "line_pattern": "dash", "stroke_width_pt": 2.0}, + "iterate": {"stroke_color": "#B15C21", "line_pattern": "dash", "stroke_width_pt": 2.0}, + "return_flow": {"stroke_color": "#B15C21", "line_pattern": "dash", "stroke_width_pt": 2.0}, + "branch": {"stroke_color": "#62529A", "stroke_width_pt": 1.8}, + "conditioning": {"stroke_color": "#1F756F", "line_pattern": "dash", "stroke_width_pt": 1.7}, + "feature_flow": {"stroke_color": "#2A63A1", "stroke_width_pt": 1.8}, + "data_flow": {"stroke_color": "#506F81", "stroke_width_pt": 1.7}, +} + + +def _ratio_value(value: str | float | int | None) -> float: + if isinstance(value, (float, int)): + return max(0.5, min(3.0, float(value))) + text = str(value or "16:9").strip().lower().replace("x", ":") + try: + if ":" in text: + left, right = text.split(":", 1) + return max(0.5, min(3.0, float(left) / max(float(right), 0.001))) + return max(0.5, min(3.0, float(text))) + except (TypeError, ValueError, ZeroDivisionError): + return 16 / 9 + + +def _font_size(label: str, bbox: dict[str, Any]) -> float: + width = float(bbox.get("w") or 0.12) + longest_token = max((len(token) for token in label.replace("/", " ").replace("-", " ").split()), default=0) + if longest_token >= 13: + return 10.5 + if len(label) >= 30 or width < 0.12: + return 12.0 + if len(label) >= 24: + return 12.0 + if len(label) >= 20: + return 13.0 + return 16.0 + + +def _relation_label_bbox(path: list[list[float]], label: str) -> dict[str, float] | None: + if len(path) < 2: + return None + segments: list[tuple[list[float], list[float], float]] = [] + total = 0.0 + for start, end in zip(path[:-1], path[1:]): + length = ((float(end[0]) - float(start[0])) ** 2 + (float(end[1]) - float(start[1])) ** 2) ** 0.5 + segments.append((start, end, length)) + total += length + if total <= 0: + return None + halfway = total / 2 + traversed = 0.0 + midpoint = path[0] + for start, end, length in segments: + if traversed + length >= halfway: + fraction = (halfway - traversed) / max(length, 1e-9) + midpoint = [ + float(start[0]) + (float(end[0]) - float(start[0])) * fraction, + float(start[1]) + (float(end[1]) - float(start[1])) * fraction, + ] + break + traversed += length + width = max(0.07, min(0.13, 0.052 + len(label) * 0.006)) + x = max(0.01, min(0.99 - width, float(midpoint[0]) - width / 2)) + y = max(0.01, min(0.95, float(midpoint[1]) - 0.052)) + return {"x": round(x, 6), "y": round(y, 6), "w": round(width, 6), "h": 0.038} + + +def build_semantic_figure_program( + specification: dict[str, Any], + *, + aspect_ratio: str | float = "16:9", + title: str | None = None, + show_title: bool = False, + semantic_plan: dict[str, Any] | None = None, +) -> dict[str, Any]: + semantic_plan = semantic_plan if isinstance(semantic_plan, dict) and semantic_plan.get("applied") else compile_semantic_blueprint(specification) + if not semantic_plan.get("applied"): + raise ValueError(f"Cannot compile editable PPT semantic blueprint: {semantic_plan.get('reason') or 'not_applicable'}") + + ratio = _ratio_value(aspect_ratio) + height_in = 7.5 + width_in = round(height_in * ratio, 3) + semantic_nodes = [] + for node in semantic_plan["nodes"]: + style = dict(_NODE_STYLES.get(str(node.get("field") or "modules"), _NODE_STYLES["modules"])) + role = str(node.get("role") or "").casefold() + if "data_source" in role or role in {"dataset", "source dataset"}: + style = {"fill_color": "#F8F4EC", "stroke_color": "#896538", "text_color": "#684B28"} + elif any(term in role for term in ("shared", "joint", "fusion")): + style = {"fill_color": "#F4F1FA", "stroke_color": "#62529A", "text_color": "#4B3F78"} + label = str(node.get("label") or node["id"]) + semantic_nodes.append({ + "id": node["id"], + "label": label, + "field": node.get("field"), + "role": node.get("role"), + "bbox_percent": node["bbox_percent"], + "shape_kind": "rounded_rect", + "stroke_width_pt": 1.6, + "font_size_pt": _font_size(label, node["bbox_percent"]), + "bold": True, + "editable_in": "pptx", + "z_index": 30, + **style, + }) + + arrows = [] + labels = [] + for connector in semantic_plan["connectors"]: + relation_type = str(connector.get("type") or "data_flow") + style = dict(_RELATION_STYLES.get(relation_type, _RELATION_STYLES["data_flow"])) + arrows.append({ + "id": connector["id"], + "source": connector["source"], + "target": connector["target"], + "source_id": connector["source"], + "target_id": connector["target"], + "type": relation_type, + "semantic_role": relation_type, + "control_kind": "dashed_loop" if style.get("line_pattern") == "dash" else "elbow_connector", + "route_style": connector.get("route_style"), + "path_percent": connector.get("path_percent", []), + "line_cap": "round", + "arrowhead_size": "sm", + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + **style, + }) + relation_label = str(connector.get("label") or "").strip() + label_bbox = _relation_label_bbox(connector.get("path_percent", []), relation_label) if relation_label else None + if relation_label and label_bbox: + labels.append({ + "id": f"label_{connector['id']}", + "text": relation_label, + "target_id": connector["id"], + "role": "relation_label", + "bbox_percent": label_bbox, + "font_size_pt": 10, + "bold": False, + "color_hex": style["stroke_color"], + "editable_in": "pptx", + }) + + program: dict[str, Any] = { + "summary": "Editable PowerPoint figure compiled directly from the paper semantic contract.", + "canvas": {"width_in": width_in, "height_in": height_in, "ratio": round(ratio, 6), "background": "#FFFFFF"}, + "style": { + "palette": ["#2A63A1", "#1F756F", "#62529A", "#B15C21", "#427E50"], + "arrow_weight_pt": 1.7, + "font_family": "Arial", + }, + "paper_brief": {"title_guess": title}, + "panels": [], + "cards": [], + "slots": [], + "assets": [], + "semantic_nodes": semantic_nodes, + "arrows": arrows, + "labels": labels, + "groups": [], + "export_targets": [{"type": "pptx", "path": "editable_composition.pptx", "role": "main_editable_source"}], + "semantic_plan": semantic_plan, + } + if title and show_title: + program["title_block"] = { + "title": title, + "subtitle": "Editable paper framework", + "bbox_percent": {"x": 0.04, "y": 0.015, "w": 0.92, "h": 0.08}, + "title_font_size": 20, + "subtitle_font_size": 9, + } + return program + + +def _overlay_header_bbox(bbox: dict[str, Any]) -> dict[str, float]: + x = float(bbox.get("x") or 0.0) + y = float(bbox.get("y") or 0.0) + w = float(bbox.get("w") or 0.0) + h = float(bbox.get("h") or 0.0) + inset = min(0.004, w * 0.025, h * 0.025) + header_h = min(h * 0.42, max(0.04, h * 0.30)) + return { + "x": round(x + inset, 6), + "y": round(y + inset, 6), + "w": round(max(0.01, w - inset * 2), 6), + "h": round(max(0.025, header_h - inset), 6), + } + + +def build_visual_substrate_figure_program( + specification: dict[str, Any], + substrate_image: str | Path, + *, + aspect_ratio: str | float = "16:9", + title: str | None = None, + show_title: bool = False, + semantic_plan: dict[str, Any] | None = None, + overlay_spec: dict[str, Any] | None = None, +) -> dict[str, Any]: + substrate = Path(substrate_image).resolve() + if not substrate.exists(): + raise FileNotFoundError(f"Visual substrate image does not exist: {substrate}") + program = build_semantic_figure_program( + specification, + aspect_ratio=aspect_ratio, + title=title, + show_title=show_title, + semantic_plan=semantic_plan, + ) + exact_labels = { + str(item.get("target_id")): str(item.get("text") or "").strip() + for item in (overlay_spec or {}).get("labels", []) + if isinstance(item, dict) and item.get("target_id") and str(item.get("text") or "").strip() + } + for node in program["semantic_nodes"]: + if exact_labels: + target_id = str(node.get("id") or "") + if target_id not in exact_labels: + raise ValueError(f"Editable overlay label is missing for semantic node: {target_id}") + node["label"] = exact_labels[target_id] + node["fill_transparency"] = 1.0 + node["label_bbox_percent"] = _overlay_header_bbox(node["bbox_percent"]) + node["label_fill_color"] = node.get("fill_color") + node["label_stroke_color"] = "#FFFFFF" + node["label_stroke_width_pt"] = 0.5 + node["render_policy"] = "transparent_editable_node_outline_plus_editable_header" + requested_connectors = { + (str(item.get("source") or ""), str(item.get("target") or ""), str(item.get("type") or "data_flow")) + for item in (overlay_spec or {}).get("connectors", []) + if isinstance(item, dict) and item.get("source") and item.get("target") + } + compiled_connectors = { + (str(item.get("source") or ""), str(item.get("target") or ""), str(item.get("type") or "data_flow")) + for item in program["arrows"] + } + if requested_connectors and requested_connectors != compiled_connectors: + missing = sorted(requested_connectors - compiled_connectors) + extra = sorted(compiled_connectors - requested_connectors) + raise ValueError(f"Editable overlay connector contract mismatch; missing={missing}, extra={extra}") + for arrow in program["arrows"]: + arrow["halo_width_pt"] = max(3.2, float(arrow.get("stroke_width_pt") or 1.7) + 1.3) + arrow["halo_color"] = "#FFFFFF" + arrow["routing_algorithm"] = program["semantic_plan"].get("route_quality", {}).get("routing_algorithm") + arrow["route_generation_status"] = "semantic_plan_exact_path" + program.update({ + "summary": "Image2 visual substrate combined with native editable PowerPoint labels, node outlines, and directed connectors.", + "visual_substrate_mode": True, + "background_image": { + "path": str(substrate), + "name": "RFS Image2 Visual Substrate", + "role": "image2_visual_substrate", + "contains_scientific_text": False, + "contains_connectors": False, + "fit_policy": "contain_no_crop", + }, + "assets": [{ + "id": "image2_visual_substrate", + "path": str(substrate), + "role": "visual_substrate", + "editable_in": "pptx_picture", + "contains_scientific_text": False, + "contains_connectors": False, + "fit_policy": "contain_no_crop", + }], + }) + return program + + +def compile_semantic_ppt( + specification: dict[str, Any], + out: str | Path, + *, + aspect_ratio: str | float = "16:9", + title: str | None = None, + show_title: bool = False, +) -> dict[str, Any]: + root = Path(out) + root.mkdir(parents=True, exist_ok=True) + program = build_semantic_figure_program(specification, aspect_ratio=aspect_ratio, title=title, show_title=show_title) + write_json(root / "figure_program.json", program) + pptx_path = compile_ppt(program, root) + report = { + "summary": "Paper semantic contract compiled to native editable PowerPoint shapes, text, and connectors.", + "ok": pptx_path.exists(), + "pptx": str(pptx_path), + "figure_program": str(root / "figure_program.json"), + "node_count": len(program["semantic_nodes"]), + "connector_count": len(program["arrows"]), + "relation_label_count": len(program["labels"]), + "topology": program["semantic_plan"].get("topology"), + "editable_layers": ["native_shapes", "native_text", "native_connectors"], + } + write_json(root / "semantic_ppt_report.json", report) + return report + + +def compile_visual_substrate_ppt( + specification: dict[str, Any], + substrate_image: str | Path, + out: str | Path, + *, + aspect_ratio: str | float = "16:9", + title: str | None = None, + show_title: bool = False, + semantic_plan: dict[str, Any] | None = None, + overlay_spec: dict[str, Any] | None = None, +) -> dict[str, Any]: + root = Path(out) + root.mkdir(parents=True, exist_ok=True) + program = build_visual_substrate_figure_program( + specification, + substrate_image, + aspect_ratio=aspect_ratio, + title=title, + show_title=show_title, + semantic_plan=semantic_plan, + overlay_spec=overlay_spec, + ) + write_json(root / "figure_program.json", program) + pptx_path = compile_ppt(program, root) + composition = root / "composition_quality_report.json" + report = { + "summary": "Image2 visual substrate compiled beneath native editable PowerPoint terminology and connector geometry.", + "ok": pptx_path.exists(), + "pptx": str(pptx_path), + "figure_program": str(root / "figure_program.json"), + "composition_quality_report": str(composition), + "visual_substrate": str(Path(substrate_image).resolve()), + "node_count": len(program["semantic_nodes"]), + "connector_count": len(program["arrows"]), + "relation_label_count": len(program["labels"]), + "topology": program["semantic_plan"].get("topology"), + "route_quality": program["semantic_plan"].get("route_quality", {}), + "editable_layers": ["native_node_outlines", "native_header_labels", "native_connectors", "native_relation_labels"], + "visual_layer": "single_image2_substrate_picture", + "scientific_text_baked_into_visual_layer": False, + "connectors_baked_into_visual_layer": False, + } + write_json(root / "editable_overlay_report.json", report) + return report diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index e8d10d0..04bd712 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -4,7 +4,9 @@ import json import os import shutil +import statistics import time +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Any, Callable from urllib.parse import urlsplit, urlunsplit @@ -15,7 +17,9 @@ from ..asset_generator import _call_gemini from ..utils import ensure_dir, write_json, write_text from .critics import aggregate_reviews, review_candidate -from .planner import compile_image_prompt +from .overlay_renderer import compose_semantic_overlay, normalize_visual_substrate_geometry +from .planner import collect_visual_relations, compile_image_prompt +from .review_contract import build_repair_plan def _parse_ratio(value: str) -> float: @@ -36,6 +40,15 @@ def _image_size(aspect_ratio: str) -> str: return "1024x1024" +def native_image2_aspect_ratio(aspect_ratio: str) -> str: + ratio = _parse_ratio(aspect_ratio) + if ratio >= 1.25: + return "3:2" + if ratio <= 0.80: + return "2:3" + return "1:1" + + def _font(size: int) -> ImageFont.ImageFont: for path in [Path(r"C:\Windows\Fonts\msyh.ttc"), Path(r"C:\Windows\Fonts\arial.ttf")]: if path.exists(): @@ -92,6 +105,105 @@ def _image2_edit_url() -> str: return f"{base}/images/edits" +def _candidate_failure_modes(record: dict) -> list[str]: + review = record.get("review", {}) if isinstance(record.get("review"), dict) else {} + modes = [] + ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} + scientific = review.get("scientific", {}) if isinstance(review.get("scientific"), dict) else {} + paper_alignment = review.get("paper_alignment", {}) if isinstance(review.get("paper_alignment"), dict) else {} + topology = review.get("topology", {}) if isinstance(review.get("topology"), dict) else {} + template = review.get("template", {}) if isinstance(review.get("template"), dict) else {} + aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} + substrate_geometry = review.get("substrate_geometry", {}) if isinstance(review.get("substrate_geometry"), dict) else {} + if substrate_geometry and not substrate_geometry.get("passed", False): + modes.append("substrate_geometry") + unexpected_labels = ocr.get("unsupported_unexpected_labels") if "unsupported_unexpected_labels" in ocr else ocr.get("unexpected_labels") + if ocr.get("missing_labels") or ocr.get("misspelled_labels") or ocr.get("duplicate_labels") or unexpected_labels: + modes.append("label_accuracy") + if scientific.get("missing_modules"): + modes.append("missing_modules") + if scientific.get("missing_relations") or topology.get("missing_relations"): + modes.append("missing_relations") + if scientific.get("reversed_relations") or topology.get("reversed_relations"): + modes.append("reversed_relations") + if topology.get("bypassed_relations"): + modes.append("bypassed_relations") + if scientific.get("invented_items") or topology.get("invented_relations"): + modes.append("invented_content") + if scientific.get("role_mismatches") or scientific.get("containment_mismatches"): + modes.append("semantic_role_mismatch") + if paper_alignment.get("missing_priorities"): + modes.append("paper_missing_priorities") + if paper_alignment.get("evidence_conflicts"): + modes.append("paper_evidence_conflicts") + if paper_alignment.get("unsupported_visual_claims"): + modes.append("unsupported_visual_claims") + if not template.get("passed", False): + modes.append("template_alignment") + if not aesthetic.get("passed", False): + modes.append("aesthetic_quality") + if review.get("vlm_warning") or review.get("topology_warning"): + modes.append("review_provider") + return list(dict.fromkeys(modes or (["unknown_review_failure"] if not record.get("production_pass") else []))) + + +def _build_stability_report(records: list[dict], failures: list[dict], requested_candidates: int, repair_source: Path | None) -> dict: + independent = [ + item for item in records + if str(item.get("candidate_id") or "").startswith("candidate_") and not item.get("repair_source") + ] + independent_failures = [ + item for item in failures if str(item.get("candidate_id") or "").startswith("candidate_") + ] + seed_count = 0 if repair_source else max(int(requested_candidates), len(independent) + len(independent_failures)) + score_by_id = {str(item.get("candidate_id")): float(item.get("score") or 0.0) for item in independent} + scores = [score_by_id.get(f"candidate_{index:02d}", 0.0) for index in range(1, seed_count + 1)] + pass_count = sum(bool(item.get("production_pass")) for item in independent) + pass_rate = round(pass_count / seed_count, 4) if seed_count else 0.0 + mean_score = round(statistics.fmean(scores), 4) if scores else 0.0 + worst_score = round(min(scores), 4) if scores else 0.0 + deviation = round(statistics.pstdev(scores), 4) if len(scores) >= 2 else 0.0 + failure_modes: dict[str, int] = {} + candidates = [] + for item in independent: + modes = _candidate_failure_modes(item) + for mode in modes: + failure_modes[mode] = failure_modes.get(mode, 0) + 1 + candidates.append({ + "candidate_id": item.get("candidate_id"), + "score": round(float(item.get("score") or 0.0), 4), + "production_pass": bool(item.get("production_pass")), + "failure_modes": modes, + }) + for item in independent_failures: + failure_mode = "substrate_geometry" if item.get("failure_stage") == "substrate_geometry" else "generation_provider" if item.get("provider_failure") else "generation_or_review" + failure_modes[failure_mode] = failure_modes.get(failure_mode, 0) + 1 + candidates.append({ + "candidate_id": item.get("candidate_id"), + "score": 0.0, + "production_pass": False, + "failure_modes": [failure_mode], + "error": str(item.get("error") or "candidate generation failed"), + }) + stable = bool(seed_count >= 2 and pass_rate >= 0.8 and worst_score >= 0.8) + status = "stable" if stable else "unstable" if seed_count >= 2 else "insufficient_samples" + return { + "summary": "Cross-candidate production stability for independent Image2 generations.", + "status": status, + "stable": stable, + "seed_count": seed_count, + "completed_seed_count": len(independent), + "production_pass_count": pass_count, + "production_pass_rate": pass_rate, + "mean_score": mean_score, + "worst_case_score": worst_score, + "standard_deviation": deviation, + "minimum_recommended_seed_count": 3, + "candidate_scores": sorted(candidates, key=lambda item: str(item.get("candidate_id") or "")), + "failure_mode_counts": dict(sorted(failure_modes.items())), + } + + def _safe_endpoint(value: str) -> str: parts = urlsplit(value) return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) @@ -103,6 +215,7 @@ def _call_image2_edit(source_path: Path, prompt: str, output_path: Path, aspect_ raise RuntimeError("Reference-conditioned Image2 requires API_KEY/GEMINI_API_KEY") endpoint = _image2_edit_url() resolved_model = _image2_model_name(model) + request_timeout = max(30, int(os.getenv("RFS_IMAGE2_TIMEOUT", "120"))) last_error: Exception | None = None for attempt in range(max(0, min(5, int(retries))) + 1): try: @@ -112,13 +225,16 @@ def _call_image2_edit(source_path: Path, prompt: str, output_path: Path, aspect_ headers={"Authorization": f"Bearer {api_key}"}, data={"model": resolved_model, "prompt": prompt, "n": "1", "size": _image_size(aspect_ratio)}, files={"image": (source_path.name, handle, "image/png")}, - timeout=360, + timeout=request_timeout, ) if response.status_code == 429 or response.status_code >= 500: raise RuntimeError(f"Image2 edit returned HTTP {response.status_code}") response.raise_for_status() _write_openai_image(response.json(), output_path) return {"mode": "edit", "model": resolved_model, "endpoint": _safe_endpoint(endpoint), "source": str(source_path), "api_key_present": True} + except requests.Timeout as exc: + last_error = exc + break except Exception as exc: last_error = exc if attempt >= max(0, min(5, int(retries))): @@ -139,7 +255,31 @@ def _generate_one(prompt: str, plan: dict, blueprint: Path, path: Path, aspect_r return _call_image2_edit(blueprint, prompt, path, aspect_ratio, model, retries) -def _repair_prompt(review: dict, base_prompt: str) -> str: +def _repair_prompt( + review: dict, + repair_plan: dict, + base_prompt: str, + plan: dict, + *, + deterministic_overlay: bool = False, +) -> str: + connector_checklist = "\n".join( + f"{index}. {item['source_label']} -> {item['target_label']} [{item['type']}]" + for index, item in enumerate(collect_visual_relations(plan.get("figure_specification", {})), start=1) + ) + if deterministic_overlay: + ownership_rules = """Deterministic overlay ownership: +- Do not add, remove, rewrite, move, or repair any text, letter, number, formula, arrow, connector, arrowhead, loop, or line. +- Exact labels and directed relations are compiled after this edit from overlay_spec.json and semantic_plan.path_percent. +- Repair only the image-rich visual substrate inside the existing node regions. +- Keep every reserved header band and connector corridor visually clear.""" + connector_contract = "The overlay compiler will restore the exact connector contract after this visual-only repair." + else: + ownership_rules = "Preserve all correct scientific text and connector geometry while applying the localized repair." + connector_contract = f"""Mandatory directed connector checklist after repair: +{connector_checklist} + +Preserve every currently correct connector. Add or redirect only the connectors named by the repair findings. The repaired image must contain every checklist edge exactly in the stated direction. Never draw a shortcut that bypasses any declared intermediate module.""" return f""" # Summary @@ -149,7 +289,7 @@ def _repair_prompt(review: dict, base_prompt: str) -> str: {json.dumps(review.get('preserve', []), ensure_ascii=False)} Repair: -{json.dumps(review.get('repair', []), ensure_ascii=False)} +{json.dumps(repair_plan.get('items', []), ensure_ascii=False, indent=2)} Remove: {json.dumps(review.get('remove', []), ensure_ascii=False)} @@ -160,11 +300,65 @@ def _repair_prompt(review: dict, base_prompt: str) -> str: Hard errors: {json.dumps(review.get('hard_errors', []), ensure_ascii=False)} +{ownership_rules} + +{connector_contract} + Original scientific and visual contract: {base_prompt} """.strip() +def _failure_metadata(error: Exception) -> dict[str, Any]: + error_text = str(error) + normalized = error_text.casefold() + overlay_failure = "deterministic overlay compilation failed" in normalized + geometry_failure = "visual substrate geometry normalization failed" in normalized + provider_timeout = not overlay_failure and not geometry_failure and any(token in normalized for token in ("timed out", "read timeout", "connect timeout")) + provider_failure = not overlay_failure and not geometry_failure and ( + provider_timeout or any(token in normalized for token in ("http 429", "http 5", "provider", "connection")) + ) + return { + "error": error_text, + "failure_stage": "substrate_geometry" if geometry_failure else "overlay_compilation" if overlay_failure else "image_provider" if provider_failure else "generation_or_review", + "provider_failure": provider_failure, + "provider_timeout": provider_timeout, + } + + +def _review_regressions(source: dict, candidate: dict, tolerance: float = 0.001) -> list[str]: + regressions = [] + for section in ("basic", "ocr", "scientific", "paper_alignment", "topology", "template", "aesthetic"): + previous = source.get(section, {}) if isinstance(source.get(section), dict) else {} + current = candidate.get(section, {}) if isinstance(candidate.get(section), dict) else {} + if previous.get("passed") and not current.get("passed"): + regressions.append(f"{section} changed from pass to fail") + if section in {"ocr", "scientific", "paper_alignment", "topology"}: + previous_score = float(previous.get("score") or 0.0) + current_score = float(current.get("score") or 0.0) + if current_score + tolerance < previous_score: + regressions.append(f"{section} score decreased from {previous_score:.4f} to {current_score:.4f}") + if len(candidate.get("hard_errors", [])) > len(source.get("hard_errors", [])): + regressions.append("hard error count increased") + return regressions + + +def _strictly_improves(source: dict, candidate: dict) -> tuple[bool, list[str]]: + regressions = _review_regressions(source, candidate) + if regressions: + return False, regressions + source_score = float(source.get("overall_score") or 0.0) + candidate_score = float(candidate.get("overall_score") or 0.0) + source_hard = len(source.get("hard_errors", [])) + candidate_hard = len(candidate.get("hard_errors", [])) + improved = bool( + candidate.get("production_pass") and not source.get("production_pass") + or candidate_score > source_score + 0.001 + or candidate_hard < source_hard + ) + return improved, [] if improved else ["candidate did not strictly improve score, pass status, or hard-error count"] + + def generate_and_select( plan: dict, preferences: dict, @@ -177,68 +371,481 @@ def generate_and_select( image_retries: int = 2, review_mode: str = "vlm", review_model: str | None = None, - repair_rounds: int = 1, + repair_rounds: int = 3, + repair_source: str | Path | None = None, ocr_engine: str = "auto", ocr_lang: str = "en_ch", ocr_adapter: Callable | None = None, critic_adapter: Callable | None = None, + topology_adapter: Callable | None = None, + resume_candidates: bool = False, + require_visual_enrichment: bool = True, + generation_blueprint_path: str | Path | None = None, + overlay_spec: dict[str, Any] | None = None, + deterministic_overlay: bool = False, ) -> dict: root = ensure_dir(out_dir) candidate_dir = ensure_dir(root / "candidates") prompt_dir = ensure_dir(root / "prompts") + review_dir = ensure_dir(root / "reviews") + overlay_dir = ensure_dir(root / "overlays") + geometry_dir = ensure_dir(root / "geometry") blueprint = Path(blueprint_path) + generation_blueprint = Path(generation_blueprint_path) if generation_blueprint_path else blueprint + semantic_plan = selected_template.get("semantic_plan", {}) if isinstance(selected_template.get("semantic_plan"), dict) else {} + overlay_enabled = bool(deterministic_overlay and semantic_plan.get("applied") and isinstance(overlay_spec, dict)) aspect_ratio = str(preferences.get("aspect_ratio") or "16:9") count = max(1, min(4, int(candidates))) records: list[dict] = [] failures: list[dict] = [] requests_manifest: list[dict] = [] - for index in range(1, count + 1): + worker_default = min(2, count) + try: + candidate_workers = max(1, min(count, int(os.getenv("RFS_PAPER_IMAGE_WORKERS", str(worker_default))))) + except ValueError: + candidate_workers = worker_default + acceptable_aspect_ratios = [ + str(value) + for value in (preferences.get("requested_aspect_ratio"), preferences.get("aspect_ratio")) + if value and str(value) != "auto" + ] + + try: + review_attempt_limit = max(1, min(3, int(os.getenv("RFS_PAPER_IMAGE_REVIEW_ATTEMPTS", "2")))) + except ValueError: + review_attempt_limit = 2 + + def review_is_inconclusive(review: dict[str, Any]) -> bool: + return bool( + review_mode == "vlm" + and critic_adapter is None + and ( + review.get("vlm_warning") + or review.get("topology_warning") + or "Production critic did not return paper-evidence alignment" in review.get("hard_errors", []) + ) + ) + + def run_review(path: Path) -> dict[str, Any]: + attempts = [] + final_review: dict[str, Any] = {} + for attempt in range(1, review_attempt_limit + 1): + final_review = review_candidate( + path, + blueprint, + plan, + selected_template, + mode=review_mode, + model=review_model, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + ocr_adapter=ocr_adapter, + critic_adapter=critic_adapter, + topology_adapter=topology_adapter, + acceptable_aspect_ratios=acceptable_aspect_ratios, + require_visual_enrichment=asset_mode == "image2" and require_visual_enrichment, + ) + inconclusive = review_is_inconclusive(final_review) + attempts.append({ + "attempt": attempt, + "inconclusive": inconclusive, + "vlm_warning": final_review.get("vlm_warning"), + "topology_warning": final_review.get("topology_warning"), + }) + if not inconclusive or attempt >= review_attempt_limit: + break + time.sleep(min(4.0, float(attempt))) + final_review["review_attempts"] = len(attempts) + final_review["review_attempt_history"] = attempts + return final_review + + def apply_geometry_gate(review: dict[str, Any], geometry_report: dict[str, Any] | None) -> dict[str, Any]: + if not overlay_enabled: + return review + passed = bool( + geometry_report + and geometry_report.get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("unique_node_card_assignment") + and not geometry_report.get("post_normalization_alignment", {}).get("semantic_crop_used") + ) + review["substrate_geometry"] = { + "summary": "Hard gate for exact one-to-one visual-card normalization before semantic overlay.", + "passed": passed, + "expected_node_count": geometry_report.get("expected_node_count") if geometry_report else None, + "detected_card_count": geometry_report.get("detected_card_count") if geometry_report else None, + "matched_node_count": geometry_report.get("matched_node_count") if geometry_report else None, + "post_normalization_alignment": geometry_report.get("post_normalization_alignment", {}) if geometry_report else {}, + } + if not passed: + hard_errors = list(review.get("hard_errors", [])) + message = "Visual substrate geometry did not pass exact one-to-one normalization" + if message not in hard_errors: + hard_errors.append(message) + review["hard_errors"] = hard_errors + review["production_pass"] = False + return review + + repair_source_path = Path(repair_source).resolve() if repair_source else None + if repair_source_path and not repair_source_path.exists(): + raise FileNotFoundError(f"Repair source image does not exist: {repair_source_path}") + + def compile_visual_candidate( + raw_path: Path, + substrate_path: Path, + composed_path: Path, + geometry_report_path: Path, + overlay_report_path: Path, + ) -> tuple[dict[str, Any], dict[str, Any]]: + try: + geometry_report = normalize_visual_substrate_geometry( + raw_path, + generation_blueprint, + semantic_plan, + substrate_path, + report_path=geometry_report_path, + ) + except Exception as exc: + raise RuntimeError(f"Visual substrate geometry normalization failed: {exc}") from exc + try: + overlay_report = compose_semantic_overlay( + substrate_path, + semantic_plan, + overlay_spec or {}, + composed_path, + report_path=overlay_report_path, + ) + except Exception as exc: + raise RuntimeError(f"Deterministic overlay compilation failed: {exc}") from exc + return geometry_report, overlay_report + + def generate_candidate(index: int) -> dict[str, Any]: candidate_id = f"candidate_{index:02d}" path = candidate_dir / f"{candidate_id}.png" - prompt = compile_image_prompt(plan, preferences, candidate_variant=index, selected_template=selected_template) + raw_path = candidate_dir / f"{candidate_id}_raw.png" if overlay_enabled else path + substrate_path = candidate_dir / f"{candidate_id}_substrate.png" if overlay_enabled else path + geometry_report_path = geometry_dir / f"{candidate_id}_geometry.json" + overlay_report_path = overlay_dir / f"{candidate_id}_overlay.json" + prompt = compile_image_prompt(plan, preferences, candidate_variant=index, selected_template=selected_template, deterministic_overlay=overlay_enabled) prompt_path = prompt_dir / f"{candidate_id}_prompt.txt" write_text(prompt_path, prompt) + started = time.monotonic() try: - generation = _generate_one(prompt, plan, blueprint, path, aspect_ratio, asset_mode, image_model, image_retries) - requests_manifest.append({"candidate_id": candidate_id, **generation}) - review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter) - records.append({"candidate_id": candidate_id, "path": str(path), "prompt_path": str(prompt_path), "generation": generation, "review": review, "production_pass": bool(asset_mode == "image2" and review["production_pass"]), "score": review["overall_score"]}) + overlay_report = None + geometry_report = None + record_raw_path: Path | None = raw_path if overlay_enabled else None + record_substrate_path: Path | None = substrate_path if overlay_enabled else None + if resume_candidates and raw_path.exists(): + generation = {"mode": "existing_candidate_resume", "source": str(raw_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + generation_seconds = 0.0 + elif resume_candidates and path.exists(): + generation = {"mode": "existing_candidate_resume_legacy_composed", "source": str(path), "overlay_assumed_precomposed": overlay_enabled, "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + raw_path = path + record_raw_path = None + record_substrate_path = None + geometry_report = { + "summary": "Legacy composed-only candidate cannot prove visual-substrate geometry alignment.", + "passed": False, + "failure_stage": "substrate_geometry", + "reason": "raw_visual_substrate_missing", + } + if overlay_report_path.exists(): + overlay_report = json.loads(overlay_report_path.read_text(encoding="utf-8")) + generation_seconds = 0.0 + else: + generation_started = time.monotonic() + generation = _generate_one(prompt, plan, generation_blueprint, raw_path, aspect_ratio, asset_mode, image_model, image_retries) + generation_seconds = round(time.monotonic() - generation_started, 3) + if overlay_enabled and raw_path != path: + geometry_report, overlay_report = compile_visual_candidate(raw_path, substrate_path, path, geometry_report_path, overlay_report_path) + generation = { + **generation, + "raw_path": str(raw_path), + "visual_substrate_path": str(substrate_path), + "composed_path": str(path), + "geometry_alignment_report": str(geometry_report_path), + "overlay_report": str(overlay_report_path), + "deterministic_overlay": True, + } + review_started = time.monotonic() + review = run_review(path) + review = apply_geometry_gate(review, geometry_report) + review_seconds = round(time.monotonic() - review_started, 3) + return { + "record": { + "candidate_id": candidate_id, + "path": str(path), + "raw_path": str(record_raw_path) if record_raw_path else None, + "visual_substrate_path": str(record_substrate_path) if record_substrate_path else None, + "geometry_alignment_report": geometry_report, + "overlay_report": overlay_report, + "prompt_path": str(prompt_path), + "generation": generation, + "review": review, + "production_pass": bool(asset_mode == "image2" and review["production_pass"] and (not overlay_enabled or geometry_report and geometry_report.get("passed"))), + "score": review["overall_score"], + "timings": { + "generation_seconds": generation_seconds, + "review_seconds": review_seconds, + "total_seconds": round(time.monotonic() - started, 3), + }, + }, + "request": {"candidate_id": candidate_id, "generation_seconds": generation_seconds, **generation}, + } except Exception as exc: - failures.append({"candidate_id": candidate_id, "error": str(exc), "prompt_path": str(prompt_path)}) + return {"failure": {"candidate_id": candidate_id, "prompt_path": str(prompt_path), "elapsed_seconds": round(time.monotonic() - started, 3), **_failure_metadata(exc)}} + + if repair_source_path: + candidate_workers = 1 + candidate_id = "source_candidate" + path = candidate_dir / f"{candidate_id}{repair_source_path.suffix.lower() or '.png'}" + if path.resolve() != repair_source_path: + shutil.copyfile(repair_source_path, path) + prompt = compile_image_prompt(plan, preferences, candidate_variant=1, selected_template=selected_template) + prompt_path = prompt_dir / f"{candidate_id}_prompt.txt" + write_text(prompt_path, prompt) + review_started = time.monotonic() + review = run_review(path) + review_seconds = round(time.monotonic() - review_started, 3) + generation = {"mode": "existing_candidate", "source": str(repair_source_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + records.append({ + "candidate_id": candidate_id, + "path": str(path), + "raw_path": None, + "prompt_path": str(prompt_path), + "generation": generation, + "review": review, + "production_pass": bool(asset_mode == "image2" and review["production_pass"]), + "score": review["overall_score"], + "timings": {"generation_seconds": 0.0, "review_seconds": review_seconds, "total_seconds": review_seconds}, + }) + requests_manifest.append({"candidate_id": candidate_id, "reused_source": True, **generation}) + else: + with ThreadPoolExecutor(max_workers=candidate_workers) as executor: + futures = [executor.submit(generate_candidate, index) for index in range(1, count + 1)] + for future in as_completed(futures): + result = future.result() + if result.get("record"): + records.append(result["record"]) + requests_manifest.append(result["request"]) + elif result.get("failure"): + failures.append(result["failure"]) + replacement_budget = 0 + if asset_mode == "image2" and count >= 3: + try: + replacement_budget = max(0, min(2, int(os.getenv("RFS_STABILITY_REPLACEMENT_RETRIES", "1")))) + except ValueError: + replacement_budget = 1 + for replacement_attempt in range(1, replacement_budget + 1): + retryable = [ + item for item in failures + if any(token in str(item.get("error") or "").casefold() for token in ("timed out", "timeout", "http 429", "http 5", "provider")) + ] + if not retryable: + break + for failure in retryable: + try: + candidate_index = int(str(failure.get("candidate_id") or "").rsplit("_", 1)[-1]) + except ValueError: + continue + result = generate_candidate(candidate_index) + failures.remove(failure) + if result.get("record"): + result["record"]["provider_replacement_attempt"] = replacement_attempt + result["request"]["provider_replacement_attempt"] = replacement_attempt + records.append(result["record"]) + requests_manifest.append(result["request"]) + else: + replacement_failure = result.get("failure", failure) + replacement_failure["provider_replacement_attempts"] = replacement_attempt + failures.append(replacement_failure) + records.sort(key=lambda item: item["candidate_id"]) + requests_manifest.sort(key=lambda item: item["candidate_id"]) + failures.sort(key=lambda item: item["candidate_id"]) if not records: - manifest = {"summary": "Image2 request manifest with no successful candidate.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _image2_edit_url() if asset_mode == "image2" and os.getenv("API_BASE") else None, "requests": requests_manifest, "failures": failures} + manifest = {"summary": "Image2 request manifest with no successful candidate.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _image2_edit_url() if asset_mode == "image2" and os.getenv("API_BASE") else None, "candidate_workers": candidate_workers, "requests": requests_manifest, "failures": failures} write_json(root / "image2_request_manifest.json", manifest) raise RuntimeError(f"All image candidates failed: {failures}") passing = [item for item in records if item["production_pass"]] repair_records = [] - for repair_index in range(max(0, min(1, int(repair_rounds)))): - if passing or asset_mode != "image2": + repair_history = [] + no_improvement_streak = 0 + repair_stop_reason = "not_requested" if int(repair_rounds) <= 0 else "pending" + max_repair_rounds = max(0, min(4, int(repair_rounds))) + for repair_index in range(max_repair_rounds): + if passing: + repair_stop_reason = "all_gates_passed" + break + if asset_mode != "image2": + repair_stop_reason = "image2_required" break - source = max(records, key=lambda item: item["score"]) + accepted_records = [item for item in records if item.get("accepted", True)] + source = max(accepted_records, key=lambda item: item["score"]) repaired_id = f"repair_{repair_index + 1:02d}" repaired_path = candidate_dir / f"{repaired_id}.png" - base_prompt = Path(source["prompt_path"]).read_text(encoding="utf-8") - prompt = _repair_prompt(source["review"], base_prompt) + repaired_raw_path = candidate_dir / f"{repaired_id}_raw.png" if overlay_enabled else repaired_path + repaired_substrate_path = candidate_dir / f"{repaired_id}_substrate.png" if overlay_enabled else repaired_path + geometry_report_path = geometry_dir / f"{repaired_id}_geometry.json" + overlay_report_path = overlay_dir / f"{repaired_id}_overlay.json" prompt_path = prompt_dir / f"{repaired_id}_prompt.txt" + repair_plan_path = prompt_dir / f"{repaired_id}_plan.json" + review_path = review_dir / f"{repaired_id}_review.json" + repair_plan = build_repair_plan(source["review"], source["candidate_id"], repair_index + 1) + if overlay_enabled: + all_items = list(repair_plan.get("items", [])) + overlay_owned = [item for item in all_items if item.get("owner") == "overlay_compiler"] + image_owned = [item for item in all_items if item.get("owner") != "overlay_compiler"] + repair_plan = { + **repair_plan, + "summary": "Ownership-aware repair plan: Image2 edits only the visual substrate; exact text and connectors remain compiler-owned.", + "all_items": all_items, + "overlay_owned_items": overlay_owned, + "overlay_owned_item_count": len(overlay_owned), + "image2_item_count": len(image_owned), + "items": image_owned, + "item_count": len(image_owned), + } + write_json(repair_plan_path, repair_plan) + write_json(root / "repair_plan.json", repair_plan) + if not repair_plan["items"]: + repair_stop_reason = "overlay_recompile_required" if repair_plan.get("overlay_owned_item_count") else "no_actionable_repairs" + break + if overlay_enabled and not source.get("raw_path"): + repair_stop_reason = "overlay_raw_source_missing" + break + base_prompt = Path(source["prompt_path"]).read_text(encoding="utf-8") + prompt = _repair_prompt(source["review"], repair_plan, base_prompt, plan, deterministic_overlay=overlay_enabled) write_text(prompt_path, prompt) + repair_started = time.monotonic() try: - generation = _call_image2_edit(Path(source["path"]), prompt, repaired_path, aspect_ratio, image_model, image_retries) - requests_manifest.append({"candidate_id": repaired_id, "repair_source": source["candidate_id"], **generation}) - review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter) - repaired = {"candidate_id": repaired_id, "path": str(repaired_path), "prompt_path": str(prompt_path), "generation": generation, "review": review, "production_pass": review["production_pass"], "score": review["overall_score"], "repair_source": source["candidate_id"]} + overlay_report = None + geometry_report = None + if overlay_enabled and resume_candidates and repaired_raw_path.exists(): + geometry_report, overlay_report = compile_visual_candidate(repaired_raw_path, repaired_substrate_path, repaired_path, geometry_report_path, overlay_report_path) + generation = {"mode": "existing_repair_reoverlay", "source": str(repaired_raw_path), "raw_path": str(repaired_raw_path), "visual_substrate_path": str(repaired_substrate_path), "composed_path": str(repaired_path), "geometry_alignment_report": str(geometry_report_path), "overlay_report": str(overlay_report_path), "deterministic_overlay": True, "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + generation_seconds = 0.0 + review_started = time.monotonic() + review = run_review(repaired_path) + review = apply_geometry_gate(review, geometry_report) + review_seconds = round(time.monotonic() - review_started, 3) + write_json(review_path, review) + elif resume_candidates and repaired_path.exists() and review_path.exists() and not overlay_enabled: + stored_review = json.loads(review_path.read_text(encoding="utf-8")) + generation_seconds = 0.0 + if review_is_inconclusive(stored_review): + generation = {"mode": "existing_repair_reaudit", "source": str(repaired_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + review_started = time.monotonic() + review = run_review(repaired_path) + review = apply_geometry_gate(review, geometry_report) + review_seconds = round(time.monotonic() - review_started, 3) + write_json(review_path, review) + else: + generation = {"mode": "existing_repair_resume", "source": str(repaired_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + review = stored_review + review_seconds = 0.0 + else: + generation_started = time.monotonic() + source_visual = Path(str(source.get("raw_path") or source["path"])) + generation = _call_image2_edit(source_visual, prompt, repaired_raw_path, aspect_ratio, image_model, image_retries) + generation_seconds = round(time.monotonic() - generation_started, 3) + if overlay_enabled and repaired_raw_path != repaired_path: + geometry_report, overlay_report = compile_visual_candidate(repaired_raw_path, repaired_substrate_path, repaired_path, geometry_report_path, overlay_report_path) + generation = {**generation, "raw_path": str(repaired_raw_path), "visual_substrate_path": str(repaired_substrate_path), "composed_path": str(repaired_path), "geometry_alignment_report": str(geometry_report_path), "overlay_report": str(overlay_report_path), "deterministic_overlay": True} + review_started = time.monotonic() + review = run_review(repaired_path) + review = apply_geometry_gate(review, geometry_report) + review_seconds = round(time.monotonic() - review_started, 3) + write_json(review_path, review) + requests_manifest.append({"candidate_id": repaired_id, "repair_source": source["candidate_id"], "generation_seconds": generation_seconds, **generation}) + accepted, rejection_reasons = _strictly_improves(source["review"], review) + repaired = { + "candidate_id": repaired_id, + "path": str(repaired_path), + "raw_path": str(repaired_raw_path) if overlay_enabled else None, + "visual_substrate_path": str(repaired_substrate_path) if overlay_enabled else None, + "geometry_alignment_report": geometry_report, + "overlay_report": overlay_report, + "prompt_path": str(prompt_path), + "repair_plan_path": str(repair_plan_path), + "review_path": str(review_path), + "generation": generation, + "review": review, + "production_pass": bool(review["production_pass"] and accepted and (not overlay_enabled or geometry_report and geometry_report.get("passed"))), + "raw_production_pass": bool(review["production_pass"]), + "score": review["overall_score"], + "repair_source": source["candidate_id"], + "accepted": accepted, + "rejection_reasons": rejection_reasons, + "timings": {"generation_seconds": generation_seconds, "review_seconds": review_seconds, "total_seconds": round(time.monotonic() - repair_started, 3)}, + } repair_records.append(repaired) records.append(repaired) - if repaired["production_pass"]: - passing.append(repaired) + repair_history.append({ + "round": repair_index + 1, + "candidate_id": repaired_id, + "source_candidate_id": source["candidate_id"], + "source_score": source["score"], + "candidate_score": repaired["score"], + "accepted": accepted, + "production_pass": repaired["production_pass"], + "rejection_reasons": rejection_reasons, + }) + if accepted: + no_improvement_streak = 0 + if repaired["production_pass"]: + passing.append(repaired) + repair_stop_reason = "all_gates_passed" + break + else: + no_improvement_streak += 1 + if no_improvement_streak >= 2: + repair_stop_reason = "two_consecutive_rounds_without_improvement" + break except Exception as exc: - failures.append({"candidate_id": repaired_id, "error": str(exc), "prompt_path": str(prompt_path)}) + metadata = _failure_metadata(exc) + error_text = str(metadata["error"]) + provider_timeout = bool(metadata["provider_timeout"]) + provider_failure = bool(metadata["provider_failure"]) + if not provider_failure: + no_improvement_streak += 1 + failure = {"candidate_id": repaired_id, "prompt_path": str(prompt_path), "repair_source": source["candidate_id"], **metadata} + failures.append(failure) + repair_history.append({"round": repair_index + 1, "candidate_id": repaired_id, "source_candidate_id": source["candidate_id"], "accepted": False, **metadata}) + if provider_timeout: + repair_stop_reason = "provider_timeout" + break + if no_improvement_streak >= 2: + repair_stop_reason = "two_consecutive_rounds_without_improvement" + break + else: + if max_repair_rounds: + repair_stop_reason = "maximum_repair_rounds_reached" + if repair_stop_reason == "pending": + repair_stop_reason = "maximum_repair_rounds_reached" if repair_records else "not_needed" + write_json(root / "repair_history.json", { + "summary": "Accepted/rejected multi-round Image2 repairs with no-regression checks.", + "requested_rounds": int(repair_rounds), + "maximum_rounds": max_repair_rounds, + "completed_rounds": len(repair_history), + "stop_reason": repair_stop_reason, + "rounds": repair_history, + }) + if not (root / "repair_plan.json").exists(): + write_json(root / "repair_plan.json", { + "summary": "No repair plan was required for the selected candidate set.", + "source_candidate_id": None, + "round_index": None, + "item_count": 0, + "preserve": [], + "items": [], + }) aggregates = aggregate_reviews(records) for name, data in aggregates.items(): write_json(root / f"{name}.json", data) - request_manifest = {"summary": "Safe Image2 request manifest; secrets are never recorded.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _safe_endpoint(_image2_edit_url()) if asset_mode == "image2" and (os.getenv("API_BASE") or os.getenv("RFS_IMAGE_EDIT_URL")) else None, "blueprint": str(blueprint), "requests": requests_manifest, "failures": failures} + request_manifest = {"summary": "Safe Image2 request manifest; secrets are never recorded.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _safe_endpoint(_image2_edit_url()) if asset_mode == "image2" and (os.getenv("API_BASE") or os.getenv("RFS_IMAGE_EDIT_URL")) else None, "blueprint": str(blueprint), "generation_blueprint": str(generation_blueprint), "deterministic_overlay": overlay_enabled, "candidate_workers": candidate_workers, "repair_source": str(repair_source_path) if repair_source_path else None, "requests": requests_manifest, "failures": failures} write_json(root / "image2_request_manifest.json", request_manifest) selected_path = None @@ -253,8 +860,10 @@ def generate_and_select( engineering_preview = root / "engineering_preview.png" shutil.copyfile(selected["path"], engineering_preview) - report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "requested_candidates": count, "successful_candidates": len(records), "repair_candidates": len(repair_records), "failures": failures, "candidates": records, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} + stability = _build_stability_report(records, failures, count, repair_source_path) + write_json(root / "stability_report.json", stability) + report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "deterministic_overlay": overlay_enabled, "requested_candidates": 0 if repair_source_path else count, "candidate_workers": candidate_workers, "resume_candidates": bool(resume_candidates), "repair_source": str(repair_source_path) if repair_source_path else None, "successful_candidates": len(records), "repair_candidates": len(repair_records), "repair_stop_reason": repair_stop_reason, "failures": failures, "candidates": records, "stability": stability, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} write_json(root / "candidate_review.json", report) if report["delivery_blocked"]: - raise RuntimeError("No Image2 candidate passed scientific, OCR, template, and aesthetic production gates") + raise RuntimeError("No Image2 candidate passed paper alignment, scientific, OCR, topology, template, and aesthetic production gates") return report diff --git a/rfs/paper_to_image/inspection.py b/rfs/paper_to_image/inspection.py new file mode 100644 index 0000000..57f7aa2 --- /dev/null +++ b/rfs/paper_to_image/inspection.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import hashlib +import shutil +import time +from pathlib import Path +from typing import Any, Callable + +from ..utils import ensure_dir, read_json, write_json, write_text +from .analyzer import paper_markdown, parse_paper +from .document_cache import DOCUMENT_CACHE_VERSION, read_document_cache, write_document_cache + + +def _source_hash(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def inspect_paper( + paper: str | Path, + out: str | Path, + deadline_seconds: int = 180, + ocr_engine: str = "auto", + ocr_lang: str = "en_ch", + ocr_adapter: Callable | None = None, + archive_input: bool = False, +) -> dict[str, Any]: + started = time.monotonic() + source = Path(paper).resolve() + if not source.exists(): + raise FileNotFoundError(f"Paper does not exist: {source}") + root = ensure_dir(out).resolve() + active_source = source + if archive_input: + inputs = ensure_dir(root / "inputs") + active_source = inputs / f"paper{source.suffix.lower()}" + if active_source.resolve() != source: + shutil.copyfile(source, active_source) + digest = _source_hash(active_source) + model_path = root / "document_model.json" + if model_path.exists(): + cached = read_json(model_path) + if ( + isinstance(cached, dict) + and cached.get("source_sha256") == digest + and cached.get("document_cache_version") == DOCUMENT_CACHE_VERSION + ): + report = dict(cached.get("extraction_report") or {}) + elapsed = round(time.monotonic() - started, 3) + return { + "summary": "PDF inspection reused a matching structured document cache.", + "ok": report.get("status") != "fail", + "status": "cached", + "out_dir": str(root), + "source_sha256": digest, + "extraction_report": str(root / "extraction_report.json"), + "document_model": str(model_path), + "elapsed_seconds": elapsed, + } + deadline = max(30, int(deadline_seconds)) + parsed = None if ocr_adapter else read_document_cache(active_source, ocr_engine=ocr_engine, ocr_lang=ocr_lang) + document_cache_hit = parsed is not None + if parsed is None: + parsed = parse_paper( + active_source, + deadline_at=time.monotonic() + deadline, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + ocr_adapter=ocr_adapter, + ) + if not ocr_adapter: + write_document_cache(active_source, parsed, ocr_engine=ocr_engine, ocr_lang=ocr_lang) + parsed["document_cache_version"] = DOCUMENT_CACHE_VERSION + write_json(model_path, parsed) + write_json(root / "extraction_report.json", parsed["extraction_report"]) + write_json(root / "section_index.json", parsed["document_index"]) + write_text(root / "paper.md", paper_markdown(parsed)) + elapsed = round(time.monotonic() - started, 3) + return { + "summary": "Paper inspection completed without model calls.", + "ok": parsed["extraction_report"].get("status") != "fail", + "status": parsed["extraction_report"].get("status"), + "out_dir": str(root), + "source_sha256": digest, + "page_count": parsed["page_count"], + "evidence_count": len(parsed["evidence"]), + "document_cache_hit": document_cache_hit, + "extraction_report": str(root / "extraction_report.json"), + "document_model": str(model_path), + "elapsed_seconds": elapsed, + } diff --git a/rfs/paper_to_image/ocr_worker.py b/rfs/paper_to_image/ocr_worker.py new file mode 100644 index 0000000..76618d0 --- /dev/null +++ b/rfs/paper_to_image/ocr_worker.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import argparse +import time +from pathlib import Path + +from ..utils import write_json +from .analyzer import _ocr_page + + +def main() -> int: + parser = argparse.ArgumentParser(description="Isolated, deadline-safe PDF OCR page worker.") + parser.add_argument("--paper", required=True) + parser.add_argument("--page", required=True, type=int) + parser.add_argument("--engine", required=True) + parser.add_argument("--lang", default="en_ch") + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--out", required=True) + args = parser.parse_args() + + import fitz + + started = time.monotonic() + document = fitz.open(str(Path(args.paper).resolve())) + try: + blocks, engine, error, diagnostics = _ocr_page( + document[args.page - 1], + args.page, + args.engine, + args.lang, + None, + rapidocr_threads=max(1, int(args.threads)), + ) + finally: + document.close() + write_json(Path(args.out), { + "blocks": blocks, + "engine": engine, + "error": error, + "diagnostics": diagnostics, + "elapsed_seconds": round(time.monotonic() - started, 4), + }) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/rfs/paper_to_image/overlay_renderer.py b/rfs/paper_to_image/overlay_renderer.py new file mode 100644 index 0000000..f1b7e30 --- /dev/null +++ b/rfs/paper_to_image/overlay_renderer.py @@ -0,0 +1,590 @@ +from __future__ import annotations + +import hashlib +import math +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np +from PIL import Image, ImageDraw, ImageFont + +from ..utils import write_json + + +_NODE_COLORS = { + "inputs": ((237, 245, 252, 235), (42, 99, 161, 255), (36, 75, 114, 255)), + "modules": ((237, 249, 247, 235), (31, 117, 126, 255), (25, 91, 87, 255)), + "outputs": ((242, 248, 243, 235), (66, 126, 80, 255), (49, 95, 59, 255)), + "innovations": ((252, 244, 235, 235), (177, 92, 33, 255), (140, 72, 26, 255)), +} + +_CONNECTOR_COLORS = { + "feedback_loop": (177, 92, 33, 255), + "iteration": (177, 92, 33, 255), + "iterate": (177, 92, 33, 255), + "return_flow": (177, 92, 33, 255), + "branch": (98, 84, 147, 255), + "conditioning": (31, 117, 126, 255), + "feature_flow": (42, 99, 161, 255), + "prediction": (66, 126, 80, 255), + "data_flow": (80, 111, 134, 255), +} + + +def _font(size: int, *, bold: bool = False) -> ImageFont.ImageFont: + names = [ + r"C:\Windows\Fonts\arialbd.ttf" if bold else r"C:\Windows\Fonts\arial.ttf", + r"C:\Windows\Fonts\msyhbd.ttc" if bold else r"C:\Windows\Fonts\msyh.ttc", + "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf", + ] + for name in names: + try: + return ImageFont.truetype(name, max(10, int(size))) + except OSError: + continue + return ImageFont.load_default() + + +def _bbox_pixels(bbox: dict[str, Any], width: int, height: int) -> tuple[int, int, int, int]: + x = float(bbox.get("x") or 0.0) + y = float(bbox.get("y") or 0.0) + w = float(bbox.get("w") or 0.0) + h = float(bbox.get("h") or 0.0) + return ( + round(x * width), + round(y * height), + round((x + w) * width), + round((y + h) * height), + ) + + +def _wrap(draw: ImageDraw.ImageDraw, text: str, font: ImageFont.ImageFont, max_width: int, max_lines: int = 3) -> list[str]: + tokens = text.split() if " " in text.strip() else list(text.strip()) + separator = " " if " " in text.strip() else "" + lines: list[str] = [] + current = "" + for token in tokens: + candidate = token if not current else f"{current}{separator}{token}" + if not current or draw.textlength(candidate, font=font) <= max_width: + current = candidate + else: + lines.append(current) + current = token + if current: + lines.append(current) + if len(lines) <= max_lines: + return lines + merged = lines[: max_lines - 1] + merged.append(separator.join(lines[max_lines - 1 :])) + return merged + + +def _fit_text(draw: ImageDraw.ImageDraw, text: str, box: tuple[int, int, int, int], *, bold: bool = True) -> tuple[ImageFont.ImageFont, list[str]]: + max_width = max(20, box[2] - box[0] - 14) + max_height = max(18, box[3] - box[1] - 8) + starting = max(12, min(28, round(max_height * 0.42))) + for size in range(starting, 9, -1): + font = _font(size, bold=bold) + lines = _wrap(draw, text, font, max_width) + rendered = "\n".join(lines) + bounds = draw.multiline_textbbox((0, 0), rendered, font=font, spacing=2, align="center") + if bounds[2] - bounds[0] <= max_width and bounds[3] - bounds[1] <= max_height: + return font, lines + return _font(10, bold=bold), _wrap(draw, text, _font(10, bold=bold), max_width) + + +def _draw_centered_text(draw: ImageDraw.ImageDraw, text: str, box: tuple[int, int, int, int], fill: tuple[int, int, int, int]) -> None: + font, lines = _fit_text(draw, text, box) + rendered = "\n".join(lines) + bounds = draw.multiline_textbbox((0, 0), rendered, font=font, spacing=2, align="center") + x = box[0] + ((box[2] - box[0]) - (bounds[2] - bounds[0])) / 2 + y = box[1] + ((box[3] - box[1]) - (bounds[3] - bounds[1])) / 2 - bounds[1] + draw.multiline_text((x, y), rendered, font=font, fill=fill, spacing=2, align="center") + + +def _draw_dashed_line(draw: ImageDraw.ImageDraw, points: list[tuple[int, int]], fill: tuple[int, int, int, int], width: int, dash: int = 12, gap: int = 8) -> None: + for start, end in zip(points[:-1], points[1:]): + dx, dy = end[0] - start[0], end[1] - start[1] + length = math.hypot(dx, dy) + if length <= 0: + continue + distance = 0.0 + while distance < length: + stop = min(length, distance + dash) + x0 = start[0] + dx * distance / length + y0 = start[1] + dy * distance / length + x1 = start[0] + dx * stop / length + y1 = start[1] + dy * stop / length + draw.line((x0, y0, x1, y1), fill=fill, width=width) + distance += dash + gap + + +def _draw_arrow(draw: ImageDraw.ImageDraw, points: list[tuple[int, int]], relation_type: str, canvas_width: int) -> None: + if len(points) < 2: + return + color = _CONNECTOR_COLORS.get(relation_type, _CONNECTOR_COLORS["data_flow"]) + line_width = max(3, round(canvas_width * 0.0024)) + halo_width = line_width + max(3, round(canvas_width * 0.0022)) + draw.line(points, fill=(255, 255, 255, 225), width=halo_width, joint="curve") + dashed = relation_type in {"conditioning", "feedback_loop", "iteration", "iterate", "return_flow"} + if dashed: + _draw_dashed_line(draw, points, color, line_width, dash=max(9, line_width * 3), gap=max(6, line_width * 2)) + else: + draw.line(points, fill=color, width=line_width, joint="curve") + tail, end = points[-2], points[-1] + angle = math.atan2(end[1] - tail[1], end[0] - tail[0]) + arrow = max(10, round(canvas_width * 0.010)) + wing = 0.48 + triangle = [ + end, + (round(end[0] - arrow * math.cos(angle - wing)), round(end[1] - arrow * math.sin(angle - wing))), + (round(end[0] - arrow * math.cos(angle + wing)), round(end[1] - arrow * math.sin(angle + wing))), + ] + draw.polygon(triangle, fill=(255, 255, 255, 235)) + inner_arrow = max(7, round(arrow * 0.78)) + inner = [ + end, + (round(end[0] - inner_arrow * math.cos(angle - wing)), round(end[1] - inner_arrow * math.sin(angle - wing))), + (round(end[0] - inner_arrow * math.cos(angle + wing)), round(end[1] - inner_arrow * math.sin(angle + wing))), + ] + draw.polygon(inner, fill=color) + + +def render_visual_substrate_blueprint( + semantic_plan: dict[str, Any], + out_path: str | Path, + *, + width: int = 1536, + height: int = 1024, +) -> dict[str, Any]: + """Render geometry guides for Image2 without scientific text or connectors.""" + + if not semantic_plan.get("applied"): + raise ValueError("A deterministic visual substrate requires an applied semantic plan") + image = Image.new("RGB", (width, height), (250, 250, 249)) + draw = ImageDraw.Draw(image, "RGBA") + for panel in semantic_plan.get("panels", []): + if not isinstance(panel, dict): + continue + box = _bbox_pixels(panel.get("bbox_percent", {}), width, height) + draw.rounded_rectangle(box, radius=max(10, round(height * 0.018)), fill=(247, 249, 249, 255), outline=(104, 132, 151, 255), width=3) + for node in semantic_plan.get("nodes", []): + if not isinstance(node, dict): + continue + box = _bbox_pixels(node.get("bbox_percent", {}), width, height) + fill, outline, _ = _NODE_COLORS.get(str(node.get("field") or "modules"), _NODE_COLORS["modules"]) + draw.rounded_rectangle(box, radius=max(8, round(height * 0.014)), fill=fill, outline=outline, width=3) + header_height = max(26, round((box[3] - box[1]) * 0.30)) + header = (box[0] + 4, box[1] + 4, box[2] - 4, min(box[3] - 4, box[1] + header_height)) + draw.rounded_rectangle(header, radius=max(6, round(height * 0.008)), fill=(255, 255, 255, 185), outline=(255, 255, 255, 0), width=1) + output = Path(out_path) + output.parent.mkdir(parents=True, exist_ok=True) + image.save(output) + return { + "summary": "Visual-only Image2 substrate with semantic node geometry and reserved editable header bands.", + "path": str(output), + "width": width, + "height": height, + "node_count": len([item for item in semantic_plan.get("nodes", []) if isinstance(item, dict)]), + "panel_count": len([item for item in semantic_plan.get("panels", []) if isinstance(item, dict)]), + "contains_semantic_text": False, + "contains_connectors": False, + "render_policy": "image2_visual_substrate_only", + } + + +def _normalized_bbox(box: tuple[int, int, int, int], width: int, height: int) -> dict[str, float]: + x, y, w, h = box + return {"x": x / width, "y": y / height, "w": w / width, "h": h / height} + + +def _bbox_center(box: dict[str, Any]) -> tuple[float, float]: + return (float(box["x"]) + float(box["w"]) / 2, float(box["y"]) + float(box["h"]) / 2) + + +def _bbox_iou(first: dict[str, Any], second: dict[str, Any]) -> float: + left = max(float(first["x"]), float(second["x"])) + top = max(float(first["y"]), float(second["y"])) + right = min(float(first["x"]) + float(first["w"]), float(second["x"]) + float(second["w"])) + bottom = min(float(first["y"]) + float(first["h"]), float(second["y"]) + float(second["h"])) + intersection = max(0.0, right - left) * max(0.0, bottom - top) + union = float(first["w"]) * float(first["h"]) + float(second["w"]) * float(second["h"]) - intersection + return intersection / union if union > 0 else 0.0 + + +def _detect_visual_card_boxes(image: Image.Image) -> list[tuple[int, int, int, int]]: + rgb = np.asarray(image.convert("RGB")) + bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR) + hsv = cv2.cvtColor(bgr, cv2.COLOR_BGR2HSV) + mask = (((hsv[:, :, 1] > 35) & (hsv[:, :, 2] < 250)) | (hsv[:, :, 2] < 190)).astype(np.uint8) * 255 + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7, 7)) + mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel, iterations=2) + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + height, width = rgb.shape[:2] + boxes: list[tuple[int, int, int, int]] = [] + for contour in contours: + x, y, w, h = cv2.boundingRect(contour) + normalized_w, normalized_h = w / width, h / height + area = normalized_w * normalized_h + if 0.06 <= normalized_w <= 0.30 and 0.08 <= normalized_h <= 0.45 and area < 0.12: + boxes.append((x, y, w, h)) + return sorted(boxes, key=lambda item: (item[1], item[0])) + + +def _match_visual_cards( + nodes: list[dict[str, Any]], + boxes: list[tuple[int, int, int, int]], + width: int, + height: int, +) -> list[dict[str, Any]]: + pairs = [] + node_costs: dict[int, list[tuple[float, int]]] = {} + box_costs: dict[int, list[tuple[float, int]]] = {} + for node_index, node in enumerate(nodes): + expected = node["bbox_percent"] + expected_center = _bbox_center(expected) + for box_index, box in enumerate(boxes): + detected = _normalized_bbox(box, width, height) + detected_center = _bbox_center(detected) + center_drift = math.dist(expected_center, detected_center) + width_error = abs(math.log(max(float(detected["w"]), 1e-6) / max(float(expected["w"]), 1e-6))) + height_error = abs(math.log(max(float(detected["h"]), 1e-6) / max(float(expected["h"]), 1e-6))) + cost = center_drift * 4.0 + (width_error + height_error) * 0.25 + pairs.append((cost, node_index, box_index)) + node_costs.setdefault(node_index, []).append((cost, box_index)) + box_costs.setdefault(box_index, []).append((cost, node_index)) + assignments: dict[int, tuple[int, float]] = {} + used_boxes: set[int] = set() + for cost, node_index, box_index in sorted(pairs): + if node_index in assignments or box_index in used_boxes: + continue + assignments[node_index] = (box_index, cost) + used_boxes.add(box_index) + matches = [] + for node_index, node in enumerate(nodes): + if node_index not in assignments: + continue + box_index, cost = assignments[node_index] + detected = _normalized_bbox(boxes[box_index], width, height) + expected = node["bbox_percent"] + center_drift = math.dist(_bbox_center(expected), _bbox_center(detected)) + width_relative_error = abs(float(detected["w"]) - float(expected["w"])) / max(float(expected["w"]), 1e-6) + height_relative_error = abs(float(detected["h"]) - float(expected["h"])) / max(float(expected["h"]), 1e-6) + node_alternatives = sorted(node_costs.get(node_index, [])) + box_alternatives = sorted(box_costs.get(box_index, [])) + node_margin = node_alternatives[1][0] - cost if len(node_alternatives) > 1 else float("inf") + box_margin = box_alternatives[1][0] - cost if len(box_alternatives) > 1 else float("inf") + matches.append({ + "node_id": node["id"], + "node_index": node_index, + "detected_box_index": box_index, + "expected_bbox_percent": expected, + "detected_bbox_percent": {key: round(value, 6) for key, value in detected.items()}, + "detected_bbox_pixels": list(boxes[box_index]), + "assignment_cost": round(cost, 6), + "node_assignment_margin": round(node_margin, 6) if math.isfinite(node_margin) else None, + "box_assignment_margin": round(box_margin, 6) if math.isfinite(box_margin) else None, + "center_drift": round(center_drift, 6), + "width_relative_error": round(width_relative_error, 6), + "height_relative_error": round(height_relative_error, 6), + "iou": round(_bbox_iou(expected, detected), 6), + }) + return matches + + +def _header_separator(card_rgb: np.ndarray) -> int: + height, width = card_rgb.shape[:2] + hsv = cv2.cvtColor(cv2.cvtColor(card_rgb, cv2.COLOR_RGB2BGR), cv2.COLOR_BGR2HSV) + edge_mask = ((hsv[:, :, 1] > 35) & (hsv[:, :, 2] < 245)).astype(np.uint8) + left, right = max(2, int(width * 0.03)), min(width - 2, int(width * 0.97)) + for y in range(max(3, int(height * 0.07)), max(4, int(height * 0.36))): + if edge_mask[y, left:right].mean() > 0.58: + return y + return int(height * 0.20) + + +def _contain_dimensions(source_w: int, source_h: int, target_w: int, target_h: int) -> tuple[int, int]: + source_ratio = source_w / max(source_h, 1) + target_ratio = target_w / max(target_h, 1) + if source_ratio > target_ratio: + width = target_w + height = max(1, round(width / source_ratio)) + else: + height = target_h + width = max(1, round(height * source_ratio)) + return width, height + + +def _trim_background_only(image: Image.Image) -> tuple[Image.Image, dict[str, Any]]: + """Trim only border-connected near-background pixels; preserve all detected foreground.""" + + rgb = np.asarray(image.convert("RGB")) + height, width = rgb.shape[:2] + if width < 12 or height < 12: + return image, {"applied": False, "reason": "image_too_small", "bbox_pixels": [0, 0, width, height]} + border = max(2, round(min(width, height) * 0.04)) + border_pixels = np.concatenate( + [ + rgb[:border, :, :].reshape(-1, 3), + rgb[-border:, :, :].reshape(-1, 3), + rgb[:, :border, :].reshape(-1, 3), + rgb[:, -border:, :].reshape(-1, 3), + ], + axis=0, + ) + background = np.median(border_pixels.astype(np.float32), axis=0) + delta = np.linalg.norm(rgb.astype(np.float32) - background, axis=2) + gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY) + hsv = cv2.cvtColor(cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR), cv2.COLOR_BGR2HSV) + foreground = ((delta > 22.0) | (gray < 205) | (hsv[:, :, 1] > 42)).astype(np.uint8) * 255 + foreground = cv2.morphologyEx(foreground, cv2.MORPH_CLOSE, np.ones((3, 3), np.uint8), iterations=1) + count, labels, stats, _ = cv2.connectedComponentsWithStats(foreground, connectivity=8) + minimum_component = max(6, round(width * height * 0.0008)) + kept = [index for index in range(1, count) if int(stats[index, cv2.CC_STAT_AREA]) >= minimum_component] + if not kept: + return image, { + "applied": False, + "reason": "no_reliable_foreground", + "background_rgb": [round(float(value), 2) for value in background], + "bbox_pixels": [0, 0, width, height], + } + left = min(int(stats[index, cv2.CC_STAT_LEFT]) for index in kept) + top = min(int(stats[index, cv2.CC_STAT_TOP]) for index in kept) + right = max(int(stats[index, cv2.CC_STAT_LEFT] + stats[index, cv2.CC_STAT_WIDTH]) for index in kept) + bottom = max(int(stats[index, cv2.CC_STAT_TOP] + stats[index, cv2.CC_STAT_HEIGHT]) for index in kept) + padding = max(2, round(min(width, height) * 0.025)) + left, top = max(0, left - padding), max(0, top - padding) + right, bottom = min(width, right + padding), min(height, bottom + padding) + retained_area = (right - left) * (bottom - top) / max(1, width * height) + if retained_area < 0.04: + return image, { + "applied": False, + "reason": "foreground_bbox_too_small", + "background_rgb": [round(float(value), 2) for value in background], + "bbox_pixels": [0, 0, width, height], + } + applied = bool(left > 0 or top > 0 or right < width or bottom < height) + return image.crop((left, top, right, bottom)) if applied else image, { + "applied": applied, + "reason": "border_connected_background_removed" if applied else "foreground_reaches_canvas_edges", + "background_rgb": [round(float(value), 2) for value in background], + "bbox_pixels": [left, top, right, bottom], + "retained_area_percent": round(retained_area * 100, 2), + "semantic_crop_used": False, + } + + +def normalize_visual_substrate_geometry( + source_path: str | Path, + guide_path: str | Path, + semantic_plan: dict[str, Any], + out_path: str | Path, + *, + report_path: str | Path | None = None, +) -> dict[str, Any]: + """Recompose Image2 node visuals into exact semantic boxes before overlay.""" + + if not semantic_plan.get("applied"): + raise ValueError("Visual substrate normalization requires an applied semantic plan") + source = Path(source_path) + guide = Path(guide_path) + if not source.exists() or not guide.exists(): + raise FileNotFoundError(f"Visual substrate normalization inputs are missing: source={source}, guide={guide}") + with Image.open(source) as raw_image: + raw = raw_image.convert("RGB") + with Image.open(guide) as guide_image: + base = guide_image.convert("RGB").resize(raw.size, Image.Resampling.LANCZOS) + width, height = raw.size + nodes = [item for item in semantic_plan.get("nodes", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + boxes = _detect_visual_card_boxes(raw) + if len(boxes) != len(nodes): + raise ValueError(f"Detected {len(boxes)} visual cards for {len(nodes)} semantic nodes; exact cardinality is required") + matches = _match_visual_cards(nodes, boxes, width, height) + if len(matches) != len(nodes): + raise ValueError(f"Matched only {len(matches)} visual cards for {len(nodes)} semantic nodes") + if any(float(item["assignment_cost"]) > 1.0 for item in matches): + raise ValueError("Visual card matching is too ambiguous for deterministic geometry normalization") + if any( + item.get("node_assignment_margin") is not None and float(item["node_assignment_margin"]) < 0.015 + or item.get("box_assignment_margin") is not None and float(item["box_assignment_margin"]) < 0.015 + for item in matches + ): + raise ValueError("Visual card matching is ambiguous because multiple node-card assignments have nearly equal cost") + + raw_array = np.asarray(raw) + normalized_items = [] + for match in matches: + x, y, card_w, card_h = [int(value) for value in match["detected_bbox_pixels"]] + card = raw_array[y:y + card_h, x:x + card_w] + separator = _header_separator(card) + crop_left = max(2, int(card_w * 0.035)) + crop_right = min(card_w - 2, int(card_w * 0.965)) + crop_top = min(card_h - 5, separator + 3) + crop_bottom = max(crop_top + 2, int(card_h * 0.96)) + visual_crop = raw.crop((x + crop_left, y + crop_top, x + crop_right, y + crop_bottom)) + visual_crop, background_trim = _trim_background_only(visual_crop) + + expected = match["expected_bbox_percent"] + node_left = round(float(expected["x"]) * width) + node_top = round(float(expected["y"]) * height) + node_right = round((float(expected["x"]) + float(expected["w"])) * width) + node_bottom = round((float(expected["y"]) + float(expected["h"])) * height) + content_top = round(node_top + (node_bottom - node_top) * 0.32) + padding = max(3, round(min(node_right - node_left, node_bottom - content_top) * 0.04)) + target_left, target_top = node_left + padding, content_top + padding + target_right, target_bottom = node_right - padding, node_bottom - padding + resized_w, resized_h = _contain_dimensions( + visual_crop.width, + visual_crop.height, + max(1, target_right - target_left), + max(1, target_bottom - target_top), + ) + resized = visual_crop.resize((resized_w, resized_h), Image.Resampling.LANCZOS) + paste_left = target_left + (target_right - target_left - resized_w) // 2 + paste_top = target_top + (target_bottom - target_top - resized_h) // 2 + base.paste(resized, (paste_left, paste_top)) + fill_percent = resized_w * resized_h / max(1, (target_right - target_left) * (target_bottom - target_top)) * 100 + normalized_items.append({ + **match, + "header_separator_fraction": round(separator / max(card_h, 1), 6), + "visual_crop_pixels": [x + crop_left, y + crop_top, x + crop_right, y + crop_bottom], + "background_trim": background_trim, + "target_content_pixels": [target_left, target_top, target_right, target_bottom], + "pasted_visual_pixels": [paste_left, paste_top, paste_left + resized_w, paste_top + resized_h], + "content_area_fill_percent": round(fill_percent, 2), + "fit_policy": "contain_no_crop", + "semantic_crop_used": False, + }) + + output = Path(out_path) + output.parent.mkdir(parents=True, exist_ok=True) + base.save(output) + pre_passed = all( + float(item["center_drift"]) <= 0.035 + and float(item["width_relative_error"]) <= 0.30 + and float(item["height_relative_error"]) <= 0.30 + and float(item["iou"]) >= 0.55 + for item in matches + ) + report = { + "summary": "Image2 visual cards detected, matched to semantic nodes, and recomposed into exact deterministic geometry without semantic cropping.", + "source": str(source), + "guide": str(guide), + "output": str(output), + "expected_node_count": len(nodes), + "detected_card_count": len(boxes), + "matched_node_count": len(matches), + "pre_normalization_alignment": { + "passed": pre_passed, + "max_center_drift": round(max(float(item["center_drift"]) for item in matches), 6), + "min_iou": round(min(float(item["iou"]) for item in matches), 6), + "max_width_relative_error": round(max(float(item["width_relative_error"]) for item in matches), 6), + "max_height_relative_error": round(max(float(item["height_relative_error"]) for item in matches), 6), + }, + "post_normalization_alignment": { + "passed": True, + "node_geometry_source": "semantic_plan_exact_bbox", + "unique_node_card_assignment": len({item["node_id"] for item in matches}) == len(nodes) + and len({item["detected_box_index"] for item in matches}) == len(nodes), + "connector_corridors_restored": True, + "semantic_crop_used": False, + }, + "nodes": normalized_items, + "passed": bool( + len(boxes) == len(nodes) + and len(normalized_items) == len(nodes) + and len({item["node_id"] for item in matches}) == len(nodes) + and len({item["detected_box_index"] for item in matches}) == len(nodes) + ), + } + if report_path: + write_json(report_path, report) + return report + + +def compose_semantic_overlay( + source_path: str | Path, + semantic_plan: dict[str, Any], + overlay_spec: dict[str, Any], + out_path: str | Path, + *, + report_path: str | Path | None = None, +) -> dict[str, Any]: + """Compose exact paper labels and directed connectors over a visual substrate.""" + + if not semantic_plan.get("applied"): + raise ValueError("Deterministic overlay requires an applied semantic plan") + source = Path(source_path) + with Image.open(source) as raw: + image = raw.convert("RGBA") + width, height = image.size + layer = Image.new("RGBA", image.size, (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer, "RGBA") + nodes = {str(item.get("id")): item for item in semantic_plan.get("nodes", []) if isinstance(item, dict) and item.get("id")} + connectors = [item for item in semantic_plan.get("connectors", []) if isinstance(item, dict)] + requested_labels = [item for item in overlay_spec.get("labels", []) if isinstance(item, dict)] + requested_connectors = [item for item in overlay_spec.get("connectors", []) if isinstance(item, dict)] + + connector_index = { + (str(item.get("source") or ""), str(item.get("target") or ""), str(item.get("type") or "data_flow")): item + for item in connectors + } + rendered_connectors = [] + for requested in requested_connectors: + key = (str(requested.get("source") or ""), str(requested.get("target") or ""), str(requested.get("type") or "data_flow")) + connector = connector_index.get(key) + if connector is None: + relaxed = [item for item in connectors if str(item.get("source")) == key[0] and str(item.get("target")) == key[1]] + connector = relaxed[0] if len(relaxed) == 1 else None + if connector is None: + raise ValueError(f"Overlay connector has no semantic geometry: {key[0]} -> {key[1]} [{key[2]}]") + path_pixels = [(round(float(point[0]) * width), round(float(point[1]) * height)) for point in connector.get("path_percent", [])] + if len(path_pixels) < 2: + raise ValueError(f"Overlay connector has fewer than two path points: {key[0]} -> {key[1]}") + _draw_arrow(draw, path_pixels, key[2], width) + rendered_connectors.append({ + "source": key[0], "target": key[1], "type": key[2], + "path_percent": connector.get("path_percent", []), "path_pixels": [list(point) for point in path_pixels], + }) + + rendered_labels = [] + for item in requested_labels: + target_id = str(item.get("target_id") or "") + text = str(item.get("text") or "").strip() + node = nodes.get(target_id) + if not text or node is None: + raise ValueError(f"Overlay label has no semantic node geometry: {target_id or text}") + box = _bbox_pixels(node.get("bbox_percent", {}), width, height) + fill, outline, text_color = _NODE_COLORS.get(str(node.get("field") or "modules"), _NODE_COLORS["modules"]) + draw.rounded_rectangle(box, radius=max(8, round(height * 0.014)), fill=(255, 255, 255, 0), outline=outline, width=max(2, round(width * 0.0018))) + header_height = max(30, min(round((box[3] - box[1]) * 0.42), 74)) + header = (box[0] + 4, box[1] + 4, box[2] - 4, min(box[3] - 4, box[1] + header_height)) + draw.rounded_rectangle(header, radius=max(6, round(height * 0.008)), fill=fill, outline=(255, 255, 255, 200), width=1) + _draw_centered_text(draw, text, header, text_color) + rendered_labels.append({"target_id": target_id, "text": text, "bbox_pixels": list(header), "role": item.get("role")}) + + composed = Image.alpha_composite(image, layer).convert("RGB") + output = Path(out_path) + output.parent.mkdir(parents=True, exist_ok=True) + composed.save(output) + report = { + "summary": "Exact scientific labels and directed connectors composed deterministically over the Image2 visual substrate.", + "source": str(source), + "output": str(output), + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "width": width, + "height": height, + "label_count": len(rendered_labels), + "connector_count": len(rendered_connectors), + "expected_label_count": len(requested_labels), + "expected_connector_count": len(requested_connectors), + "labels": rendered_labels, + "connectors": rendered_connectors, + "text_render_policy": "deterministic_raster_overlay; exact source terminology", + "connector_render_policy": "deterministic_raster_overlay; semantic_plan_path_percent", + "passed": len(rendered_labels) == len(requested_labels) and len(rendered_connectors) == len(requested_connectors), + } + if report_path: + write_json(report_path, report) + return report diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index a2f269f..952ac3e 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -2,10 +2,11 @@ import json import re +import unicodedata from typing import Any from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available -from .analyzer import evidence_excerpt +from .analyzer import evidence_excerpt, overview_figure_panel_excerpt, overview_figure_spatial_excerpt DEFAULT_PREFERENCES = { @@ -35,7 +36,7 @@ def merge_preferences(raw: dict | None, aspect_ratio: str | None = None, languag return merged -def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = None) -> str: +def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = None, evidence_max_chars: int = 58000) -> str: return f""" # Summary @@ -45,6 +46,15 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = Every important factual item must include one or more evidence_ids copied exactly from the supplied evidence labels. Mark uncertain fields as unknown. Use the paper's exact terminology. +Contract completeness rules: +- Treat Figure 1 and captions containing "overview", "framework", "architecture", or "pipeline" as high-priority evidence for the intended paper overview figure. +- Identify every major contribution pillar from the abstract and introduction. For system, dataset, or foundation-model papers, do not omit a data engine, dataset construction process, training loop, or deployment stage when it is a central contribution. +- Represent each separately named scientific component as a separate entity. Do not merge conditioning signals, special tokens, encoders, heads, inputs, or outputs into composite nodes when the paper names them independently. +- Give every input and output a stable id and include the boundary relations from input to the first processing entity and from the final processing entity to the output. +- Every relation endpoint must be an id declared in inputs, modules, outputs, or innovations. +- Include feedback and control edges explicitly instead of describing them only in prose. +- Before returning, check that every item in must_show is represented by a distinct entity or an explicit relation label. + Return this schema: {{ "summary": "Paper-to-image planning result.", @@ -66,16 +76,34 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = "figure_specification": {{ "summary": "Scientific contract for the figure.", "figure_goal": "...", + "research_problem": {{"text": "...", "evidence_ids": ["E0001"]}}, + "central_claim": {{"text": "...", "evidence_ids": ["E0001"]}}, "storyline": [], "must_show": [{{"text": "...", "evidence_ids": ["E0001"]}}], "modules": [{{"id": "...", "name": "...", "role": "...", "evidence_ids": ["E0001"]}}], "relations": [{{"source": "module_id", "target": "module_id", "type": "data_flow|control_flow|training_only|inference_only|comparison|feedback", "label": "", "evidence_ids": ["E0001"]}}], - "inputs": [], - "outputs": [], + "inputs": [{{"id": "stable_input_id", "name": "exact paper term", "evidence_ids": ["E0001"]}}], + "outputs": [{{"id": "stable_output_id", "name": "exact paper term", "evidence_ids": ["E0001"]}}], "innovations": [], + "feedback_loops": [], + "panel_graphs": [{{ + "id": "panel_a", + "panel_label": "a", + "label": "exact panel meaning", + "evidence_ids": [], + "nodes": [{{"instance_id": "panel_a_unique_instance", "name": "exact visible term", "role": "input|module|objective|output|group", "evidence_ids": []}}], + "relations": [{{"source": "panel_a_unique_instance", "target": "panel_a_other_instance", "type": "data_flow|conditioning|prediction|training_objective|feedback", "label": "", "evidence_ids": []}}] + }}], + "training_flow": [], + "inference_flow": [], + "topology": "linear|branch|feedback|multimodal|dense_multiframe|unknown", + "overview_panels": [{{"id": "panel_a", "panel_label": "a", "label": "exact evidence-backed panel title", "description": "...", "evidence_ids": ["E0001"], "entity_ids": []}}], + "required_labels": [], + "optional_visible_labels": [], "visual_priorities": [], "terminology": {{}}, - "forbidden_inventions": [] + "forbidden_inventions": [], + "uncertainties": [] }}, "design_plan": {{ "summary": "Information narrative for one complete image.", @@ -118,29 +146,231 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = {json.dumps(paper_review or {}, ensure_ascii=False, indent=2)} Paper evidence: -{evidence_excerpt(parsed)} +{evidence_excerpt(parsed, max_chars=evidence_max_chars)} +""".strip() + + +def _overview_figure_candidates(parsed: dict, limit: int = 4) -> list[dict[str, Any]]: + positive = re.compile(r"\b(overview|framework|architecture|pipeline|procedure|approach|method|model|system|workflow)\b", re.IGNORECASE) + negative = re.compile(r"\b(comparison|performance|result|ablation|visualization|qualitative|attention map|distribution)\b", re.IGNORECASE) + ranked = [] + for index, item in enumerate(parsed.get("document_index", {}).get("figures", [])): + caption = str(item.get("caption") or "").strip() + if not caption: + continue + score = 0 + if positive.search(caption): + score += 8 + if re.match(r"^(figure|fig\.)\s*[12]\b", caption, re.IGNORECASE): + score += 5 + if 180 <= len(caption) <= 1400: + score += 4 + score += min(5, len(re.findall(r"\b(?:uses?|takes?|feeds?|passes?|predicts?|produces?|outputs?|trains?|samples?|renders?|synthesizes?|optimizes?)\b", caption, re.IGNORECASE))) + if negative.search(caption): + score -= 7 + ranked.append((score, -index, item)) + return [item for _, _, item in sorted(ranked, reverse=True)[: max(1, int(limit))]] + + +def _fast_planner_prompt(parsed: dict, preferences: dict, evidence_max_chars: int = 36000) -> str: + candidates = _overview_figure_candidates(parsed) + return f""" +# Summary + +You are compiling a scientific paper into a directed semantic graph for ONE framework figure. Return JSON only. Focus exclusively on scientific meaning; layout, illustration style, and image-generation wording will be compiled later by code. + +Primary task: +1. Select the most information-rich method/architecture/framework figure caption from the candidates below. Figure 1 is not automatically best; prefer a caption that explicitly names components and directed operations. However, when an overview caption explicitly declares panels such as "a, high-level pre-training and usage" and "b, architecture", treat all declared panels as one indivisible target figure. Use detailed architecture or method evidence to enrich those panels, never to replace or omit one of them. +2. Extract every separately named input, component, intermediate representation, conditioning signal, training-only objective, inference-only step, and output needed to reproduce that overview. +3. Use exact short paper terms as visible names. Do not replace "image encoder" with a prose description such as "network that extracts image features". + A visible name must be copied verbatim from one of its cited evidence blocks. Preserve the paper's original language and writing system; never translate a Chinese, Japanese, Korean, or other non-English term because the user preference says English. If no concise source phrase exists, use the shortest verbatim source-language phrase. Relation labels must follow the same rule or be left blank. +4. Split compound outputs and branches into separate entities. If a component predicts class and box, create separate class and box outputs. If two encoders produce two embeddings, create both embeddings. +5. Preserve direction. Every relation endpoint must be a declared entity id. Include input boundary edges and final output edges. +6. Separate training and inference. Matching, losses, supervision, optimization, and feedback belong in training_flow or training-only entities; test-time classification, decoding, retrieval, rendering, or deployment belong in inference_flow. +7. Every factual item must cite evidence_ids copied exactly from the evidence. Unsupported details must be omitted and recorded under uncertainties. +8. Ignore baseline/comparison methods mentioned only to contrast with the proposed method. Never import terms from references, acknowledgements, bibliography, or result-only figures. +9. For a multi-panel overview, set topology to dense_multiframe and include evidence-backed high-level training/usage stages as well as internal architecture modules. Do not collapse a training-versus-usage overview into only the internal neural-network layers. +10. Treat the bbox-aware figure-local blocks below as direct visual evidence. Short labels inside the target figure outrank generic architecture-family prose. A phrase such as "adaptation of the standard transformer encoder" is a description, not a separate visible module, unless "Transformer Encoder" itself appears as a short figure label or is required as an explicit operational stage. +11. Reconstruct each declared panel as a separate local graph. Do not connect a high-level workflow panel directly into an internal architecture panel merely because both describe the same model. Add cross-panel relations only when the evidence explicitly states that edge. +12. Preserve exact operational order. When a panel contains exact component labels and the paper describes them as followed-by or transformed-by stages, include every intermediate relation instead of jumping from a container title directly to the output. +13. Audit every panel against the concise panel-local inventory. A high-level workflow panel is incomplete if it contains only tensor variables while the inventory also shows a named model/process box, prediction, objective/loss, dataset, or deployment stage. Architecture-family prose must not substitute for those exact panel-local objects. +14. For every declared multi-panel overview, populate panel_graphs. Each panel-local visual occurrence gets a unique instance_id, so repeated labels such as the same model used in pre-training and inference remain separate instances. panel_graph relations must stay within their panel; use the global graph only as a compatibility summary. +15. In the concise panel-local inventory, every entry marked required_candidate=true must map to a panel_graph node using its role_hint and evidence_id. Do not merge two repeated occurrences with different evidence IDs into one instance. Group/container labels may be isolated groups, but operational inputs, modules, objectives, and outputs must have their directed panel-local relations. + +Return exactly this compact schema: +{{ + "summary": "Fast evidence-grounded semantic plan.", + "paper_summary": {{ + "summary": "Structured paper summary.", + "title": "...", + "paper_type": "method|system|dataset|benchmark|analysis|survey|application|unknown", + "research_problem": {{"text": "...", "evidence_ids": []}}, + "central_claim": {{"text": "...", "evidence_ids": []}}, + "inputs": [], + "outputs": [], + "core_modules": [], + "innovations": [], + "training_flow": [], + "inference_flow": [], + "terminology": {{}}, + "unknowns": [] + }}, + "figure_specification": {{ + "summary": "Scientific contract for the figure.", + "figure_goal": "...", + "research_problem": {{"text": "...", "evidence_ids": []}}, + "central_claim": {{"text": "...", "evidence_ids": []}}, + "storyline": [], + "must_show": [], + "inputs": [{{"id": "...", "name": "exact short term", "role": "input", "evidence_ids": []}}], + "modules": [{{"id": "...", "name": "exact short term", "role": "module|intermediate|training_objective|conditioning", "evidence_ids": []}}], + "outputs": [{{"id": "...", "name": "exact short term", "role": "output", "evidence_ids": []}}], + "relations": [{{"source": "...", "target": "...", "type": "data_flow|encoding|conditioning|branch|alignment|prediction|training_objective|rendering_input|feedback", "label": "", "evidence_ids": []}}], + "innovations": [], + "feedback_loops": [], + "panel_graphs": [{{ + "id": "panel_a", + "panel_label": "a", + "label": "exact panel meaning", + "evidence_ids": [], + "nodes": [{{"instance_id": "panel_a_unique_instance", "name": "exact visible term", "role": "input|module|objective|output|group", "evidence_ids": []}}], + "relations": [{{"source": "panel_a_unique_instance", "target": "panel_a_other_instance", "type": "data_flow|conditioning|prediction|training_objective|feedback", "label": "", "evidence_ids": []}}] + }}], + "training_flow": [], + "inference_flow": [], + "topology": "linear|branch|feedback|multimodal|dense_multiframe|unknown", + "required_labels": [], + "terminology": {{}}, + "forbidden_inventions": [], + "uncertainties": [] + }} +}} + +Candidate overview captions: +{json.dumps(candidates, ensure_ascii=False, indent=2)} + +Extraction scope (do not claim coverage beyond this scope): +{json.dumps({key: parsed.get('extraction_report', {}).get(key) for key in ('pdf_type', 'semantic_scope', 'readable_page_ratio', 'ocr_pages', 'warnings')}, ensure_ascii=False, indent=2)} + +User preferences relevant to explanatory wording only (never translate scientific labels away from the paper's source language): +{json.dumps({key: preferences.get(key) for key in ('language', 'must_show', 'must_not_show')}, ensure_ascii=False, indent=2)} + +Prioritized paper evidence: +{evidence_excerpt(parsed, max_chars=evidence_max_chars)} + +BBox-aware target overview figure blocks (normalized [x0,y0,x1,y1], page-local; use for panel membership and visible-label selection, not as permission to invent arrows): +{overview_figure_spatial_excerpt(parsed)} + +Concise panel-local label inventory (use this as a completeness checklist for each declared panel): +{overview_figure_panel_excerpt(parsed)} """.strip() +def _compile_fast_plan(raw: dict, preferences: dict) -> dict: + result = normalize_plan(raw, preferences) + spec = result["figure_specification"] + entities = [ + item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + entity_ids = [str(item.get("id") or "") for item in entities if str(item.get("id") or "")] + labels = [str(item.get("name") or item.get("text") or item.get("statement") or "").strip() for item in entities] + labels = [value for value in labels if value] + topology = str(spec.get("topology") or "unknown") + pattern = {"feedback": "loop", "multimodal": "hub_and_spoke", "dense_multiframe": "stacked", "branch": "two_stage"}.get(topology, "left_to_right") + result["design_plan"] = { + "summary": "Deterministic information narrative compiled from the fast semantic graph.", + "reading_order": entity_ids, + "groups": [], + "innovation_emphasis": [str(item.get("name") or item.get("text") or item.get("statement") or "") for item in spec.get("innovations", []) if isinstance(item, dict)], + "preserve": labels, + "remove": list(spec.get("forbidden_inventions", []) if isinstance(spec.get("forbidden_inventions"), list) else []), + } + result["layout_intent"] = { + "summary": "Deterministic layout intent compiled from semantic topology.", + "pattern": pattern, + "canvas_ratio": preferences.get("aspect_ratio", "16:9"), + "regions": [], + "flow_description": "Render all declared directed relations without reversing arrows.", + "whitespace": "moderate", + } + result["visual_metaphors"] = { + "summary": "Minimal paper-grounded visual objects; exact labels and arrows are added as an editable overlay.", + "items": [{"module_id": str(item.get("id") or ""), "metaphor": str(item.get("name") or "scientific module"), "must_show": [], "avoid_showing": []} for item in entities], + } + result["style_plan"] = { + "summary": "Fast-path style contract.", + "medium": preferences.get("style_description"), + "palette": preferences.get("preferred_palette", []), + "viewpoint": "front-facing academic diagram", + "line_and_shadow": "crisp thin lines with restrained shadows", + "visual_density": preferences.get("visual_density", "high"), + "background": "clean light background", + "text_policy": "background regions only; exact labels and arrows are supplied by overlay_spec.json", + "positive_reference_rules": [], + "negative_reference_rules": preferences.get("avoid", []), + } + return result + + def _first_sentence(text: str) -> str: sentences = [item.strip() for item in re.split(r"(?<=[.!?。！？])\s+", text) if len(item.strip()) > 20] return sentences[0][:500] if sentences else text[:500].strip() +def _plausible_component_heading(value: str) -> bool: + text = re.sub(r"^\d+(?:\.\d+)*\s*", "", str(value or "")).strip() + low = text.casefold() + if not 3 <= len(text) <= 72 or len(text.split()) > 9: + return False + if any(term in low for term in ("http", "www.", "acknowledg", "reference", "appendix", "picture credit", "et al.", "university", "institute")): + return False + if re.search(r"[π∑=·]|\([^)]*\d{4}[^)]*\)|\b(?:fig(?:ure)?|table)\s*\d", text, re.IGNORECASE): + return False + if text.count(",") >= 2 or text.count(":") >= 1: + return False + component_terms = ("architecture", "pipeline", "framework", "module", "encoder", "decoder", "backbone", "parser", "reasoner", "renderer", "retrieval", "generation", "refinement", "training", "inference", "data engine", "model") + return any(term in low for term in component_terms) + + def _heuristic_plan(parsed: dict, preferences: dict, paper_review: dict | None = None) -> dict: if paper_review and paper_review.get("modules"): - review_modules = paper_review.get("modules", [])[:10] + review_modules = [item for item in (list(paper_review.get("modules", [])) + list(paper_review.get("research_objects", [])) + list(paper_review.get("concepts", []))) if item.get("evidence_ids")][:16] modules = [{"id": str(item.get("id")), "name": str(item.get("visible_label") or item.get("statement") or item.get("id")), "role": str(item.get("visual_role") or "module"), "evidence_ids": list(item.get("evidence_ids", []))} for item in review_modules] - relations = [{"source": str(item.get("source_id") or item.get("source") or ""), "target": str(item.get("target_id") or item.get("target") or ""), "type": str(item.get("relation_type") or item.get("type") or "data_flow"), "label": str(item.get("statement") or "")[:48], "evidence_ids": list(item.get("evidence_ids", []))} for item in paper_review.get("relations", []) if (item.get("source_id") or item.get("source")) and (item.get("target_id") or item.get("target"))] + relations = [{"source": str(item.get("source_id") or item.get("source") or ""), "target": str(item.get("target_id") or item.get("target") or ""), "type": str(item.get("relation_type") or item.get("type") or "data_flow"), "label": str(item.get("statement") or "")[:48], "evidence_ids": list(item.get("evidence_ids", []))} for item in paper_review.get("relations", []) if (item.get("source_id") or item.get("source")) and (item.get("target_id") or item.get("target")) and item.get("evidence_ids")] terminology = {str(item.get("statement")): str(item.get("visible_label") or item.get("statement")) for item in paper_review.get("terminology", []) if str(item.get("statement") or "").strip()} title = str(paper_review.get("paper_identity", {}).get("title") or parsed.get("source_name")) research_questions = paper_review.get("research_questions", []) claims = paper_review.get("central_claims", []) figure_goal = str((claims or research_questions or [{"statement": "Explain the paper method faithfully."}])[0].get("statement")) - innovations = paper_review.get("innovations", []) + innovations = [item for item in paper_review.get("innovations", []) if item.get("evidence_ids")] forbidden = [str(item.get("statement")) for item in paper_review.get("forbidden_inventions", [])] - inputs = paper_review.get("inputs", []) - outputs = paper_review.get("outputs", []) + inputs = [item for item in paper_review.get("inputs", []) if item.get("evidence_ids")] + outputs = [item for item in paper_review.get("outputs", []) if item.get("evidence_ids")] + known_modalities = ["image", "text", "audio", "depth", "thermal", "imu", "video"] + expanded_inputs = [] + for item in inputs: + statement = str(item.get("visible_label") or item.get("name") or item.get("statement") or "") + present = [term for term in known_modalities if re.search(rf"\b{term}(?:s)?\b", statement, re.IGNORECASE)] + if len(present) >= 3: + for term in present: + expanded_inputs.append({"id": f"input_{term}", "name": term.upper() if term == "imu" else term.title(), "role": "input modality", "evidence_ids": list(item.get("evidence_ids", []))}) + else: + expanded_inputs.append(item) + inputs = expanded_inputs + if modules: + relation_pairs = {(item["source"], item["target"]) for item in relations} + for item in inputs: + input_id = str(item.get("id") or "") + if input_id and (input_id, modules[0]["id"]) not in relation_pairs: + relations.append({"source": input_id, "target": modules[0]["id"], "type": "data_flow", "label": "encoding", "evidence_ids": list(item.get("evidence_ids", []))}) + if outputs and innovations: + output_id = str(outputs[0].get("id") or "") + innovation_id = str(innovations[0].get("id") or "") + if output_id and innovation_id and (output_id, innovation_id) not in relation_pairs: + relations.append({"source": output_id, "target": innovation_id, "type": "enables", "label": "emergent alignment", "evidence_ids": list(innovations[0].get("evidence_ids", []))}) return { "summary": "Paper-review-grounded fallback planning result.", "paper_summary": {"summary": "Structured paper summary derived from paper_review.json.", "title": title, "paper_type": paper_review.get("paper_identity", {}).get("paper_type", "unknown"), "research_problem": (research_questions or [{}])[0], "central_claim": (claims or [{}])[0], "inputs": inputs, "outputs": outputs, "core_modules": modules, "innovations": innovations, "training_flow": paper_review.get("workflows", {}).get("training", []), "inference_flow": paper_review.get("workflows", {}).get("inference", []), "terminology": terminology, "unknowns": paper_review.get("unknowns", [])}, @@ -155,7 +385,7 @@ def _heuristic_plan(parsed: dict, preferences: dict, paper_review: dict | None = first = evidence[0] title = next((line.strip() for line in first["text"].splitlines() if 10 <= len(line.strip()) <= 180), parsed["source_name"]) generic = {"abstract", "introduction", "related work", "method", "methods", "experiments", "results", "conclusion", "references"} - names = [item for item in headings if item.lower() not in generic][:6] + names = [item for item in headings if item.lower() not in generic and _plausible_component_heading(item)][:6] if len(names) < 3: names = ["Research Input", "Core Method", "Research Output"] modules = [] @@ -233,6 +463,11 @@ def _heuristic_plan(parsed: dict, preferences: dict, paper_review: dict | None = } +def build_review_grounded_plan(parsed: dict, preferences: dict, paper_review: dict) -> dict: + """Compile a deterministic figure plan from an evidence-validated paper review.""" + return normalize_plan(_heuristic_plan(parsed, preferences, paper_review=paper_review), preferences) + + def _ensure_summary(value: Any, fallback: str) -> dict: data = value if isinstance(value, dict) else {} data.setdefault("summary", fallback) @@ -252,13 +487,46 @@ def normalize_plan(raw: dict, preferences: dict) -> dict: return result -def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: str | None = None, reference_images: list[str] | None = None, paper_review: dict | None = None) -> tuple[dict, dict]: - prompt = _planner_prompt(parsed, preferences, paper_review=paper_review) - metadata = {"requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt} +def plan_fast_paper_contract( + parsed: dict, + preferences: dict, + mode: str = "vlm", + model: str | None = None, + timeout_seconds: int = 45, + retries: int = 2, + evidence_max_chars: int = 36000, + deadline_at: float | None = None, +) -> tuple[dict, dict]: + prompt = _fast_planner_prompt(parsed, preferences, evidence_max_chars=evidence_max_chars) + metadata: dict[str, Any] = {"requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt, "provider": {}} + if mode == "vlm" and vlm_credentials_available(): + resolved = resolve_vlm_model("RFS_FAST_FRAMEWORK_MODEL", "RFS_PAPER_TO_IMAGE_MODEL", "RFS_PAPER_PLANNER_MODEL", explicit_model=model) + try: + raw = call_vlm_json( + prompt, + [], + model=resolved, + timeout=max(10, int(timeout_seconds)), + retries=max(0, int(retries)), + call_metadata=metadata["provider"], + deadline_at=deadline_at, + ) + metadata["model"] = resolved + return _compile_fast_plan(raw, preferences), metadata + except Exception as exc: + metadata.update({"mode": "heuristic_after_vlm_failure", "warning": str(exc), "model": resolved}) + elif mode == "vlm": + metadata.update({"mode": "heuristic_without_credentials", "warning": "API_BASE and API_KEY/GEMINI_API_KEY are required for VLM planning"}) + return normalize_plan(_heuristic_plan(parsed, preferences), preferences), metadata + + +def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: str | None = None, reference_images: list[str] | None = None, paper_review: dict | None = None, timeout_seconds: int = 240, retries: int = 1, evidence_max_chars: int = 58000, deadline_at: float | None = None) -> tuple[dict, dict]: + prompt = _planner_prompt(parsed, preferences, paper_review=paper_review, evidence_max_chars=evidence_max_chars) + metadata = {"requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt, "provider": {}} if mode == "vlm" and vlm_credentials_available(): resolved = resolve_vlm_model("RFS_PAPER_TO_IMAGE_MODEL", "RFS_PAPER_PLANNER_MODEL", explicit_model=model) try: - raw = call_vlm_json(prompt, reference_images or [], model=resolved, timeout=240, retries=1) + raw = call_vlm_json(prompt, reference_images or [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries)), call_metadata=metadata["provider"], deadline_at=deadline_at) metadata["model"] = resolved return normalize_plan(raw, preferences), metadata except Exception as exc: @@ -268,75 +536,644 @@ def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: return normalize_plan(_heuristic_plan(parsed, preferences, paper_review=paper_review), preferences), metadata -def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = 1, selected_template: dict | None = None) -> str: +def _item_label_for_prompt(item: dict[str, Any]) -> str: + return str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("label") or item.get("id") or "").strip() + + +def collect_visible_labels(spec: dict, limit: int = 40) -> list[str]: + labels: list[str] = [] + normalized_labels: set[str] = set() + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + + def add(value: object) -> None: + if isinstance(value, dict): + value = value.get("visible_label") or value.get("name") or value.get("text") or value.get("statement") + label = str(value or "").strip() + normalized = re.sub(r"[^\w\u4e00-\u9fff]+", "", label.casefold()) + if label and len(label) <= 64 and normalized and normalized not in normalized_labels: + labels.append(label) + normalized_labels.add(normalized) + + explicit_required = spec.get("required_labels", []) if isinstance(spec.get("required_labels"), list) else [] + for item in explicit_required: + add(item) + if not explicit_required: + terminology = spec.get("terminology", {}) + if isinstance(terminology, dict): + for value in terminology.values(): + add(value) + elif isinstance(terminology, list): + for item in terminology: + add(item) + for field in ("inputs", "modules", "outputs"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + add(item) + for item in spec.get("innovations", []) if isinstance(spec.get("innovations"), list) else []: + if isinstance(item, dict) and (item.get("visible_label_required") or item.get("must_appear_in_figure")): + add(item) + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if isinstance(relation, dict): + add(relation.get("label")) + for panel in panel_graphs: + if not isinstance(panel, dict): + continue + add(panel.get("label")) + for node in panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []: + add(node) + for relation in panel.get("relations", []) if isinstance(panel.get("relations"), list) else []: + if isinstance(relation, dict): + add(relation.get("label")) + return labels[: max(1, int(limit))] + + +def collect_visual_relations(spec: dict) -> list[dict[str, str]]: + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + if len(panel_graphs) >= 2: + result: list[dict[str, str]] = [] + for panel in panel_graphs: + panel_id = str(panel.get("id") or "") + panel_label = str(panel.get("panel_label") or panel_id) + nodes = { + str(item.get("instance_id") or item.get("id") or ""): item + for item in panel.get("nodes", []) if isinstance(item, dict) + } + for relation in panel.get("relations", []) if isinstance(panel.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + if source not in nodes or target not in nodes or source == target: + continue + result.append({ + "source": source, + "source_label": _item_label_for_prompt(nodes[source]), + "target": target, + "target_label": _item_label_for_prompt(nodes[target]), + "type": str(relation.get("type") or "data_flow"), + "label": str(relation.get("label") or "").strip(), + "panel_id": panel_id, + "panel_label": panel_label, + }) + return result + entities = { + str(item.get("id")): str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("id")) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + repeatable = {re.sub(r"[^\w\u4e00-\u9fff]+", "", str(value).casefold()) for value in spec.get("repeatable_labels", []) if str(value).strip()} if isinstance(spec.get("repeatable_labels"), list) else set() + shared_ids = { + item_id for item_id, label in entities.items() + if re.sub(r"[^\w\u4e00-\u9fff]+", "", label.casefold()) in repeatable + } + grouped: dict[tuple[str, str], dict[str, object]] = {} + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + source, target = str(relation.get("source") or ""), str(relation.get("target") or "") + if not source or not target or source == target or source in shared_ids or target in shared_ids: + continue + entry = grouped.setdefault((source, target), {"types": [], "labels": []}) + relation_type = str(relation.get("type") or "data_flow") + if relation_type not in entry["types"]: + entry["types"].append(relation_type) + label = str(relation.get("label") or "").strip() + if label and label not in entry["labels"]: + entry["labels"].append(label) + priority = ("feedback_loop", "evaluation", "revision_input", "feedback", "conditioning", "data_flow", "prediction", "generation_input") + result: list[dict[str, str]] = [] + for (source, target), entry in grouped.items(): + types = list(entry["types"]) + relation_type = next((value for value in priority if value in types), types[0] if types else "data_flow") + result.append({ + "source": source, + "source_label": entities.get(source, source), + "target": target, + "target_label": entities.get(target, target), + "type": relation_type, + "label": str(entry["labels"][0]) if entry["labels"] else "", + }) + return result + + +def collect_visual_roles(spec: dict) -> list[dict[str, str]]: + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + if len(panel_graphs) >= 2: + return [ + { + "id": str(item.get("instance_id") or item.get("id") or ""), + "label": _item_label_for_prompt(item), + "field": "panel_instance", + "role": str(item.get("role") or "module"), + "visual_constraint": "Keep this occurrence inside its declared panel and connect it only through panel-local relations.", + "visual_cue": "A compact paper-grounded visual object matching this panel-local role.", + "panel_id": str(panel.get("id") or ""), + } + for panel in panel_graphs + for item in panel.get("nodes", []) if isinstance(item, dict) + ] + result: list[dict[str, str]] = [] + label_to_index: dict[str, int] = {} + for field in ("inputs", "modules", "outputs", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict) or not item.get("id"): + continue + label = str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("id")) + normalized_label = re.sub(r"[^\w\u4e00-\u9fff]+", "", label.casefold()) + if field == "innovations" and normalized_label in label_to_index: + existing = result[label_to_index[normalized_label]] + existing["visual_constraint"] += " Also highlight this entity as an evidence-backed paper innovation without duplicating the node." + existing["visual_cue"] += " Add a compact innovation emphasis or mechanism inset inside the same entity." + existing["innovation"] = "true" + continue + role = str(item.get("role") or field.rstrip("s")) + normalized_role = re.sub(r"[^a-z0-9]+", "", role.casefold()) + if "datasource" in normalized_role or role.casefold() in {"dataset", "source dataset"}: + constraint = "Show as an upstream data source that provides declared modalities; never show it as a peer modality or processing module." + visual_cue = "Layered dataset cards, database/archive symbols, or source-record thumbnails grounded in the named source." + elif "modality" in normalized_role: + constraint = "Show as an observed input modality downstream of any declared data sources and upstream of the processing method." + visual_cue = "A representative sample or measurement thumbnail for the named modality, not an empty text card." + elif field == "inputs": + constraint = "Show at the input boundary and connect only to its declared downstream target." + visual_cue = "A compact paper-grounded input example or input-object icon." + elif field == "outputs" or any(term in normalized_role for term in ("output", "prediction", "evaluation", "classification", "regressionhead")): + constraint = "Show at the output/evaluation boundary, not as an upstream input or internal method module." + visual_cue = "A compact outcome, prediction, or evaluation visual that does not introduce new metrics or numbers." + elif field == "innovations": + constraint = "Show as a highlighted paper innovation attached to its supported method context, not as an unrelated pipeline stage." + visual_cue = "A small mechanism inset that visually explains the named innovation." + else: + constraint = "Show as an internal method operation or component between its declared incoming and outgoing relations." + visual_cue = "A mechanism-level mini diagram or scientific icon showing what the named operation does." + result.append({ + "id": str(item.get("id")), + "label": label, + "field": field, + "role": role, + "visual_constraint": constraint, + "visual_cue": visual_cue, + }) + if normalized_label: + label_to_index[normalized_label] = len(result) - 1 + return result + + +def _topology_specific_prompt_rules(spec: dict) -> str: + topology = str(spec.get("topology") or "unknown") + normalized_labels = { + re.sub(r"[^\w\u4e00-\u9fff]+", "", str(item.get("name") or item.get("label") or item.get("title") or "").casefold()) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + } + feedback_signature = { + "inputx", "generate", "initialoutput", "feedback", "selffeedback", "refine", "refinedoutput", "modelm", + } + dense_signature = { + "promptablesegmentationtask", "segmentanythingmodel", "dataengine", "image", "prompt", + "imageencoder", "promptencoder", "maskdecoder", "validsegmentationmask", + "assistedmanual", "semiautomatic", "fullyautomatic", "sa1b", + } + if topology == "feedback" and feedback_signature.issubset(normalized_labels): + return """Feedback-loop topology hard rules: +- Keep one top-row forward chain: input x -> Generate -> Initial Output -> FEEDBACK. +- Initial Output must have exactly two distinct outgoing arrows: one to FEEDBACK and one directly to REFINE. +- FEEDBACK must produce Self-Feedback, and Self-Feedback must have its own distinct outgoing arrow into REFINE. +- REFINE must therefore receive exactly two visually separate incoming arrows: Initial Output -> REFINE and Self-Feedback -> REFINE. Do not merge them upstream and do not redirect either arrow to the other source node. +- The connector labeled iterate must start at Refined output and terminate only at FEEDBACK. Its arrowhead must visibly touch the FEEDBACK boundary. +- Model M is a reusable badge inside Generate, FEEDBACK, and/or REFINE; it is not a separate flow node and must not receive extra arrows. +- Forbidden connectors: Initial Output -> Self-Feedback; Refined output -> Generate; REFINE -> Self-Feedback; Self-Feedback -> FEEDBACK; FEEDBACK -> REFINE as a shortcut that omits the visible Self-Feedback node. +- Preserve the supplied blueprint's two-row geometry and right-side return loop. Do not rearrange the loop into a different circular or bottom-return layout.""" + if topology == "feedback": + return """Feedback-loop topology hard rules: +- Follow only the declared nodes and directed relations in the scientific contract. +- Keep the forward path and return path visually separate, with the return arrowhead touching its declared target. +- Preserve every declared intermediate feedback artifact; do not shortcut across it. +- If two declared sources enter one refinement node, draw two visibly distinct incoming arrows. +- Do not introduce Self-Refine-specific labels, shared-model badges, or connector names unless they are present in the contract.""" + if topology == "dense_multiframe" and dense_signature.issubset(normalized_labels): + return """Dense multi-frame topology hard rules: +- Preserve exactly three macro panels: Promptable Segmentation Task, Segment Anything Model, and Data Engine. +- Image must connect only to Image Encoder; Prompt must connect only to Prompt Encoder. +- Image Encoder and Prompt Encoder must each have a separate directed arrow into Mask Decoder. Do not chain one encoder through the other. +- Mask Decoder must connect directly to Valid Segmentation Mask inside the model panel. +- Data Engine must be a separate vertical chain: Assisted-manual -> Semi-automatic -> Fully Automatic -> SA-1B. +- Draw the Segment Anything Model -> Data Engine annotation-support relation as a distinct high-level container-to-container arrow, visually separate from the mask-prediction path. +- Forbidden connectors: Prompt -> Image Encoder; Image -> Prompt Encoder; Image Encoder -> Prompt Encoder; Prompt Encoder -> Image Encoder; Fully Automatic -> Valid Segmentation Mask; Mask Decoder -> Data Engine as a replacement for the required container-level support arrow. +- Do not let any Data Engine stage enter Valid Segmentation Mask, and do not let the model inference path enter SA-1B.""" + if topology == "dense_multiframe": + panels = [item for item in spec.get("overview_panels", []) if isinstance(item, dict) and str(item.get("label") or "").strip()] + panel_lines = "\n".join( + f" {str(item.get('panel_label') or index + 1)}. {str(item.get('label') or '').strip()} — {str(item.get('description') or '').strip()}" + for index, item in enumerate(panels) + ) + panel_rule = ( + f"- Preserve exactly {len(panels)} macro panels with these evidence-backed meanings:\n{panel_lines}" + if panels else + "- Preserve the declared macro-panel grouping and every directed cross-panel relation." + ) + return f"""Dense multi-frame topology hard rules: +{panel_rule} +- Preserve the declared macro-panel grouping and every directed cross-panel relation. +- Keep independent input branches separate until their declared convergence node. +- Keep panel-local output paths separate from dataset, training, or data-engine paths. +- Draw high-level container-to-container relations separately from internal module arrows. +- Do not replace evidence-backed pretraining, data, or evaluation panels with an invented architecture-detail inset. +- Do not add a titled internal inset, legend, or architecture breakdown unless its title is in the exact visible label whitelist. +- Do not introduce SAM-specific encoders, masks, data-engine stages, or dataset labels unless they are present in the contract.""" + return "No additional topology-specific rules. Follow the directed connector checklist exactly." + + +def compile_image_prompt( + plan: dict, + preferences: dict, + candidate_variant: int = 1, + selected_template: dict | None = None, + *, + deterministic_overlay: bool = False, +) -> str: spec = plan["figure_specification"] design = plan["design_plan"] layout = plan["layout_intent"] metaphors = plan["visual_metaphors"] style = plan["style_plan"] template = selected_template or {} - labels = [] - terminology = spec.get("terminology", {}) - if isinstance(terminology, dict): - labels.extend(str(value) for value in terminology.values()) - for module in spec.get("modules", []): - if isinstance(module, dict): - labels.append(str(module.get("name") or "")) - labels = list(dict.fromkeys(label.strip() for label in labels if label.strip() and len(label.strip()) <= 48))[:24] + semantic_plan = template.get("semantic_plan", {}) if isinstance(template.get("semantic_plan"), dict) else {} + labels = collect_visible_labels(spec) + repeatable_labels = [str(value).strip() for value in spec.get("repeatable_labels", []) if str(value).strip()] if isinstance(spec.get("repeatable_labels"), list) else [] + optional_visible_labels = [str(value).strip() for value in spec.get("optional_visible_labels", []) if str(value).strip()] if isinstance(spec.get("optional_visible_labels"), list) else [] + repeatable_normalized = {re.sub(r"[^\w\u4e00-\u9fff]+", "", value.casefold()) for value in repeatable_labels} + numbered_labels = "\n".join( + f"{index}. {label}" + (" (may repeat to show evidence-supported component reuse)" if re.sub(r"[^\w\u4e00-\u9fff]+", "", label.casefold()) in repeatable_normalized else "") + for index, label in enumerate(labels, start=1) + ) + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + panel_graph_active = len(panel_graphs) >= 2 + visual_relations = [] if panel_graph_active else collect_visual_relations(spec) + visual_roles = collect_visual_roles(spec) + panel_instance_lines: list[str] = [] + panel_connector_lines: list[str] = [] + for panel in panel_graphs: + panel_id = str(panel.get("id") or "panel") + panel_label = str(panel.get("panel_label") or panel_id) + node_by_id = { + str(item.get("instance_id") or item.get("id") or ""): item + for item in panel.get("nodes", []) if isinstance(item, dict) + } + for item in node_by_id.values(): + instance_id = str(item.get("instance_id") or item.get("id") or "") + panel_instance_lines.append(f"- Panel {panel_label}: {instance_id} = {_item_label_for_prompt(item)} [{str(item.get('role') or 'module')}]" ) + for relation in panel.get("relations", []) if isinstance(panel.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + if source not in node_by_id or target not in node_by_id: + continue + source_label = _item_label_for_prompt(node_by_id[source]) + target_label = _item_label_for_prompt(node_by_id[target]) + relation_label = str(relation.get("label") or "").strip() + panel_connector_lines.append( + f"{len(panel_connector_lines) + 1}. Panel {panel_label}: {source_label} ({source}) -> {target_label} ({target}) [{str(relation.get('type') or 'data_flow')}]" + + (f" label: {relation_label}" if relation_label else "") + ) + connector_checklist = "\n".join(panel_connector_lines) if panel_graph_active else "\n".join( + f"{index}. {item['source_label']} -> {item['target_label']} [{item['type']}]" + (f" label: {item['label']}" if item["label"] else "") + for index, item in enumerate(visual_relations, start=1) + ) + input_roles = [item for item in visual_roles if item.get("role") == "input"] + output_roles = [item for item in visual_roles if item.get("role") == "output"] + relations_by_target: dict[str, list[dict]] = {} + for relation in visual_relations: + relations_by_target.setdefault(str(relation.get("target") or ""), []).append(relation) + shared_input_rules = [] + for target_id, incoming_relations in relations_by_target.items(): + peer_inputs = [item for item in incoming_relations if any(role.get("id") == item.get("source") for role in input_roles)] + if len(peer_inputs) < 2: + continue + target_label = str(peer_inputs[0].get("target_label") or target_id) + source_labels = [str(item.get("source_label") or item.get("source")) for item in peer_inputs] + shared_input_rules.append( + f"- Treat {', '.join(source_labels)} as peer input sources. Never draw arrows between these peer cards. Draw one separate directed arrow from each peer input to {target_label}." + ) + output_labels = [str(item.get("label") or item.get("id")) for item in output_roles] + boundary_role_rules = ("- Panel-local instance roles and panel-local connector checklists are authoritative. Never connect instances across panels unless a separate explicit cross-panel relation is listed." if panel_graph_active else "\n".join([ + *shared_input_rules, + *( [f"- Treat {', '.join(output_labels)} as output-only nodes. Never draw an arrow from an output back into an input or method module unless an explicit feedback relation appears in the connector checklist."] if output_labels else [] ), + ]) or "- Keep all declared inputs as source roles and all declared outputs as sink roles unless an explicit feedback relation says otherwise.") + topology_specific_rules = _topology_specific_prompt_rules(spec) + compact_roles = ( + [ + { + "id": str(item.get("instance_id") or item.get("id") or ""), + "label": _item_label_for_prompt(item), + "role": str(item.get("role") or "module"), + "panel_id": str(panel.get("id") or ""), + "visual_cue": "A compact paper-grounded visual object matching this panel-local role.", + } + for panel in panel_graphs + for item in panel.get("nodes", []) if isinstance(item, dict) + ] + if panel_graph_active else + [ + {"id": item["id"], "label": item["label"], "role": item["role"], "visual_cue": item["visual_cue"]} + for item in visual_roles + ] + ) + compact_contract = { + "figure_goal": spec.get("figure_goal"), + "central_claim": spec.get("central_claim", {}).get("text") if isinstance(spec.get("central_claim"), dict) else spec.get("central_claim"), + "topology": spec.get("topology"), + "training_flow": spec.get("training_flow", []), + "inference_flow": spec.get("inference_flow", []), + "forbidden_inventions": spec.get("forbidden_inventions", []), + "uncertainties": spec.get("uncertainties", []), + "overview_panels": ([ + { + "id": item.get("id"), + "panel_label": item.get("panel_label"), + "label": item.get("label"), + "description": item.get("description"), + "entity_ids": [str(node.get("instance_id") or node.get("id") or "") for node in item.get("nodes", []) if isinstance(node, dict)], + } + for item in panel_graphs + ] if panel_graph_active else [ + { + "id": item.get("id"), + "panel_label": item.get("panel_label"), + "label": item.get("label"), + "description": item.get("description"), + "entity_ids": item.get("entity_ids", []), + } + for item in spec.get("overview_panels", []) if isinstance(item, dict) + ]), + "panel_graphs": [ + { + "id": panel.get("id"), + "panel_label": panel.get("panel_label"), + "label": panel.get("label"), + "nodes": [ + { + "instance_id": item.get("instance_id") or item.get("id"), + "name": _item_label_for_prompt(item), + "role": item.get("role"), + } + for item in panel.get("nodes", []) if isinstance(item, dict) + ], + "relations": [ + { + "source": item.get("source"), + "target": item.get("target"), + "type": item.get("type"), + "label": item.get("label"), + } + for item in panel.get("relations", []) if isinstance(item, dict) + ], + } + for panel in panel_graphs + ], + "supporting_visual_cues": [ + { + "visual_cue": item.get("visual_cue") or "one unlabeled panel-local scientific mechanism detail", + "must_be_unlabeled": True, + } + for item in spec.get("supporting_details", []) if isinstance(item, dict) + ], + } + compact_style = { + "medium": style.get("medium"), + "palette": style.get("palette", []), + "viewpoint": style.get("viewpoint"), + "line_and_shadow": style.get("line_and_shadow"), + "visual_density": style.get("visual_density"), + "background": style.get("background"), + "negative_reference_rules": style.get("negative_reference_rules", []), + "template_style": style.get("template_style", {}), + } + compact_template = { + "template_id": template.get("template_id"), + "visual_density": template.get("visual_density"), + "style": template.get("style", {}), + "panels": [] if semantic_plan.get("applied") else template.get("panels", []), + "connectors": [] if semantic_plan.get("applied") else template.get("connectors", []), + } + compact_semantic_plan = { + "applied": bool(semantic_plan.get("applied")), + "topology": semantic_plan.get("topology"), + "panels": [ + { + "id": item.get("id"), + "panel_label": item.get("panel_label"), + "label": item.get("label"), + "bbox_percent": item.get("bbox_percent"), + "groups": item.get("groups", []), + } + for item in semantic_plan.get("panels", []) if isinstance(item, dict) + ], + "nodes": [ + { + "id": item.get("id"), + "label": item.get("label"), + "rank": item.get("rank"), + "bbox_percent": item.get("bbox_percent"), + } + for item in semantic_plan.get("nodes", []) if isinstance(item, dict) + ], + "connectors": [ + { + "source": item.get("source"), + "target": item.get("target"), + "type": item.get("type"), + "label": item.get("label"), + "path_percent": item.get("path_percent"), + "route_style": item.get("route_style"), + } + for item in semantic_plan.get("connectors", []) if isinstance(item, dict) + ], + } + if deterministic_overlay: + visible_label_section = f"""Scientific concept-to-node mapping. These terms describe node meaning only; do not render any of them as visible text in the visual substrate: +{json.dumps(labels, ensure_ascii=False)} + +Reserve a clean translucent or light header band inside every node for later deterministic text composition. Do not render letters, words, numbers, formulas, axis labels, panel IDs, legends, headings, watermarks, or pseudo-text anywhere in the image.""" + connector_section = f"""Post-composition directed connector contract. These relations will be drawn later by the deterministic overlay compiler; do not render any arrow, line, connector, arrowhead, loop, or flow path in the visual substrate: +{connector_checklist or '- No visible connector is rendered by Image2.'} + +Keep the whitespace corridors around the supplied semantic connector paths clear enough for the later overlay. Preserve node bounding boxes, but leave all relationships visually unconnected.""" + overlay_requirements = """- Generate only the image-rich visual substrate inside the supplied node geometry. +- Do not render any scientific text or any connector geometry; exact labels and arrows are owned by overlay_spec.json and semantic_plan.path_percent. +- Reserve a clean header band in each node and clear connector corridors between nodes. +- Enrich every input, core method, innovation, and output node with a paper-grounded visual cue from the role checklist. +- Use roughly 55-85% of each node interior below the reserved header band for a representative sample, mechanism inset, scientific icon, or outcome cue. +- For modality nodes, show a representative sample; for data sources, show archive cues; for method nodes, show the internal operation; for outputs, show a compact outcome cue. +- Preserve every node bounding box and layer assignment. Do not merge, split, move, rename, or add nodes. +- Do not add arrows, lines, loops, legends, headings, architecture submodule names, letters, numbers, fake UI text, formulas, axes, charts, citations, or watermarks. +- Keep all major objects complete and inside the canvas; avoid large blank margins. +- Do not invent datasets, mechanisms, metrics, modules, or results. +- Treat the supplied visual-only blueprint as a hard macro-layout guide and replace only the empty node interiors with paper-grounded visual explanation.""" + output_mode_summary = "Create one publication-quality visual substrate for a scientific framework figure. Exact labels and connectors will be composed deterministically after generation." + else: + visible_label_section = f"""Exact visible label whitelist (render these exactly and do not invent alternatives): +{json.dumps(labels, ensure_ascii=False)} + +Evidence-backed optional visible labels (allowed if useful, but not required): +{json.dumps(optional_visible_labels, ensure_ascii=False)} + +Mandatory visible label checklist: +{numbered_labels} + +Every checklist item must appear at least once as clearly readable text in the completed figure. Labels explicitly marked as repeatable may appear more than once only to show reuse of the same paper-supported component; every other label must appear exactly once. An icon, symbol, equation variable, or unlabeled output card does not satisfy a checklist item. Reserve enough width for long labels and place each output label beside or inside its output node.""" + connector_section = f"""Mandatory directed connector checklist: +{connector_checklist} + +Every connector above must be visible with the stated direction. Do not bypass an intermediate module, do not replace two required inputs with one shortcut arrow, and do not add direct source-to-output paths that are absent from this checklist. Shared component labels may repeat as visual badges without adding extra implementation-level arrows.""" + overlay_requirements = """- Preserve every paper-supported core module and relation. +- Use the exact short labels listed in terminology when labels are necessary. +- Make arrows unambiguous and follow the specified source-to-target direction. +- Emphasize the paper's actual innovation rather than generic AI decoration. +- Use a clean academic composition with dense but readable information hierarchy. +- Enrich every input, core method, innovation, and output node with a paper-grounded visual cue from the role checklist. Do not return a bare text-only flowchart. +- Use roughly 35-60% of each node interior for a representative example, mechanism inset, scientific icon, or outcome cue while keeping the exact label clearly readable. +- For modality nodes, show a representative sample of that modality; for data sources, show dataset/archive cues; for method nodes, show the internal operation; for outputs, show a compact outcome/evaluation cue. +- Preserve the supplied geometry and connectors while adding visual explanation inside the nodes; do not merely add gradients or shadows to the blueprint boxes. +- Keep all major objects complete and inside the canvas; avoid large blank margins. +- Do not invent datasets, mechanisms, formulas, metrics, modules, or results. +- Do not render paragraphs, citations, fake equations, fake axes, fake charts, or invented numbers. +- Avoid commercial-poster styling, cyberpunk effects, unrelated robots, generic brains, and repeated dashboard cards. +- Treat the supplied blueprint as a hard macro-layout guide, but replace all reference content with this paper's content. +- When semantic blueprint geometry is supplied, preserve every node bounding box, layer assignment, connector path, and arrow direction; visual decoration may change but graph geometry may not. +- Template panels are macro spatial regions, not an exact limit on the number of scientific content nodes inside those regions. +- Every visible scientific word must come from the exact label whitelist; do not paraphrase labels. +- The exact visible label whitelist is the only permitted visible text. Do not add a legend, inset title, process verb, architecture submodule, or explanatory heading outside that whitelist. +- Render every mandatory visible label at least once; repeat only labels explicitly marked as evidence-supported shared components, and never replace an output label with an icon-only node. +- Keep separately named scientific components as separately editable nodes; do not collapse special tokens, embeddings, heads, inputs, or outputs into composite labels. +- Include explicit input-boundary and output-boundary connectors.""" + output_mode_summary = "Create one complete publication-quality scientific framework figure as a raster image." return f""" # Summary -Create one complete publication-quality scientific framework figure as a raster image. +{output_mode_summary} Candidate variant: {candidate_variant}. Scientific goal: {spec.get('figure_goal')}. Target canvas ratio: {preferences.get('aspect_ratio', '16:9')}. Text language: {preferences.get('language', 'English')}. -Scientific contract: -{json.dumps(spec, ensure_ascii=False, indent=2)} +Compact scientific contract: +{json.dumps(compact_contract, ensure_ascii=False, separators=(',', ':'))} -Information narrative: -{json.dumps(design, ensure_ascii=False, indent=2)} +Style contract: +{json.dumps(compact_style, ensure_ascii=False, separators=(',', ':'))} -Layout intent: -{json.dumps(layout, ensure_ascii=False, indent=2)} +Reference-derived content-free template contract: +{json.dumps(compact_template, ensure_ascii=False, separators=(',', ':'))} -Concrete visual metaphors: -{json.dumps(metaphors, ensure_ascii=False, indent=2)} +Paper-grounded semantic blueprint geometry: +{json.dumps(compact_semantic_plan, ensure_ascii=False, separators=(',', ':'))} -Style contract: -{json.dumps(style, ensure_ascii=False, indent=2)} +{visible_label_section} -Reference-derived content-free template contract: -{json.dumps({key: template.get(key) for key in ['template_id', 'panels', 'connectors', 'visual_density', 'style', 'selection']}, ensure_ascii=False, indent=2)} +{connector_section} -Exact visible label whitelist (render these exactly and do not invent alternatives): -{json.dumps(labels, ensure_ascii=False)} +Boundary-role anti-hallucination rules: +{boundary_role_rules} + +Mandatory panel-local instance checklist: +{chr(10).join(panel_instance_lines) if panel_instance_lines else '- No separate panel-local instance graph; use the global role checklist.'} + +When the panel-local instance checklist is present, it is the authoritative visible occurrence model. Repeated labels are separate visible instances with different instance IDs. The global entity graph is only a compatibility summary and must not create extra cross-panel arrows or replace panel-local nodes. + +Mandatory entity role and containment checklist: +{json.dumps(compact_roles, ensure_ascii=False, separators=(',', ':'))} + +Panel-local supporting visual cues in the compact contract are evidence-backed planning metadata only. Use them as unlabeled internal texture or mechanism detail when helpful. Never render the cue wording itself, never turn the cues into a legend, never give them titles, never promote them to peer top-level nodes, and never invent connectors for them. + +Preserve these semantic roles in the visible composition. A data source, modality, processing module, innovation, and output are not interchangeable even when all labels are present. In particular, do not flatten a source -> modality -> method chain into peer inputs or place a dataset beside modalities as though it were another sensor type. + +Topology-specific rendering rules: +{topology_specific_rules} Forbidden copied reference terms: {json.dumps(template.get('forbidden_copy_terms', []), ensure_ascii=False)} Hard requirements: -- Preserve every paper-supported core module and relation. -- Use the exact short labels listed in terminology when labels are necessary. -- Make arrows unambiguous and follow the specified source-to-target direction. -- Emphasize the paper's actual innovation rather than generic AI decoration. -- Use a clean academic composition with dense but readable information hierarchy. -- Keep all major objects complete and inside the canvas; avoid large blank margins. -- Do not invent datasets, mechanisms, formulas, metrics, modules, or results. -- Do not render paragraphs, citations, fake equations, fake axes, fake charts, or invented numbers. -- Avoid commercial-poster styling, cyberpunk effects, unrelated robots, generic brains, and repeated dashboard cards. -- Treat the supplied blueprint as a hard macro-layout guide, but replace all reference content with this paper's content. -- Every visible scientific word must come from the exact label whitelist; do not paraphrase labels. +{overlay_requirements} """.strip() def validate_plan_grounding(plan: dict, parsed: dict) -> dict: - valid_evidence = {item["id"] for item in parsed.get("evidence", [])} + evidence_items = [item for item in parsed.get("evidence", []) if isinstance(item, dict) and item.get("id")] + valid_evidence = {item["id"] for item in evidence_items} + evidence_text = {str(item["id"]): str(item.get("text") or "") for item in evidence_items} + all_evidence_text = " ".join(evidence_text.values()) errors: list[str] = [] warnings: list[str] = [] spec = plan.get("figure_specification", {}) + + def normalized_term(value: str) -> str: + return re.sub(r"\s+", " ", unicodedata.normalize("NFKC", value).casefold()).strip() + + def script_counts(value: str) -> tuple[int, int]: + cjk = sum( + "\u3400" <= char <= "\u9fff" + or "\u3040" <= char <= "\u30ff" + or "\uac00" <= char <= "\ud7af" + for char in value + ) + latin = sum(("a" <= char.casefold() <= "z") for char in value) + return cjk, latin + + def latin_language(value: str) -> str | None: + folded = "".join(char for char in unicodedata.normalize("NFKD", value.casefold()) if not unicodedata.combining(char)) + words = re.findall(r"[a-z]{2,}", folded) + profiles = { + "english": {"the", "and", "that", "with", "from", "this", "we", "our", "for", "into", "method", "results"}, + "spanish": {"el", "la", "los", "las", "un", "una", "que", "con", "para", "por", "del", "este", "esta", "metodo", "resultados", "relaciones"}, + "french": {"le", "la", "les", "un", "une", "des", "que", "avec", "pour", "dans", "cette", "methode", "resultats", "relations"}, + "german": {"der", "die", "das", "den", "dem", "ein", "eine", "und", "mit", "fur", "von", "diese", "methode", "ergebnisse"}, + "portuguese": {"o", "os", "as", "um", "uma", "que", "com", "para", "por", "este", "esta", "metodo", "resultados", "relacoes"}, + } + scores = {name: sum(word in markers for word in words) for name, markers in profiles.items()} + language, score = max(scores.items(), key=lambda item: item[1]) + if language == "english" or score < 5 or score < scores["english"] + 2: + return None + return language + + detected_latin_language = latin_language(all_evidence_text) + + def english_label_ratio(value: str) -> float | None: + tokens = re.findall(r"[A-Za-z]{3,}", value) + if not tokens: + return None + try: + import wordninja + + known_words = wordninja.DEFAULT_LANGUAGE_MODEL._wordcost + except Exception: + return None + known = sum(token.casefold() in known_words or token.casefold() in {"encoder", "decoder", "embedding", "transformer"} for token in tokens) + return known / len(tokens) + + def validate_visible_label(location: str, label: str, evidence_ids: list[str], *, fallback_text: str = "") -> None: + value = str(label or "").strip() + if not value: + return + source = " ".join(evidence_text.get(str(evidence_id), "") for evidence_id in evidence_ids).strip() or fallback_text + if not source or normalized_term(value) in normalized_term(source): + return + label_cjk, label_latin = script_counts(value) + source_cjk, source_latin = script_counts(source) + changed_from_cjk = source_cjk >= 2 and label_cjk == 0 and label_latin >= 2 + changed_from_latin = source_latin >= 4 and source_cjk == 0 and label_cjk >= 1 + if changed_from_cjk or changed_from_latin: + errors.append(f"{location} visible label changes the source writing system and is not verbatim evidence: {value}") + return + if detected_latin_language and normalized_term(value) not in normalized_term(all_evidence_text): + ratio = english_label_ratio(value) + if ratio is not None and ratio >= 0.8: + errors.append(f"{location} visible label appears translated from {detected_latin_language} and is not verbatim evidence: {value}") + modules = spec.get("modules") if isinstance(spec.get("modules"), list) else [] if not modules: errors.append("figure_specification.modules is empty") @@ -358,6 +1195,34 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: invalid = [item for item in evidence_ids if item not in valid_evidence] if invalid: errors.append(f"module {module_id or index + 1} references unknown evidence ids: {invalid}") + validate_visible_label(f"module {module_id or index + 1}", str(module.get("name") or module.get("visible_label") or ""), evidence_ids) + + endpoint_ids = set(module_ids) + for field in ("inputs", "outputs", "innovations"): + for index, item in enumerate(spec.get(field) if isinstance(spec.get(field), list) else []): + if not isinstance(item, dict): + continue + endpoint_id = str(item.get("id") or item.get("name") or item.get("visible_label") or item.get("text") or "").strip() + if endpoint_id: + endpoint_ids.add(endpoint_id) + evidence_ids = item.get("evidence_ids") if isinstance(item.get("evidence_ids"), list) else [] + if not evidence_ids: + errors.append(f"{field}[{index}] has no evidence_ids") + invalid = [value for value in evidence_ids if value not in valid_evidence] + if invalid: + errors.append(f"{field}[{index}] references unknown evidence ids: {invalid}") + validate_visible_label(f"{field}[{index}]", str(item.get("name") or item.get("visible_label") or item.get("text") or ""), evidence_ids) + + for index, panel in enumerate(spec.get("overview_panels") if isinstance(spec.get("overview_panels"), list) else []): + if not isinstance(panel, dict): + continue + evidence_ids = panel.get("evidence_ids") if isinstance(panel.get("evidence_ids"), list) else [] + if not evidence_ids: + errors.append(f"overview_panels[{index}] has no evidence_ids") + invalid = [value for value in evidence_ids if value not in valid_evidence] + if invalid: + errors.append(f"overview_panels[{index}] references unknown evidence ids: {invalid}") + validate_visible_label(f"overview_panels[{index}]", str(panel.get("label") or ""), evidence_ids) relations = spec.get("relations") if isinstance(spec.get("relations"), list) else [] for index, relation in enumerate(relations): @@ -366,14 +1231,65 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: continue source = str(relation.get("source") or "").strip() target = str(relation.get("target") or "").strip() - if source not in module_ids or target not in module_ids: + if source not in endpoint_ids or target not in endpoint_ids: errors.append(f"relation {index + 1} has unknown endpoint: {source} -> {target}") evidence_ids = relation.get("evidence_ids") if isinstance(relation.get("evidence_ids"), list) else [] if not evidence_ids: - warnings.append(f"relation {source} -> {target} has no evidence_ids") + errors.append(f"relation {source} -> {target} has no evidence_ids") invalid = [item for item in evidence_ids if item not in valid_evidence] if invalid: errors.append(f"relation {source} -> {target} references unknown evidence ids: {invalid}") + validate_visible_label(f"relation {source} -> {target}", str(relation.get("label") or ""), evidence_ids) + + panel_graph_relation_count = 0 + panel_instance_ids: set[str] = set() + for panel_index, panel in enumerate(spec.get("panel_graphs") if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(panel, dict): + errors.append(f"panel_graphs[{panel_index}] is not an object") + continue + panel_evidence_ids = panel.get("evidence_ids") if isinstance(panel.get("evidence_ids"), list) else [] + invalid_panel_evidence = [value for value in panel_evidence_ids if value not in valid_evidence] + if invalid_panel_evidence: + errors.append(f"panel_graphs[{panel_index}] references unknown evidence ids: {invalid_panel_evidence}") + local_ids: set[str] = set() + for node_index, node in enumerate(panel.get("nodes") if isinstance(panel.get("nodes"), list) else []): + if not isinstance(node, dict): + errors.append(f"panel_graphs[{panel_index}].nodes[{node_index}] is not an object") + continue + instance_id = str(node.get("instance_id") or node.get("id") or "").strip() + if not instance_id: + errors.append(f"panel_graphs[{panel_index}].nodes[{node_index}] has no instance_id") + elif instance_id in panel_instance_ids: + errors.append(f"duplicate panel instance id: {instance_id}") + else: + panel_instance_ids.add(instance_id) + local_ids.add(instance_id) + node_evidence_ids = node.get("evidence_ids") if isinstance(node.get("evidence_ids"), list) else [] + if not node_evidence_ids: + errors.append(f"panel instance {instance_id or node_index} has no evidence_ids") + invalid_node_evidence = [value for value in node_evidence_ids if value not in valid_evidence] + if invalid_node_evidence: + errors.append(f"panel instance {instance_id or node_index} references unknown evidence ids: {invalid_node_evidence}") + validate_visible_label(f"panel instance {instance_id or node_index}", _item_label_for_prompt(node), node_evidence_ids) + for relation_index, panel_relation in enumerate(panel.get("relations") if isinstance(panel.get("relations"), list) else []): + if not isinstance(panel_relation, dict): + errors.append(f"panel_graphs[{panel_index}].relations[{relation_index}] is not an object") + continue + source = str(panel_relation.get("source") or "").strip() + target = str(panel_relation.get("target") or "").strip() + if source not in local_ids or target not in local_ids: + errors.append(f"panel_graphs[{panel_index}] relation has non-local endpoint: {source} -> {target}") + relation_evidence_ids = panel_relation.get("evidence_ids") if isinstance(panel_relation.get("evidence_ids"), list) else [] + if not relation_evidence_ids: + errors.append(f"panel relation {source} -> {target} has no evidence_ids") + invalid_relation_evidence = [value for value in relation_evidence_ids if value not in valid_evidence] + if invalid_relation_evidence: + errors.append(f"panel relation {source} -> {target} references unknown evidence ids: {invalid_relation_evidence}") + panel_graph_relation_count += 1 + + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} + for source_term, visible_label in terminology.items(): + validate_visible_label(f"terminology[{source_term}]", str(visible_label or ""), [], fallback_text=all_evidence_text) for field in ("must_show", "innovations"): items = spec.get(field) if isinstance(spec.get(field), list) else [] @@ -385,12 +1301,28 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: if invalid: errors.append(f"{field}[{index}] references unknown evidence ids: {invalid}") + for field in ("research_problem", "central_claim"): + item = spec.get(field) + if not isinstance(item, dict): + errors.append(f"figure_specification.{field} is missing") + continue + text = str(item.get("text") or item.get("statement") or "").strip() + evidence_ids = item.get("evidence_ids") if isinstance(item.get("evidence_ids"), list) else [] + explicitly_unknown = item.get("status") == "unknown" or text.casefold() == "unknown" or "require" in text.casefold() and "review" in text.casefold() + if not explicitly_unknown and not evidence_ids: + errors.append(f"figure_specification.{field} has no evidence_ids") + invalid = [value for value in evidence_ids if value not in valid_evidence] + if invalid: + errors.append(f"figure_specification.{field} references unknown evidence ids: {invalid}") + return { "summary": "Scientific grounding and relation-contract validation.", "ok": not errors, "module_count": len(modules), "relation_count": len(relations), + "panel_graph_relation_count": panel_graph_relation_count, "evidence_count": len(valid_evidence), "errors": errors, "warnings": warnings, + "detected_source_language": detected_latin_language or "unknown_or_english", } diff --git a/rfs/paper_to_image/ppt_roundtrip.py b/rfs/paper_to_image/ppt_roundtrip.py new file mode 100644 index 0000000..996c8ea --- /dev/null +++ b/rfs/paper_to_image/ppt_roundtrip.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import zipfile +from pathlib import Path +from xml.etree import ElementTree + +from PIL import Image + +from ..exporter import export_outputs +from ..utils import write_json + + +_NS = { + "a": "http://schemas.openxmlformats.org/drawingml/2006/main", + "p": "http://schemas.openxmlformats.org/presentationml/2006/main", +} + + +def _slide_tree(pptx_path: Path) -> ElementTree.Element: + with zipfile.ZipFile(pptx_path) as archive: + return ElementTree.fromstring(archive.read("ppt/slides/slide1.xml")) + + +def _shape_names(root: ElementTree.Element) -> list[str]: + return [str(item.get("name") or "") for item in root.findall(".//p:cNvPr", _NS)] + + +def _visible_text(root: ElementTree.Element) -> str: + return "\n".join(str(item.text or "") for item in root.findall(".//a:t", _NS)) + + +def _flattened_picture_area_ratio(root: ElementTree.Element, slide_width_emu: int, slide_height_emu: int) -> float: + area = 0 + for picture in root.findall(".//p:pic", _NS): + ext = picture.find(".//a:xfrm/a:ext", _NS) + if ext is None: + continue + try: + area += max(0, int(ext.get("cx") or 0)) * max(0, int(ext.get("cy") or 0)) + except (TypeError, ValueError): + continue + slide_area = max(1, int(slide_width_emu) * int(slide_height_emu)) + return round(min(1.0, area / slide_area), 6) + + +def evaluate_ppt_roundtrip( + program: dict, + pptx: str | Path, + out: str | Path, + *, + render: bool = True, +) -> dict: + pptx_path = Path(pptx) + root_dir = Path(out) + root_dir.mkdir(parents=True, exist_ok=True) + slide = _slide_tree(pptx_path) + names = _shape_names(slide) + text = _visible_text(slide) + + expected_nodes = [item for item in program.get("semantic_nodes", []) if isinstance(item, dict) and item.get("id")] + expected_arrows = [item for item in program.get("arrows", []) if isinstance(item, dict) and item.get("id")] + expected_labels = [str(item.get("label") or "").strip() for item in expected_nodes if str(item.get("label") or "").strip()] + present_labels = [label for label in expected_labels if label in text] + present_nodes = [item for item in expected_nodes if any(name.startswith(f"RFS Semantic Node {item['id']}") for name in names)] + present_arrows = [item for item in expected_arrows if any(name.startswith(f"RFS Connector {item['id']} ") for name in names)] + + canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} + width_in = float(canvas.get("width_in") or 13.333) + height_in = float(canvas.get("height_in") or 7.5) + emu_per_inch = 914400 + flattened_ratio = _flattened_picture_area_ratio( + slide, + round(width_in * emu_per_inch), + round(height_in * emu_per_inch), + ) + + export = {"pdf": None, "png": None, "status": "not_requested"} + rendered_ratio_error = None + rendered_size = None + if render: + export = export_outputs(pptx_path, root_dir) + png_path = Path(export["png"]) if export.get("png") else None + if png_path and png_path.exists(): + with Image.open(png_path) as image: + rendered_size = {"width": image.width, "height": image.height} + expected_ratio = width_in / max(height_in, 0.001) + actual_ratio = image.width / max(image.height, 1) + rendered_ratio_error = round(abs(actual_ratio - expected_ratio) / expected_ratio, 6) + + label_coverage = len(present_labels) / max(1, len(expected_labels)) + node_coverage = len(present_nodes) / max(1, len(expected_nodes)) + connector_coverage = len(present_arrows) / max(1, len(expected_arrows)) + structure_ok = label_coverage == 1.0 and node_coverage == 1.0 and connector_coverage == 1.0 and flattened_ratio <= 0.35 + render_ok = not render or (export.get("status") == "ok" and rendered_ratio_error is not None and rendered_ratio_error <= 0.01) + report = { + "summary": "Editable PowerPoint round-trip report comparing the semantic program, native PPT object tree, and optional PowerPoint render.", + "ok": bool(structure_ok and render_ok), + "pptx": str(pptx_path), + "structure": { + "expected_node_count": len(expected_nodes), + "native_editable_node_count": len(present_nodes), + "node_editability_ratio": round(node_coverage, 6), + "expected_connector_count": len(expected_arrows), + "native_editable_connector_count": len(present_arrows), + "connector_editability_ratio": round(connector_coverage, 6), + "expected_label_count": len(expected_labels), + "editable_label_count": len(present_labels), + "editable_label_ratio": round(label_coverage, 6), + "missing_labels": [label for label in expected_labels if label not in present_labels], + "missing_node_ids": [str(item["id"]) for item in expected_nodes if item not in present_nodes], + "missing_connector_ids": [str(item["id"]) for item in expected_arrows if item not in present_arrows], + "flattened_picture_area_ratio": flattened_ratio, + "max_flattened_picture_area_ratio": 0.35, + }, + "render": { + **export, + "rendered_size": rendered_size, + "canvas_ratio_relative_error": rendered_ratio_error, + "max_canvas_ratio_relative_error": 0.01, + }, + } + write_json(root_dir / "ppt_roundtrip_report.json", report) + return report diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py new file mode 100644 index 0000000..04132f1 --- /dev/null +++ b/rfs/paper_to_image/preparation.py @@ -0,0 +1,3255 @@ +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import time +from pathlib import Path +from typing import Any, Callable + +from ..utils import ensure_dir, read_json, write_json, write_text +from .analyzer import overview_figure_panel_excerpt, paper_markdown, parse_paper +from .contract_completion import augment_contract_from_evidence +from .document_cache import DOCUMENT_CACHE_VERSION, read_document_cache, write_document_cache +from .planner import build_review_grounded_plan, compile_image_prompt, merge_preferences, plan_fast_paper_contract, plan_paper_image, validate_plan_grounding +from .review import build_paper_review, detect_domain_profile, validate_review_coverage +from .review_contract import compile_review_ground_truth + + +def _load_preferences(path: str | Path | None) -> dict[str, Any]: + if not path: + return {} + source = Path(path).resolve() + if not source.exists(): + raise FileNotFoundError(f"Preferences file does not exist: {source}") + raw = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError("Preferences file must contain a JSON object") + return raw + + +def _archive_file(source: str | Path, target_dir: Path, target_name: str | None = None) -> str: + path = Path(source).resolve() + if not path.exists(): + raise FileNotFoundError(f"Input file does not exist: {path}") + target_dir.mkdir(parents=True, exist_ok=True) + target = target_dir / (target_name or path.name) + if target.resolve() != path: + shutil.copyfile(path, target) + return str(target) + + +def _section_summary_markdown(parsed: dict[str, Any]) -> str: + grouped: dict[str, list[str]] = {} + for item in parsed.get("evidence", []): + section = str(item.get("section_hint") or "Unclassified") + grouped.setdefault(section, []).append(str(item.get("text") or "")) + lines = ["# Section Summary", ""] + for section, values in grouped.items(): + combined = " ".join(value.replace("\n", " ") for value in values) + sentences = [value.strip() for value in combined.split(". ") if value.strip()] + summary = ". ".join(sentences[:3])[:900].strip() + lines.extend([f"## {section}", "", summary or "No reliable summary extracted.", ""]) + return "\n".join(lines).strip() + "\n" + + +def _collect_evidence_ids(value: Any) -> set[str]: + found: set[str] = set() + if isinstance(value, dict): + for key, child in value.items(): + if key == "evidence_ids" and isinstance(child, list): + found.update(str(item) for item in child if str(item).strip()) + else: + found.update(_collect_evidence_ids(child)) + elif isinstance(value, list): + for child in value: + found.update(_collect_evidence_ids(child)) + return found + + +def _review_evidence_map(value: Any) -> dict[str, list[str]]: + mapping: dict[str, list[str]] = {} + if isinstance(value, dict): + item_id = str(value.get("id") or "").strip() + evidence_ids = [str(item) for item in value.get("evidence_ids", []) if str(item).startswith("E")] + if item_id and evidence_ids: + mapping[item_id] = evidence_ids + for child in value.values(): + mapping.update(_review_evidence_map(child)) + elif isinstance(value, list): + for child in value: + mapping.update(_review_evidence_map(child)) + return mapping + + +def expand_plan_evidence(plan: dict[str, Any], paper_review: dict[str, Any], parsed: dict[str, Any]) -> None: + mapping = _review_evidence_map(paper_review) + valid = {item["id"] for item in parsed.get("evidence", [])} + legacy = {item.get("legacy_id"): item["id"] for item in parsed.get("evidence", []) if item.get("legacy_id")} + def expand(value: Any) -> None: + if isinstance(value, dict): + if isinstance(value.get("evidence_ids"), list): + expanded = [] + for item in value["evidence_ids"]: + key = str(item) + if key in valid: + expanded.append(key) + elif key in legacy: + expanded.append(legacy[key]) + else: + expanded.extend(mapping.get(key, [])) + value["evidence_ids"] = list(dict.fromkeys(expanded)) + for child in value.values(): + expand(child) + elif isinstance(value, list): + for child in value: + expand(child) + expand(plan) + + +def _item_id(item: dict[str, Any], field: str, index: int) -> str: + return str(item.get("id") or item.get("name") or item.get("visible_label") or item.get("text") or item.get("statement") or f"{field}_{index}").strip() + + +def _item_label(item: dict[str, Any]) -> str: + return str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("label") or item.get("id") or "").strip() + + +def _normalized_label(value: str) -> str: + return "".join(char for char in value.casefold() if char.isalnum()) + + +def _script_counts(value: str) -> tuple[int, int]: + cjk = sum( + "\u3400" <= char <= "\u9fff" + or "\u3040" <= char <= "\u30ff" + or "\uac00" <= char <= "\ud7af" + for char in str(value or "") + ) + latin = sum(("a" <= char.casefold() <= "z") for char in str(value or "")) + return cjk, latin + + +def _repair_cross_script_terminology(spec: dict[str, Any], parsed: dict[str, Any]) -> None: + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} + if not terminology: + return + corpus = " ".join(str(item.get("text") or "") for item in parsed.get("evidence", []) if isinstance(item, dict)) + normalized_corpus = re.sub(r"\s+", " ", corpus.casefold()) + repaired: dict[str, str] = {} + for source_term, visible_label in terminology.items(): + source = str(source_term or "").strip() + visible = str(visible_label or "").strip() + source_present = bool(source and re.sub(r"\s+", " ", source.casefold()) in normalized_corpus) + visible_present = bool(visible and re.sub(r"\s+", " ", visible.casefold()) in normalized_corpus) + source_cjk, source_latin = _script_counts(source) + visible_cjk, visible_latin = _script_counts(visible) + changed_script = (source_cjk >= 2 and visible_cjk == 0 and visible_latin >= 2) or (source_latin >= 4 and source_cjk == 0 and visible_cjk >= 2 and visible_latin == 0) + repaired[source] = source if source_present and not visible_present and changed_script else visible + spec["terminology"] = repaired + + +def _repair_cross_script_entity_labels(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + evidence_by_id = { + str(item.get("id")): str(item.get("text") or "") + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + repaired: list[str] = [] + collections: list[tuple[str, list[Any]]] = [ + (field, spec.get(field, []) if isinstance(spec.get(field), list) else []) + for field in ("inputs", "modules", "outputs", "innovations") + ] + collections.extend( + (f"panel_graphs.{panel.get('id')}.nodes", panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []) + for panel in spec.get("panel_graphs", []) if isinstance(panel, dict) + ) + for field, items in collections: + for item in items: + if not isinstance(item, dict): + continue + label = _item_label(item) + label_cjk, label_latin = _script_counts(label) + records = [evidence_by_id[str(value)] for value in item.get("evidence_ids", []) if str(value) in evidence_by_id] + source = " ".join(records) + source_cjk, source_latin = _script_counts(source) + if not (label_cjk >= 1 and label_latin >= 2 and source_cjk == 0 and source_latin >= 4): + continue + label_without_cjk = re.sub(r"[\u3400-\u9fff\u3040-\u30ff\uac00-\ud7af]+", "", label) + label_key = _normalized_label(label_without_cjk) + exact_record = next( + ( + str(record).strip() + for record in sorted(records, key=len) + if str(record).strip() + and len(str(record).strip()) <= max(120, len(label) + 24) + and _normalized_label(str(record)) == label_key + ), + None, + ) + if exact_record: + item["name"] = exact_record + item.pop("visible_label", None) + repaired.append(f"{item.get('id') or item.get('instance_id')}:{label}->{exact_record}") + continue + tokens = re.findall(r"[A-Za-z]+|\d+(?:\.\d+)?", label) + if len(tokens) < 2: + continue + pattern = r"\b" + r"[^A-Za-z0-9]+".join(re.escape(token) for token in tokens) + r"\b" + match = re.search(pattern, source, re.IGNORECASE) + if not match: + continue + exact = re.sub(r"\s+", " ", match.group(0)).strip(" .,:;") + if not exact: + continue + item["name"] = exact + item.pop("visible_label", None) + repaired.append(f"{item.get('id') or item.get('instance_id')}:{label}->{exact}") + return repaired + + +def _ground_exact_visible_entities(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + evidence = [item for item in parsed.get("evidence", []) if isinstance(item, dict) and item.get("id") and item.get("text")] + novelty_pattern = re.compile(r"\b(?:we\s+(?:introduce|propose|present|develop)|novel|new\s+(?:method|model|system|framework)|contribution)\b|\u672c\u6587\u63d0\u51fa|\u6211\u4eec\u63d0\u51fa|\u9996\u6b21|\u521b\u65b0", re.IGNORECASE) + overview_pattern = re.compile(r"overview|framework|architecture|pipeline|procedure|pre-training|fine-tuning|system|model|\u6846\u67b6|\u67b6\u6784|\u7cfb\u7edf", re.IGNORECASE) + overview_pages = { + int(item.get("page") or 0) + for item in parsed.get("document_index", {}).get("figures", []) + if isinstance(item, dict) and overview_pattern.search(str(item.get("caption") or "")) + } + grounded: list[str] = [] + for field in ("inputs", "modules", "outputs", "innovations"): + for index, item in enumerate(spec.get(field, []) if isinstance(spec.get(field), list) else []): + if not isinstance(item, dict) or item.get("evidence_ids"): + continue + label = _item_label(item) + compact = _normalized_label(label) + if not compact: + continue + matches = [] + for record in evidence: + record_compact = _normalized_label(str(record.get("text") or "")) + short_diagram_label = len(compact) < 3 and int(record.get("page") or 0) in overview_pages and record_compact == compact + if not short_diagram_label and (len(compact) < 3 or compact not in record_compact): + continue + if field == "innovations" and not novelty_pattern.search(str(record.get("text") or "")): + continue + matches.append(str(record["id"])) + if not matches: + continue + item["evidence_ids"] = list(dict.fromkeys(matches))[:3] + grounded.append(f"{field}[{index}]") + return grounded + + +def _repair_relation_evidence_after_grounding(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + """Attach conservative endpoint evidence after late exact-label grounding.""" + endpoints = { + str(item.get("id")): item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + evidence_by_id = { + str(item.get("id")): item + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + repaired: list[str] = [] + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict) or relation.get("evidence_ids"): + continue + source = endpoints.get(str(relation.get("source") or "")) + target = endpoints.get(str(relation.get("target") or "")) + if not source or not target: + continue + source_ids = [str(value) for value in source.get("evidence_ids", []) if str(value) in evidence_by_id] + target_ids = [str(value) for value in target.get("evidence_ids", []) if str(value) in evidence_by_id] + candidate_ids = list(dict.fromkeys([*source_ids, *target_ids])) + if not candidate_ids: + continue + source_pages = {int(evidence_by_id[value].get("page") or 0) for value in source_ids} + target_pages = {int(evidence_by_id[value].get("page") or 0) for value in target_ids} + nearby = bool(source_pages and target_pages and min(abs(left - right) for left in source_pages for right in target_pages) <= 1) + source_label = _normalized_label(_item_label(source)) + target_label = _normalized_label(_item_label(target)) + bridge_ids = [ + str(item["id"]) + for item in parsed.get("evidence", []) + if isinstance(item, dict) + and item.get("id") + and source_label + and target_label + and source_label in _normalized_label(str(item.get("text") or "")) + and target_label in _normalized_label(str(item.get("text") or "")) + ][:2] + if not nearby and not bridge_ids: + continue + relation["evidence_ids"] = list(dict.fromkeys([*candidate_ids, *bridge_ids])) + repaired.append(f"{relation.get('source')}->{relation.get('target')}") + return repaired + + +def _prune_out_of_scope_entities(spec: dict[str, Any], parsed: dict[str, Any], completion_report: dict[str, Any]) -> list[str]: + selected_ids = {str(value) for value in completion_report.get("selected_overview_evidence_ids", []) if str(value)} + if not selected_ids: + return [] + evidence_by_id = { + str(item.get("id")): item + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + anchor_text = " ".join( + [ + str(spec.get("figure_goal") or ""), + str(spec.get("research_problem", {}).get("text") or "") if isinstance(spec.get("research_problem"), dict) else str(spec.get("research_problem") or ""), + str(spec.get("central_claim", {}).get("text") or "") if isinstance(spec.get("central_claim"), dict) else str(spec.get("central_claim") or ""), + *(str(evidence_by_id[value].get("text") or "") for value in selected_ids if value in evidence_by_id), + ] + ) + normalized_anchor = _normalized_label(anchor_text) + blocked_sections = ("appendix", "experiment", "result", "analysis", "visualization", "ablation", "attention map", "reference", "acknowledg") + removed_ids: set[str] = set() + removed_labels: list[str] = [] + for field in ("inputs", "modules", "outputs"): + kept = [] + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + evidence_ids = [str(value) for value in item.get("evidence_ids", []) if str(value)] + records = [evidence_by_id[value] for value in evidence_ids if value in evidence_by_id] + label = _item_label(item) + label_in_anchor = bool(_normalized_label(label) and _normalized_label(label) in normalized_anchor) + selected_grounded = any(value in selected_ids for value in evidence_ids) + only_blocked = bool(records) and all(any(term in str(record.get("section_hint") or "").casefold() for term in blocked_sections) for record in records) + weak_isolated_label = bool(records) and all(_normalized_label(str(record.get("text") or "")) == _normalized_label(label) for record in records) + if not selected_grounded and not label_in_anchor and only_blocked and weak_isolated_label: + removed_ids.add(str(item.get("id") or "")) + removed_labels.append(label) + continue + kept.append(item) + spec[field] = kept + if removed_ids: + spec["relations"] = [ + item for item in spec.get("relations", []) + if isinstance(item, dict) and str(item.get("source")) not in removed_ids and str(item.get("target")) not in removed_ids + ] + return removed_labels + + +def _remove_ungrounded_innovations(spec: dict[str, Any]) -> list[str]: + removed_ids: set[str] = set() + removed_labels: list[str] = [] + kept = [] + for item in spec.get("innovations", []) if isinstance(spec.get("innovations"), list) else []: + if isinstance(item, dict) and item.get("evidence_ids"): + kept.append(item) + continue + if isinstance(item, dict): + removed_ids.add(str(item.get("id") or "")) + removed_labels.append(_item_label(item)) + spec["innovations"] = kept + if removed_ids: + spec["relations"] = [ + item for item in spec.get("relations", []) + if isinstance(item, dict) and str(item.get("source")) not in removed_ids and str(item.get("target")) not in removed_ids + ] + return removed_labels + + +def _remove_input_shortcuts_through_intermediates(spec: dict[str, Any]) -> list[str]: + inputs = {str(item.get("id")) for item in spec.get("inputs", []) if isinstance(item, dict) and item.get("id")} + entities = { + str(item.get("id")): item + for field in ("modules", "outputs") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] + outgoing: dict[str, list[dict[str, Any]]] = {} + for relation in relations: + outgoing.setdefault(str(relation.get("source") or ""), []).append(relation) + removed: list[str] = [] + remove_ids: set[int] = set() + flow_types = {"data_flow", "encoding", "prediction", "generation_input"} + for index, direct in enumerate(relations): + source = str(direct.get("source") or "") + target = str(direct.get("target") or "") + if source not in inputs or str(direct.get("type") or "data_flow") not in flow_types: + continue + for first in outgoing.get(source, []): + middle = str(first.get("target") or "") + if middle == target or middle not in entities: + continue + role_text = f"{entities[middle].get('role', '')} {_item_label(entities[middle])}".casefold() + if not any(term in role_text for term in ("intermediate", "artifact", "patch", "token sequence", "representation")): + continue + if any(str(second.get("target") or "") == target and str(second.get("type") or "data_flow") in flow_types for second in outgoing.get(middle, [])): + remove_ids.add(index) + removed.append(f"{source}->{target}") + break + if remove_ids: + spec["relations"] = [item for index, item in enumerate(relations) if index not in remove_ids] + return removed + + +def _stable_id(prefix: str, value: str, index: int) -> str: + slug = "_".join(part for part in re.sub(r"[^a-z0-9]+", " ", value.casefold()).split() if part)[:48] + return f"{prefix}_{slug or index}" + + +def _statement_object(value: Any, fallback: Any = None) -> dict[str, Any]: + candidate = value if value is not None else fallback + if isinstance(candidate, dict): + item = dict(candidate) + text = str(item.get("text") or item.get("statement") or "unknown").strip() or "unknown" + item["text"] = text + item["evidence_ids"] = list(item.get("evidence_ids")) if isinstance(item.get("evidence_ids"), list) else [] + return item + if isinstance(candidate, str) and candidate.strip(): + return {"text": candidate.strip(), "evidence_ids": []} + return {"text": "unknown", "evidence_ids": [], "status": "unknown"} + + +def _infer_topology(spec: dict[str, Any]) -> str: + relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] + if any("feedback" in str(item.get("type") or "").casefold() for item in relations): + return "feedback" + inputs = [item for item in spec.get("inputs", []) if isinstance(item, dict)] + labels = " ".join(_item_label(item).casefold() for item in inputs) + modalities = sum(term in labels for term in ("image", "text", "audio", "video", "depth", "thermal", "imu")) + modules = [item for item in spec.get("modules", []) if isinstance(item, dict)] + encoder_count = sum("encoder" in _item_label(item).casefold() for item in modules) + if modalities >= 3 or (modalities >= 2 and encoder_count >= 2): + return "multimodal" + outgoing: dict[str, int] = {} + for relation in relations: + source = str(relation.get("source") or relation.get("source_id") or "") + outgoing[source] = outgoing.get(source, 0) + 1 + if any(value > 1 for value in outgoing.values()): + return "branch" + overview_panels = [item for item in spec.get("modules", []) if isinstance(item, dict) and str(item.get("role") or "") == "overview_panel"] + if len(overview_panels) >= 3: + return "dense_multiframe" + return "linear" if relations else "unknown" + + +def _detect_dense_overview_panels(parsed: dict[str, Any]) -> dict[str, Any]: + figures = parsed.get("document_index", {}).get("figures", []) + overview_pattern = re.compile(r"\b(?:overview|architecture|framework|pipeline|model)\b", re.IGNORECASE) + figure = next( + ( + item + for item in figures + if isinstance(item, dict) + and re.match(r"^(?:figure|fig\.)\s*1\b", str(item.get("caption") or ""), re.IGNORECASE) + and overview_pattern.search(str(item.get("caption") or "")) + ), + None, + ) + if not figure: + return {"detected": False, "page": None, "panel_labels": [], "panel_count": 0} + + page_number = int(figure.get("page") or 0) + page = next( + ( + item + for item in parsed.get("pages", []) + if isinstance(item, dict) and int(item.get("page") or 0) == page_number + ), + None, + ) + if not page: + return {"detected": False, "page": page_number, "panel_labels": [], "panel_count": 0} + + labels: set[str] = set() + for block in page.get("blocks", []): + if not isinstance(block, dict): + continue + text = re.sub(r"\s+", " ", str(block.get("text") or "")).strip() + marker = re.fullmatch(r"([a-h])", text, re.IGNORECASE) + description = re.match(r"^([a-h])\s*[,.:]\s+", text, re.IGNORECASE) + if marker: + labels.add(marker.group(1).casefold()) + if description: + labels.add(description.group(1).casefold()) + caption = re.sub(r"\s+", " ", str(figure.get("caption") or "")) + labels.update(match.group(1).casefold() for match in re.finditer(r"(?:^|[.;])\s*([a-h])\s*[,.:]\s+", caption, re.IGNORECASE)) + ordered = sorted(labels) + return { + "detected": len(ordered) >= 2, + "page": page_number, + "panel_labels": ordered, + "panel_count": len(ordered), + } + + +def _overview_caption_blocks(parsed: dict[str, Any], figure: dict[str, Any]) -> list[dict[str, Any]]: + """Recover a caption that continues across columns on the same page.""" + page_number = int(figure.get("page") or 0) + page = next( + ( + item + for item in parsed.get("pages", []) + if isinstance(item, dict) and int(item.get("page") or 0) == page_number + ), + None, + ) + if not page: + return [] + blocks = [item for item in page.get("blocks", []) if isinstance(item, dict)] + start_id = str(figure.get("block_id") or "") + start = next((item for item in blocks if str(item.get("id") or "") == start_id), None) + if not start: + return [] + start_bbox = start.get("bbox") if isinstance(start.get("bbox"), list) else [0, 0, 0, 0] + start_y = float(start_bbox[1] or 0) + page_height = max(1.0, float(page.get("height") or 1)) + max_y = start_y + max(42.0, page_height * 0.09) + start_font = float(start.get("font_size") or 0) + start_order = int(start.get("reading_order") or 0) + caption_blocks = [] + for block in blocks: + bbox = block.get("bbox") if isinstance(block.get("bbox"), list) else [0, 0, 0, 0] + block_y = float(bbox[1] or 0) + block_font = float(block.get("font_size") or 0) + if int(block.get("reading_order") or 0) < start_order: + continue + if not (start_y - 2.0 <= block_y <= max_y): + continue + if start_font and block_font and abs(block_font - start_font) > 0.8: + continue + caption_blocks.append(block) + return sorted(caption_blocks, key=lambda item: int(item.get("reading_order") or 0)) + + +def _clean_overview_panel_label(description: str) -> str: + first_sentence = re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", description).strip())[0].strip(" .") + first_sentence = re.sub( + r"^(?:flow\s+chart|diagram|schematic|illustration)\s+(?:showing|of)\s+(?:the\s+)?", + "", + first_sentence, + flags=re.IGNORECASE, + ).strip() + if len(first_sentence) > 110: + first_sentence = re.split(r"\s+(?:that|which|where|by|to)\s+", first_sentence, maxsplit=1, flags=re.IGNORECASE)[0].strip() + return first_sentence[:1].upper() + first_sentence[1:] if first_sentence else "" + + +def _extract_overview_panels(parsed: dict[str, Any], spec: dict[str, Any]) -> list[dict[str, Any]]: + figures = parsed.get("document_index", {}).get("figures", []) + figure = next( + ( + item + for item in figures + if isinstance(item, dict) + and re.match(r"^(?:figure|fig\.)\s*1\b", str(item.get("caption") or ""), re.IGNORECASE) + and re.search(r"\b(?:overview|architecture|framework|pipeline|model)\b", str(item.get("caption") or ""), re.IGNORECASE) + ), + None, + ) + if not figure: + return [] + blocks = _overview_caption_blocks(parsed, figure) + if not blocks: + return [] + evidence_by_block = { + str(item.get("block_id") or ""): str(item.get("id") or "") + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("block_id") and item.get("id") + } + marker_pattern = re.compile(r"(?:^|[.;])\s*([a-h])\s*[,.:]\s*", re.IGNORECASE) + descriptions: dict[str, list[str]] = {} + evidence_ids: dict[str, list[str]] = {} + current: str | None = None + for block in blocks: + text = re.sub(r"\s+", " ", str(block.get("text") or "")).strip() + if not text: + continue + matches = list(marker_pattern.finditer(text)) + cursor = 0 + for match in matches: + prefix = text[cursor:match.start()].strip(" ;,") + if prefix and match.group(0).lstrip().startswith((".", ";")) and not prefix.endswith((".", ";")): + prefix += match.group(0).lstrip()[0] + if current and prefix: + descriptions.setdefault(current, []).append(prefix) + evidence_id = evidence_by_block.get(str(block.get("id") or "")) + if evidence_id: + evidence_ids.setdefault(current, []).append(evidence_id) + current = match.group(1).casefold() + descriptions.setdefault(current, []) + evidence_ids.setdefault(current, []) + cursor = match.end() + suffix = text[cursor:].strip(" ;,") + if current and suffix: + descriptions.setdefault(current, []).append(suffix) + evidence_id = evidence_by_block.get(str(block.get("id") or "")) + if evidence_id: + evidence_ids.setdefault(current, []).append(evidence_id) + if len(descriptions) < 2: + return [] + + entities = [ + (field, item) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + ] + panels: list[dict[str, Any]] = [] + for label in sorted(descriptions): + description = re.sub(r"\s+", " ", " ".join(descriptions[label])).strip(" .") + visible_label = _clean_overview_panel_label(description) + if not visible_label: + continue + panel_evidence = list(dict.fromkeys(evidence_ids.get(label, []))) + panel_evidence_set = set(panel_evidence) + entity_ids = [ + str(item.get("id")) + for _, item in entities + if panel_evidence_set.intersection(str(value) for value in item.get("evidence_ids", []) if value) + ] + panels.append({ + "id": f"panel_{label}", + "panel_label": label, + "label": visible_label, + "description": description, + "page": int(figure.get("page") or 0), + "evidence_ids": panel_evidence, + "entity_ids": list(dict.fromkeys(entity_ids)), + }) + if len(panels) < 2: + return [] + + # Caption evidence and exact entity evidence often use different blocks. + # Assign otherwise-unbound entities by semantic panel role so a two-panel + # overview does not place the complete architecture into the high-level + # training/usage panel by default. + assigned_ids = {entity_id for panel in panels for entity_id in panel.get("entity_ids", [])} + architecture_terms = ("architecture", "internal", "layer", "encoder", "decoder", "network", "model structure") + high_level_terms = ("overview", "workflow", "pipeline", "pre-training", "pretraining", "training", "usage", "inference", "application") + training_terms = ("train", "pretrain", "loss", "objective", "synthetic", "supervision") + inference_terms = ("usage", "inference", "prediction", "application", "forward pass", "test") + for field, item in entities: + item_id = str(item.get("id") or "") + if not item_id or item_id in assigned_ids: + continue + label = f"{_item_label(item)} {item.get('role', '')}".casefold() + item_evidence = {str(value) for value in item.get("evidence_ids", []) if value} + scored: list[tuple[int, int]] = [] + for index, panel in enumerate(panels): + panel_text = f"{panel.get('label', '')} {panel.get('description', '')}".casefold() + score = 0 + if item_evidence.intersection(str(value) for value in panel.get("evidence_ids", []) if value): + score += 100 + normalized_label = _normalized_label(_item_label(item)) + if normalized_label and normalized_label in _normalized_label(panel_text): + score += 12 + if field == "modules": + if any(term in panel_text for term in architecture_terms): + score += 5 + if any(term in label for term in training_terms) and any(term in panel_text for term in training_terms): + score += 7 + if any(term in label for term in inference_terms) and any(term in panel_text for term in inference_terms): + score += 7 + elif field == "inputs": + if any(term in panel_text for term in high_level_terms): + score += 4 + if any(term in panel_text for term in ("input", "data", "train", "pretrain")): + score += 3 + elif field == "outputs": + if any(term in panel_text for term in high_level_terms): + score += 4 + if any(term in panel_text for term in inference_terms): + score += 3 + scored.append((score, index)) + best_score, best_index = max(scored, default=(0, 0)) + if best_score > 0: + panels[best_index]["entity_ids"].append(item_id) + assigned_ids.add(item_id) + for panel in panels: + panel["entity_ids"] = list(dict.fromkeys(panel.get("entity_ids", []))) + return panels + + +def _merge_isolated_architecture_entities(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + modules = [item for item in spec.get("modules", []) if isinstance(item, dict)] + relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] + connected_ids = { + str(value) + for relation in relations + for value in (relation.get("source"), relation.get("target")) + if str(value or "").strip() + } + evidence_by_id = { + str(item.get("id")): item + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + overview_pages = { + int(item.get("page") or 0) + for item in parsed.get("document_index", {}).get("figures", []) + if isinstance(item, dict) and re.search(r"\b(?:overview|architecture|framework|pipeline)\b", str(item.get("caption") or ""), re.IGNORECASE) + } + removed_ids: set[str] = set() + merged: list[str] = [] + for isolated in modules: + isolated_id = str(isolated.get("id") or "") + isolated_label = _item_label(isolated) + isolated_normalized = _normalized_label(isolated_label) + if not isolated_id or isolated_id in connected_ids or len(isolated_normalized) < 3: + continue + for target in modules: + target_id = str(target.get("id") or "") + target_label = _item_label(target) + target_normalized = _normalized_label(target_label) + if not target_id or target_id not in connected_ids or target_id == isolated_id or len(target_normalized) < 3: + continue + shared_ids = set(str(value) for value in isolated.get("evidence_ids", []) if value) & set(str(value) for value in target.get("evidence_ids", []) if value) + records = [evidence_by_id[value] for value in shared_ids if value in evidence_by_id] + relation_record = next( + ( + record + for record in records + if isolated_normalized in _normalized_label(str(record.get("text") or "")) + and target_normalized in _normalized_label(str(record.get("text") or "")) + and re.search(r"\b(?:based on|built on|using|uses|powered by|adapts?)\b", str(record.get("text") or ""), re.IGNORECASE) + ), + None, + ) + if not relation_record: + continue + exact_label_candidates = [ + record + for record in parsed.get("evidence", []) + if isinstance(record, dict) + and int(record.get("page") or 0) in overview_pages + and 3 <= len(str(record.get("text") or "").strip()) <= 90 + and isolated_normalized in _normalized_label(str(record.get("text") or "")) + and target_normalized in _normalized_label(str(record.get("text") or "")) + ] + exact_label_record = min( + exact_label_candidates, + key=lambda record: len(str(record.get("text") or "").strip()), + default=None, + ) + if exact_label_record: + target["name"] = re.sub(r"\s+", " ", str(exact_label_record.get("text") or "")).strip(" .") + target["evidence_ids"] = list(dict.fromkeys([*target.get("evidence_ids", []), str(exact_label_record.get("id"))])) + else: + target["name"] = f"{target_label} ({isolated_label})" + target["evidence_ids"] = list(dict.fromkeys([*target.get("evidence_ids", []), *isolated.get("evidence_ids", [])])) + removed_ids.add(isolated_id) + merged.append(f"{isolated_id}->{target_id}") + break + if removed_ids: + spec["modules"] = [item for item in modules if str(item.get("id") or "") not in removed_ids] + for relation in relations: + if str(relation.get("source") or "") in removed_ids: + relation["source"] = next(value.split("->", 1)[1] for value in merged if value.startswith(f"{relation.get('source')}->")) + if str(relation.get("target") or "") in removed_ids: + relation["target"] = next(value.split("->", 1)[1] for value in merged if value.startswith(f"{relation.get('target')}->")) + spec["relations"] = relations + return merged + + +def _supporting_visual_cue(label: str) -> str: + normalized = _normalized_label(label) + if "contrastive" in normalized: + return "paired augmented views converging on an unlabeled comparison objective" + if "mask" in normalized and ("embedding" in normalized or "token" in normalized): + return "a compact sequence of feature blocks with several visibly masked positions" + if "classificationtoken" in normalized or "classtoken" in normalized or normalized in {"cls", "clstoken"}: + return "one distinct highlighted token block among repeated token blocks" + if "patch" in normalized or "tile" in normalized or "crop" in normalized: + return "a compact group of image patches or multi-scale views" + if "embedding" in normalized: + return "a compact sequence of feature blocks" + if "loss" in normalized or "objective" in normalized: + return "an unlabeled objective or comparison cue" + return "one unlabeled panel-local scientific mechanism detail" + + +def _label_like_evidence_text(value: str) -> str | None: + text = re.sub(r"\s+", " ", str(value or "")).strip().strip(".;:") + if not 2 <= len(text) <= 64 or len(text.split()) > 9: + return None + if re.match(r"^(?:to|we|our|the|this|these|those|it|they)\b", text, re.IGNORECASE): + return None + if re.search(r"\b(?:which|that|can|could|would|should|serve as|results? in)\b", text, re.IGNORECASE): + return None + return text + + +def _defined_symbol_for_supporting_detail( + label: str, + evidence_ids: list[str], + parsed: dict[str, Any] | None, +) -> str | None: + if not parsed: + return None + evidence = [item for item in parsed.get("evidence", []) if isinstance(item, dict)] + page_by_id = {str(item.get("id")): int(item.get("page") or 0) for item in evidence if item.get("id")} + pages = {page_by_id.get(str(value), 0) for value in evidence_ids} + pages.discard(0) + normalized_label = _normalized_label(label) + if not pages or not normalized_label: + return None + for page in sorted(pages): + page_text = " ".join( + re.sub(r"\s+", " ", str(item.get("text") or "")).strip() + for item in evidence + if int(item.get("page") or 0) == page + ) + for match in re.finditer( + r"(?P\[[A-Za-z0-9_+\-]{1,16}\]|<[A-Za-z0-9_+\-]{1,16}>)\s+(?:is|denotes|represents)\s+(?:the\s+|an?\s+)?(?P[^.;]{2,80})", + page_text, + re.IGNORECASE, + ): + definition = _normalized_label(match.group("definition")) + if normalized_label in definition or definition in normalized_label: + return match.group("symbol") + return None + + +def _demote_unconnected_completion_entities( + spec: dict[str, Any], + completion_report: dict[str, Any], + parsed: dict[str, Any] | None = None, +) -> list[str]: + added_ids = {str(value) for value in completion_report.get("added_entities", []) if str(value)} + if not added_ids: + return [] + connected_ids = { + str(value) + for relation in spec.get("relations", []) if isinstance(relation, dict) + for value in (relation.get("source"), relation.get("target")) + if str(value or "").strip() + } + demoted: list[str] = [] + evidence_by_id = { + str(item.get("id")): item + for item in (parsed or {}).get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + supporting = [dict(item) for item in spec.get("supporting_details", []) if isinstance(item, dict)] + for field in ("inputs", "modules", "outputs", "innovations"): + kept = [] + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + item_id = str(item.get("id") or "") + if item_id in added_ids and item_id not in connected_ids: + supporting.append({ + **item, + "original_field": field, + "original_role": item.get("role"), + "role": "panel_local_supporting_detail", + "visibility": "unlabeled_visual_detail_not_global_node", + }) + demoted.append(item_id) + continue + kept.append(item) + spec[field] = kept + normalized_supporting: list[dict[str, Any]] = [] + by_key: dict[str, dict[str, Any]] = {} + for item in supporting: + exact_candidates = [ + _label_like_evidence_text(str(evidence_by_id[evidence_id].get("text") or "")) + for evidence_id in (str(value) for value in item.get("evidence_ids", []) if value) + if evidence_id in evidence_by_id + ] + exact_candidates = [value for value in exact_candidates if value] + if evidence_by_id and not exact_candidates: + continue + if exact_candidates: + item["name"] = min(exact_candidates, key=len) + label = _item_label(item) + if not label: + continue + item["visual_cue"] = _supporting_visual_cue(label) + item["visibility"] = "unlabeled_visual_detail_not_global_node" + optional_symbol = _defined_symbol_for_supporting_detail( + label, + [str(value) for value in item.get("evidence_ids", []) if value], + parsed, + ) + if optional_symbol: + item["optional_visible_label"] = optional_symbol + key = str(item.get("id") or "").strip() or _normalized_label(label) + label_key = _normalized_label(label) + existing = by_key.get(key) or by_key.get(label_key) + if existing: + existing["evidence_ids"] = list(dict.fromkeys([ + *existing.get("evidence_ids", []), + *item.get("evidence_ids", []), + ])) + continue + normalized_supporting.append(item) + by_key[key] = item + if label_key: + by_key[label_key] = item + spec["supporting_details"] = normalized_supporting + return demoted + + +def _complete_from_overview_caption(spec: dict[str, Any], parsed: dict[str, Any]) -> None: + figures = parsed.get("document_index", {}).get("figures", []) + figure = next((item for item in figures if re.match(r"^(figure|fig\.)\s*1\b", str(item.get("caption") or ""), re.IGNORECASE) and re.search(r"\b(components?|overview|framework|architecture|pipeline)\b", str(item.get("caption") or ""), re.IGNORECASE)), None) + figure = figure or next((item for item in figures if re.search(r"\b(overview|framework|architecture|pipeline)\b", str(item.get("caption") or ""), re.IGNORECASE)), None) + figure = figure or next((item for item in figures if re.match(r"^(figure|fig\.)\s*1\b", str(item.get("caption") or ""), re.IGNORECASE)), None) + if not figure: + return + caption = str(figure.get("caption") or "") + caption_evidence = next((item["id"] for item in parsed.get("evidence", []) if item.get("kind") == "caption" and item.get("page") == figure.get("page") and str(item.get("text") or "").startswith(str(caption)[:60])), None) + evidence_ids = [caption_evidence] if caption_evidence else [] + match = re.search(r"(?:interconnected\s+)?components\s*:\s*(.+)", caption, re.IGNORECASE) + component_ids: dict[str, str] = {} + if match: + raw_components = re.split(r",\s+(?=(?:and\s+)?(?:a|an)\s+)", match.group(1)) + existing = {_normalized_label(_item_label(item)) for field in ("modules", "inputs", "outputs", "innovations") for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) if isinstance(item, dict)} + for index, raw in enumerate(raw_components): + phrase = re.sub(r"^(?:and\s+)?(?:a|an)\s+", "", raw.strip(), flags=re.IGNORECASE) + phrase = re.split(r"\s+(?:that|which|for|by|to)\s+", phrase, maxsplit=1, flags=re.IGNORECASE)[0].strip(" .") + if not 3 <= len(phrase) <= 80: + continue + label = phrase[0].upper() + phrase[1:] + normalized = _normalized_label(label) + existing_match = next((item for item in spec.get("modules", []) if isinstance(item, dict) and normalized and (normalized in _normalized_label(_item_label(item)) or _normalized_label(_item_label(item)) in normalized)), None) + if existing_match: + component_ids[normalized] = str(existing_match.get("id")) + continue + if normalized not in existing: + item_id = _stable_id("overview", label, index) + spec.setdefault("modules", []).append({"id": item_id, "name": label, "role": "overview_panel", "evidence_ids": evidence_ids}) + existing.add(normalized) + component_ids[normalized] = item_id + dataset_match = re.search(r"\b([A-Z]{2,}[A-Z0-9-]*\d[A-Z0-9-]*)\b", caption) if re.search(r"\bdataset\b|data engine|collecting", caption, re.IGNORECASE) else None + dataset_item = None + if dataset_match: + dataset_label = dataset_match.group(1) + dataset_item = next((item for item in spec.get("outputs", []) if isinstance(item, dict) and dataset_label.casefold() in _item_label(item).casefold()), None) + if not dataset_item: + dataset_item = {"id": _stable_id("dataset", dataset_label, 0), "name": dataset_label, "role": "dataset", "evidence_ids": evidence_ids} + spec.setdefault("outputs", []).append(dataset_item) + model_item = next((item for item in spec.get("modules", []) if isinstance(item, dict) and ("segmentation model" in _item_label(item).casefold() or "segment anything model" in _item_label(item).casefold())), None) + model_item = model_item or next((item for item in spec.get("modules", []) if isinstance(item, dict) and ("model" in _item_label(item).casefold() or "sam" in _item_label(item).casefold())), None) + engine_item = next((item for item in spec.get("modules", []) if isinstance(item, dict) and "data engine" in _item_label(item).casefold()), None) + relations = spec.setdefault("relations", []) + pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + if model_item and engine_item and (str(model_item.get("id")), str(engine_item.get("id"))) not in pairs and re.search(r"powers?\s+data\s+annotation|data\s+engine", caption, re.IGNORECASE): + relations.append({"source": str(model_item.get("id")), "target": str(engine_item.get("id")), "type": "annotation_support", "label": "", "evidence_ids": evidence_ids}) + pairs.add((str(model_item.get("id")), str(engine_item.get("id")))) + evidence = parsed.get("evidence", []) + for index, item in enumerate(evidence): + if not re.search(r"has\s+(?:three|3)\s+stages", str(item.get("text") or ""), re.IGNORECASE): + continue + combined = " ".join(str(value.get("text") or "") for value in evidence[index:index + 3]) + stage_match = re.search(r"([A-Za-z]+(?:-[A-Za-z]+)?)\s*,\s*([A-Za-z]+(?:-[A-Za-z]+)?)\s*,\s*(?:and\s+)?([A-Za-z]+(?:\s+[A-Za-z]+)?)\s*\.", combined) + if not stage_match: + continue + stage_evidence = [value["id"] for value in evidence[index:index + 3]] + stage_items = [] + for stage_index, value in enumerate(stage_match.groups()): + label = value.strip().title().replace("-", "-") + existing_stage = next((module for module in spec.get("modules", []) if isinstance(module, dict) and _normalized_label(label) == _normalized_label(_item_label(module))), None) + if existing_stage: + stage_items.append(existing_stage) + else: + stage = {"id": _stable_id("stage", label, stage_index), "name": label, "role": "data_engine_stage", "evidence_ids": stage_evidence} + spec.setdefault("modules", []).append(stage) + stage_items.append(stage) + for source, target in zip(stage_items, stage_items[1:]): + pair = (str(source.get("id")), str(target.get("id"))) + if pair not in pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "stage_transition", "label": "", "evidence_ids": stage_evidence}) + pairs.add(pair) + if dataset_item and stage_items: + pair = (str(stage_items[-1].get("id")), str(dataset_item.get("id"))) + if pair not in pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "data_generation", "label": "", "evidence_ids": stage_evidence + evidence_ids}) + break + + +def _normalize_contract_entities(raw_items: Any, field: str) -> list[dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + for index, raw in enumerate(raw_items if isinstance(raw_items, list) else []): + if isinstance(raw, str) and raw.strip(): + item = {"name": raw.strip(), "evidence_ids": []} + elif isinstance(raw, dict): + item = dict(raw) + else: + continue + label = _item_label(item) + if not label: + continue + item["id"] = str(item.get("id") or _stable_id(field.rstrip("s"), label, index)).strip() + item["evidence_ids"] = list(item.get("evidence_ids")) if isinstance(item.get("evidence_ids"), list) else [] + normalized.append(item) + return normalized + + +def _normalize_panel_graphs(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: + evidence_ids = { + str(item.get("id")) + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + normalized_panels: list[dict[str, Any]] = [] + removed_nodes: list[str] = [] + removed_relations: list[str] = [] + for panel_index, raw_panel in enumerate(spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(raw_panel, dict): + continue + panel_id = str(raw_panel.get("id") or f"panel_{panel_index + 1}").strip() + panel_label = str(raw_panel.get("panel_label") or chr(ord("a") + panel_index)).strip() + label = str(raw_panel.get("label") or raw_panel.get("description") or panel_label).strip() + if not panel_id or not label: + continue + nodes: list[dict[str, Any]] = [] + seen_instance_ids: set[str] = set() + for node_index, raw_node in enumerate(raw_panel.get("nodes", []) if isinstance(raw_panel.get("nodes"), list) else []): + if not isinstance(raw_node, dict): + continue + node = dict(raw_node) + node_label = _item_label(node) + instance_id = str(node.get("instance_id") or node.get("id") or _stable_id(f"{panel_id}_instance", node_label, node_index)).strip() + if not node_label or not instance_id or instance_id in seen_instance_ids: + removed_nodes.append(f"{panel_id}:{instance_id or node_index}") + continue + node["instance_id"] = instance_id + node["name"] = node_label + node["role"] = str(node.get("role") or "module").strip() + node["evidence_ids"] = [ + str(value) for value in node.get("evidence_ids", []) + if str(value) in evidence_ids + ] + nodes.append(node) + seen_instance_ids.add(instance_id) + node_by_id = {str(item.get("instance_id")): item for item in nodes} + relations: list[dict[str, Any]] = [] + seen_relations: set[tuple[str, str, str]] = set() + for raw_relation in raw_panel.get("relations", []) if isinstance(raw_panel.get("relations"), list) else []: + if not isinstance(raw_relation, dict): + continue + source = str(raw_relation.get("source") or raw_relation.get("source_id") or "").strip() + target = str(raw_relation.get("target") or raw_relation.get("target_id") or "").strip() + relation_type = str(raw_relation.get("type") or "data_flow").strip() + key = (source, target, relation_type) + if not source or not target or source == target or source not in node_by_id or target not in node_by_id or key in seen_relations: + removed_relations.append(f"{panel_id}:{source}->{target}:{relation_type}") + continue + relation = dict(raw_relation) + relation["source"] = source + relation["target"] = target + relation["type"] = relation_type + relation["label"] = str(relation.get("label") or "").strip() + relation["evidence_ids"] = [ + str(value) for value in relation.get("evidence_ids", []) + if str(value) in evidence_ids + ] + if not relation["evidence_ids"]: + relation["evidence_ids"] = list(dict.fromkeys([ + *node_by_id[source].get("evidence_ids", []), + *node_by_id[target].get("evidence_ids", []), + ])) + relations.append(relation) + seen_relations.add(key) + normalized_panels.append({ + "id": panel_id, + "panel_label": panel_label, + "label": label, + "description": str(raw_panel.get("description") or label).strip(), + "evidence_ids": [str(value) for value in raw_panel.get("evidence_ids", []) if str(value) in evidence_ids], + "nodes": nodes, + "relations": relations, + }) + spec["panel_graphs"] = normalized_panels + return { + "panel_count": len(normalized_panels), + "node_count": sum(len(item["nodes"]) for item in normalized_panels), + "relation_count": sum(len(item["relations"]) for item in normalized_panels), + "removed_nodes": removed_nodes, + "removed_relations": removed_relations, + } + + +def _complete_panel_graphs_from_inventory(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: + try: + inventory = json.loads(overview_figure_panel_excerpt(parsed)) + except (TypeError, ValueError, json.JSONDecodeError): + inventory = {} + graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict)] if isinstance(spec.get("panel_graphs"), list) else [] + panels = [item for item in inventory.get("panels", []) if isinstance(item, dict)] if isinstance(inventory, dict) else [] + if len(graphs) < 2 or not panels: + return {"applied": False, "added_nodes": [], "rebuilt_panels": [], "required_candidate_coverage": None} + + added_nodes: list[str] = [] + removed_nodes: list[str] = [] + rebuilt_panels: list[str] = [] + required_total = 0 + covered_total = 0 + + def center(node: dict[str, Any]) -> tuple[float, float]: + bbox = node.get("bbox_percent") if isinstance(node.get("bbox_percent"), list) and len(node.get("bbox_percent")) == 4 else [0.0, 0.0, 0.0, 0.0] + return (float(bbox[0]) + float(bbox[2])) / 2.0, (float(bbox[1]) + float(bbox[3])) / 2.0 + + def relation(source: dict[str, Any], target: dict[str, Any], relation_type: str) -> dict[str, Any]: + return { + "source": str(source.get("instance_id")), + "target": str(target.get("instance_id")), + "type": relation_type, + "label": "", + "evidence_ids": list(dict.fromkeys([*source.get("evidence_ids", []), *target.get("evidence_ids", [])])), + } + + inventory_by_label = {str(item.get("panel_label") or "").casefold(): item for item in panels} + for graph in graphs: + panel_label = str(graph.get("panel_label") or "").casefold() + panel_inventory = inventory_by_label.get(panel_label) + if not panel_inventory: + continue + required_items = [item for item in panel_inventory.get("label_inventory", []) if isinstance(item, dict) and item.get("required_candidate")] + required_evidence_ids = {str(item.get("evidence_id") or "") for item in required_items if item.get("evidence_id")} + nodes = [] + for item in graph.get("nodes", []) if isinstance(graph.get("nodes"), list) else []: + if not isinstance(item, dict): + continue + source_evidence_id = str(item.get("source_evidence_id") or "") + if source_evidence_id and source_evidence_id not in required_evidence_ids: + removed_nodes.append(f"{graph.get('id')}:{item.get('instance_id')}") + continue + nodes.append(item) + by_evidence = { + str(evidence_id): node + for node in nodes + for evidence_id in node.get("evidence_ids", []) + if evidence_id + } + required_total += len(required_items) + occurrence_counts: dict[str, int] = {} + for item in required_items: + evidence_id = str(item.get("evidence_id") or "") + text_value = str(item.get("text") or "").strip() + role = str(item.get("role_hint") or "module").strip() + existing = by_evidence.get(evidence_id) + if existing: + existing["name"] = text_value + existing["role"] = role + existing["bbox_percent"] = item.get("bbox_percent") + existing["source_evidence_id"] = evidence_id + covered_total += 1 + continue + key = _normalized_label(text_value) or role + occurrence_counts[key] = occurrence_counts.get(key, 0) + 1 + instance_id = f"{graph.get('id')}_{key[:32]}_{occurrence_counts[key]}" + while any(str(node.get("instance_id")) == instance_id for node in nodes): + occurrence_counts[key] += 1 + instance_id = f"{graph.get('id')}_{key[:32]}_{occurrence_counts[key]}" + node = { + "instance_id": instance_id, + "name": text_value, + "role": role, + "evidence_ids": [evidence_id] if evidence_id else [], + "bbox_percent": item.get("bbox_percent"), + "source_evidence_id": evidence_id, + } + nodes.append(node) + if evidence_id: + by_evidence[evidence_id] = node + added_nodes.append(f"{graph.get('id')}:{instance_id}") + covered_total += 1 + + objectives = [item for item in nodes if str(item.get("role") or "").casefold() == "objective"] + if objectives: + objective_x = min(center(item)[0] for item in objectives) + for node in nodes: + if _normalized_label(_item_label(node)) == "ytest" and center(node)[0] < objective_x: + node["role"] = "target" + + groups = [item for item in nodes if str(item.get("role") or "").casefold() == "group"] + modules = [item for item in nodes if str(item.get("role") or "").casefold() == "module"] + inputs = [item for item in nodes if str(item.get("role") or "").casefold() == "input"] + targets = [item for item in nodes if str(item.get("role") or "").casefold() in {"target", "conditioning"}] + outputs = [item for item in nodes if str(item.get("role") or "").casefold() == "output"] + inferred: list[dict[str, Any]] = [] + for input_node in inputs: + input_x, input_y = center(input_node) + candidates = [item for item in modules if center(item)[0] >= input_x - 0.02] + candidates = candidates or modules + if candidates: + target_node = min(candidates, key=lambda item: abs(center(item)[0] - input_x) + 0.45 * abs(center(item)[1] - input_y)) + inferred.append(relation(input_node, target_node, "data_flow")) + ordered_modules = sorted(modules, key=lambda item: (center(item)[0], center(item)[1])) + for source, target in zip(ordered_modules, ordered_modules[1:]): + source_label = _normalized_label(_item_label(source)) + target_label = _normalized_label(_item_label(target)) + source_x, source_y = center(source) + target_x, target_y = center(target) + if source_label == target_label or target_x <= source_x or abs(target_y - source_y) > 0.22: + continue + inferred.append(relation(source, target, "data_flow")) + for output_node in outputs: + output_x, output_y = center(output_node) + candidates = [item for item in modules if center(item)[0] <= output_x + 0.02] + candidates = candidates or modules + if candidates: + source_node = min(candidates, key=lambda item: abs(output_x - center(item)[0]) + 0.45 * abs(output_y - center(item)[1])) + inferred.append(relation(source_node, output_node, "prediction")) + for objective in objectives: + objective_x, objective_y = center(objective) + sources = [ + item for item in [*outputs, *targets] + if abs(center(item)[0] - objective_x) + abs(center(item)[1] - objective_y) <= 0.38 + ] + if not sources and modules: + sources = [min(modules, key=lambda item: abs(center(item)[0] - objective_x) + abs(center(item)[1] - objective_y))] + for source in sources: + inferred.append(relation(source, objective, "training_objective")) + seen: set[tuple[str, str, str]] = set() + deduplicated = [] + for item in inferred: + key = (str(item.get("source")), str(item.get("target")), str(item.get("type"))) + if key in seen or key[0] == key[1]: + continue + seen.add(key) + deduplicated.append(item) + if deduplicated: + graph["relations"] = deduplicated + rebuilt_panels.append(str(graph.get("id"))) + graph["nodes"] = nodes + graph["group_instance_ids"] = [str(item.get("instance_id")) for item in groups] + + return { + "applied": bool(rebuilt_panels or added_nodes), + "added_nodes": added_nodes, + "removed_nodes": removed_nodes, + "rebuilt_panels": rebuilt_panels, + "required_candidate_count": required_total, + "covered_required_candidate_count": covered_total, + "required_candidate_coverage": round(covered_total / max(1, required_total), 4) if required_total else None, + } + + +def _deduplicate_contract_entities(spec: dict[str, Any]) -> dict[str, list[str]]: + replacements: dict[str, str] = {} + removed: list[str] = [] + for field in ("inputs", "modules", "outputs", "innovations"): + deduplicated: list[dict[str, Any]] = [] + local_by_label: dict[str, dict[str, Any]] = {} + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + label = _normalized_label(_item_label(item)) + existing = local_by_label.get(label) if label else None + if existing: + existing["evidence_ids"] = list(dict.fromkeys(list(existing.get("evidence_ids", [])) + list(item.get("evidence_ids", [])))) + replacements[str(item.get("id") or "")] = str(existing.get("id") or "") + removed.append(f"{field}:{item.get('id')}->{existing.get('id')}") + continue + deduplicated.append(item) + if label: + local_by_label[label] = item + spec[field] = deduplicated + + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + for key in ("source", "source_id", "target", "target_id"): + value = str(relation.get(key) or "") + if value in replacements: + relation[key] = replacements[value] + return {"removed": removed, "replacements": [f"{source}->{target}" for source, target in replacements.items() if source and target]} + + +def _simplify_branch_contract(spec: dict[str, Any]) -> dict[str, list[str]]: + if str(spec.get("topology") or "") != "branch": + return {"merged_entities": [], "removed_cross_branch_relations": [], "removed_branch_shortcuts": []} + + def group_for(item: dict[str, Any]) -> str | None: + label = _normalized_label(_item_label(item)) + if label in {"rpn", "regionproposals", "regionproposalnetwork", "regionproposalnetworkrpn", "proposals"}: + return "proposals" + if any(value in label for value in ("classificationbranch", "classificationhead", "classprediction", "classpredictions")) or label == "classification": + return "classification" + if any(value in label for value in ("boundingboxregression", "boxregression", "boxbranch", "boundingboxoffset", "boundingboxpredictions")): + return "box" + if label in {"maskbranch", "binarymask", "validsegmentationmask", "maskprediction"}: + return "mask" + return None + + preference = { + "proposals": ("rpn", "regionproposals"), + "classification": ("classification", "classificationbranch", "classprediction"), + "box": ("boundingboxregression", "boxbranch", "boundingboxpredictions", "boundingboxoffset"), + "mask": ("maskbranch", "validsegmentationmask", "binarymask"), + } + records: list[tuple[str, dict[str, Any], str]] = [] + for field in ("inputs", "modules", "outputs"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if isinstance(item, dict) and (group := group_for(item)): + records.append((field, item, group)) + + replacements: dict[str, str] = {} + removed_ids: set[str] = set() + merged_entities: list[str] = [] + for group in ("proposals", "classification", "box", "mask"): + members = [(field, item) for field, item, item_group in records if item_group == group] + if len(members) <= 1: + continue + + def rank(record: tuple[str, dict[str, Any]]) -> tuple[int, int]: + field, item = record + label = _normalized_label(_item_label(item)) + preferred = preference[group] + label_rank = next((index for index, value in enumerate(preferred) if label == value), len(preferred)) + field_rank = 0 if (group == "classification" and field == "outputs") or (group != "classification" and field == "modules") else 1 + return label_rank, field_rank + + canonical_field, canonical = min(members, key=rank) + canonical_id = str(canonical.get("id") or "") + for field, item in members: + item_id = str(item.get("id") or "") + if item is canonical: + continue + canonical["evidence_ids"] = list(dict.fromkeys([*canonical.get("evidence_ids", []), *item.get("evidence_ids", [])])) + if item_id and canonical_id: + replacements[item_id] = canonical_id + removed_ids.add(item_id) + merged_entities.append(f"{item_id}->{canonical_id}") + + if removed_ids: + for field in ("inputs", "modules", "outputs"): + spec[field] = [item for item in spec.get(field, []) if not (isinstance(item, dict) and str(item.get("id") or "") in removed_ids)] + + removed_cross_branch_relations: list[str] = [] + deduplicated: list[dict[str, Any]] = [] + seen_relations: dict[tuple[str, str, str], dict[str, Any]] = {} + endpoints = { + str(item.get("id")): item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + source = replacements.get(str(relation.get("source") or ""), str(relation.get("source") or "")) + target = replacements.get(str(relation.get("target") or ""), str(relation.get("target") or "")) + if not source or not target or source == target or source not in endpoints or target not in endpoints: + continue + source_group = group_for(endpoints[source]) + target_group = group_for(endpoints[target]) + branch_groups = {"classification", "box", "mask"} + if source_group in branch_groups and target_group in branch_groups and source_group != target_group: + removed_cross_branch_relations.append(f"{source}->{target}") + continue + relation["source"], relation["target"] = source, target + key = (source, target, str(relation.get("type") or "data_flow")) + if key in seen_relations: + existing = seen_relations[key] + existing["evidence_ids"] = list(dict.fromkeys([*existing.get("evidence_ids", []), *relation.get("evidence_ids", [])])) + continue + seen_relations[key] = relation + deduplicated.append(relation) + branch_head_ids = { + str(item.get("id")) + for item in spec.get("outputs", []) if isinstance(spec.get("outputs"), list) + if isinstance(item, dict) + and item.get("id") + and (group_for(item) in branch_groups or "outputhead" in _normalized_label(str(item.get("role") or ""))) + } + source_head_counts: dict[str, set[str]] = {} + for relation in deduplicated: + source, target = str(relation.get("source") or ""), str(relation.get("target") or "") + if target in branch_head_ids: + source_head_counts.setdefault(source, set()).add(target) + branch_source = next( + ( + source for source, targets in sorted(source_head_counts.items(), key=lambda item: (-len(item[1]), item[0])) + if len(targets) >= 2 + ), + None, + ) + removed_branch_shortcuts: list[str] = [] + if branch_source: + adjacency: dict[str, set[str]] = {} + for relation in deduplicated: + source, target = str(relation.get("source") or ""), str(relation.get("target") or "") + if source and target and target not in branch_head_ids: + adjacency.setdefault(source, set()).add(target) + + def reaches_branch_source(start: str) -> bool: + pending = [start] + visited: set[str] = set() + while pending: + current = pending.pop() + if current == branch_source: + return True + if current in visited: + continue + visited.add(current) + pending.extend(adjacency.get(current, set()) - visited) + return False + + direct_branch_pairs = { + (str(item.get("source") or ""), str(item.get("target") or "")) + for item in deduplicated + } + kept_relations: list[dict[str, Any]] = [] + for relation in deduplicated: + source, target = str(relation.get("source") or ""), str(relation.get("target") or "") + bypasses_branch_point = ( + target in branch_head_ids + and source != branch_source + and (branch_source, target) in direct_branch_pairs + and reaches_branch_source(source) + ) + if bypasses_branch_point: + removed_branch_shortcuts.append(f"{source}->{target}") + continue + kept_relations.append(relation) + deduplicated = kept_relations + + spec["relations"] = deduplicated + return { + "merged_entities": merged_entities, + "removed_cross_branch_relations": removed_cross_branch_relations, + "removed_branch_shortcuts": list(dict.fromkeys(removed_branch_shortcuts)), + } + + +def _simplify_multimodal_contract(spec: dict[str, Any]) -> dict[str, Any]: + modality_names = {"image", "images", "text", "audio", "video", "depth", "thermal", "imu"} + modality_inputs = [] + modality_by_name: dict[str, dict[str, Any]] = {} + for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: + if not isinstance(item, dict): + continue + normalized = _normalized_label(_item_label(item)) + if normalized not in modality_names: + continue + key = "image" if normalized == "images" else normalized + if key in modality_by_name: + canonical = modality_by_name[key] + canonical["evidence_ids"] = list(dict.fromkeys([*canonical.get("evidence_ids", []), *item.get("evidence_ids", [])])) + continue + modality_by_name[key] = item + modality_inputs.append(item) + all_entities = [ + (field, item) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + encoder_candidates = [ + (field, item) for field, item in all_entities + if "modalityencoder" in _normalized_label(_item_label(item)) or "encodersforeachmodality" in _normalized_label(_item_label(item)) + ] + joint_candidates = [ + (field, item) for field, item in all_entities + if "embedding" in _normalized_label(_item_label(item)) + and any(term in _normalized_label(_item_label(item)) for term in ("joint", "common", "shared")) + ] + emergent_candidates = [ + (field, item) for field, item in all_entities + if ("emergent" in _normalized_label(_item_label(item)) and "align" in _normalized_label(_item_label(item))) + or "bindingproperty" in _normalized_label(_item_label(item)) + or "bindingagent" in _normalized_label(_item_label(item)) + ] + if len(modality_inputs) < 3 or not encoder_candidates or not joint_candidates or not emergent_candidates: + return {"applied": False, "removed_entities": [], "rebuilt_relations": []} + + def choose(candidates: list[tuple[str, dict[str, Any]]], preferred: tuple[str, ...]) -> tuple[str, dict[str, Any]]: + return min( + candidates, + key=lambda record: next( + (index for index, value in enumerate(preferred) if _normalized_label(_item_label(record[1])) == value), + len(preferred), + ), + ) + + _, encoder = choose(encoder_candidates, ("modalityencoders", "encodersforeachmodality")) + _, joint = choose(joint_candidates, ("jointembeddingspace", "singlejointembeddingspace", "commonembeddingspace", "sharedrepresentationspace")) + _, emergent = choose(emergent_candidates, ("emergentcrossmodalalignment", "emergentalignment", "bindingproperty", "bindingagent")) + encoder["name"] = "Modality Encoders" + encoder["role"] = "module group" + joint["name"] = "Joint Embedding Space" + joint["role"] = "shared representation" + emergent["name"] = "Emergent Cross-modal Alignment" + emergent["role"] = "output" + + for canonical, candidates in ((encoder, encoder_candidates), (joint, joint_candidates), (emergent, emergent_candidates)): + canonical["evidence_ids"] = list(dict.fromkeys( + value for _, item in candidates for value in item.get("evidence_ids", []) + )) + + keep_ids = {str(item.get("id")) for item in modality_inputs + [encoder, joint, emergent] if item.get("id")} + removed_entities = [ + str(item.get("id")) for _, item in all_entities if item.get("id") and str(item.get("id")) not in keep_ids + ] + spec["inputs"] = modality_inputs + spec["modules"] = [encoder, joint] + spec["outputs"] = [emergent] + spec["innovations"] = [] + + encoder_id = str(encoder.get("id")) + joint_id = str(joint.get("id")) + emergent_id = str(emergent.get("id")) + relations = [] + for item in modality_inputs: + relations.append({ + "source": str(item.get("id")), + "target": encoder_id, + "type": "encoding", + "label": "", + "evidence_ids": list(dict.fromkeys([*item.get("evidence_ids", []), *encoder.get("evidence_ids", [])])), + }) + relations.extend([ + { + "source": encoder_id, + "target": joint_id, + "type": "alignment", + "label": "", + "evidence_ids": list(dict.fromkeys([*encoder.get("evidence_ids", []), *joint.get("evidence_ids", [])])), + }, + { + "source": joint_id, + "target": emergent_id, + "type": "enables", + "label": "", + "evidence_ids": list(dict.fromkeys([*joint.get("evidence_ids", []), *emergent.get("evidence_ids", [])])), + }, + ]) + spec["relations"] = relations + spec["topology"] = "multimodal" + return { + "applied": True, + "removed_entities": removed_entities, + "rebuilt_relations": [f"{item['source']}->{item['target']}" for item in relations], + } + + +def _simplify_dense_multiframe_contract(spec: dict[str, Any]) -> dict[str, Any]: + all_entities = [ + (field, item) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + + def candidates(predicate: Callable[[str, dict[str, Any]], bool]) -> list[tuple[str, dict[str, Any]]]: + return [(field, item) for field, item in all_entities if predicate(_normalized_label(_item_label(item)), item)] + + promptable = candidates(lambda label, item: "segmentationtask" in label and "prompt" in label) + sam_model = candidates(lambda label, item: "segmentanythingmodel" in label or "segmentationmodelsam" in label or label == "sam") + data_engine = candidates(lambda label, item: label == "dataengine") + image = candidates(lambda label, item: label in {"image", "images", "inputimage"} and item in spec.get("inputs", [])) + prompt = candidates(lambda label, item: label in {"prompt", "inputprompts", "segmentationprompt", "sparseprompts", "denseprompts"} and item in spec.get("inputs", [])) + image_encoder = candidates(lambda label, item: label.endswith("imageencoder") and "prompt" not in label) + prompt_encoder = candidates(lambda label, item: label.endswith("promptencoder")) + mask_decoder = candidates(lambda label, item: label.endswith("maskdecoder")) + valid_mask = candidates(lambda label, item: label in {"validsegmentationmask", "validmask", "validmasks"}) + assisted = candidates(lambda label, item: label == "assistedmanual") + semi = candidates(lambda label, item: label == "semiautomatic") + fully = candidates(lambda label, item: label == "fullyautomatic") + sa1b = candidates(lambda label, item: label == "sa1b") + groups = (promptable, sam_model, data_engine, image, prompt, image_encoder, prompt_encoder, mask_decoder, valid_mask, assisted, semi, fully, sa1b) + if any(not group for group in groups): + return {"applied": False, "removed_entities": [], "rebuilt_relations": []} + + def merge(group: list[tuple[str, dict[str, Any]]], name: str, role: str) -> dict[str, Any]: + canonical = group[0][1] + canonical["name"] = name + canonical["role"] = role + canonical["evidence_ids"] = list(dict.fromkeys(value for _, item in group for value in item.get("evidence_ids", []))) + return canonical + + promptable_item = merge(promptable, "Promptable Segmentation Task", "overview_panel") + sam_item = merge(sam_model, "Segment Anything Model", "overview_panel") + data_engine_item = merge(data_engine, "Data Engine", "overview_panel") + image_item = merge(image, "Image", "input") + prompt_item = merge(prompt, "Prompt", "input") + image_encoder_item = merge(image_encoder, "Image Encoder", "module") + prompt_encoder_item = merge(prompt_encoder, "Prompt Encoder", "module") + mask_decoder_item = merge(mask_decoder, "Mask Decoder", "module") + valid_mask_item = merge(valid_mask, "Valid Segmentation Mask", "output") + assisted_item = merge(assisted, "Assisted-manual", "data_engine_stage") + semi_item = merge(semi, "Semi-automatic", "data_engine_stage") + fully_item = merge(fully, "Fully Automatic", "data_engine_stage") + sa1b_item = merge(sa1b, "SA-1B", "dataset") + + kept = [ + promptable_item, sam_item, data_engine_item, image_item, prompt_item, + image_encoder_item, prompt_encoder_item, mask_decoder_item, valid_mask_item, + assisted_item, semi_item, fully_item, sa1b_item, + ] + keep_ids = {str(item.get("id")) for item in kept if item.get("id")} + removed_entities = [str(item.get("id")) for _, item in all_entities if item.get("id") and str(item.get("id")) not in keep_ids] + spec["inputs"] = [image_item, prompt_item] + spec["modules"] = [ + promptable_item, sam_item, data_engine_item, + image_encoder_item, prompt_encoder_item, mask_decoder_item, + assisted_item, semi_item, fully_item, + ] + spec["outputs"] = [valid_mask_item, sa1b_item] + spec["innovations"] = [] + + def relation(source: dict[str, Any], target: dict[str, Any], relation_type: str) -> dict[str, Any]: + return { + "source": str(source.get("id")), + "target": str(target.get("id")), + "type": relation_type, + "label": "", + "evidence_ids": list(dict.fromkeys([*source.get("evidence_ids", []), *target.get("evidence_ids", [])])), + } + + relations = [ + relation(image_item, image_encoder_item, "data_flow"), + relation(prompt_item, prompt_encoder_item, "data_flow"), + relation(image_encoder_item, mask_decoder_item, "feature_flow"), + relation(prompt_encoder_item, mask_decoder_item, "conditioning"), + relation(mask_decoder_item, valid_mask_item, "data_flow"), + relation(assisted_item, semi_item, "stage_transition"), + relation(semi_item, fully_item, "stage_transition"), + relation(fully_item, sa1b_item, "data_generation"), + relation(sam_item, data_engine_item, "annotation_support"), + ] + spec["relations"] = relations + spec["topology"] = "dense_multiframe" + return { + "applied": True, + "removed_entities": removed_entities, + "rebuilt_relations": [f"{item['source']}->{item['target']}" for item in relations], + } + + +def _simplify_feedback_contract(spec: dict[str, Any]) -> dict[str, Any]: + if str(spec.get("topology") or "") != "feedback": + return {"applied": False, "removed_entities": [], "rebuilt_relations": []} + all_entities = [ + (field, item) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + + def find(labels: set[str]) -> list[tuple[str, dict[str, Any]]]: + return [(field, item) for field, item in all_entities if _normalized_label(_item_label(item)) in labels] + + groups = { + "input": find({"input", "inputx", "taskprompt", "taskinstruction"}), + "generate": find({"generate", "init", "generator"}), + "initial": find({"initialoutput"}), + "feedback": find({"feedback", "feedbackprovider"}), + "self_feedback": find({"selffeedback", "selffeedbacknl"}), + "refine": find({"refine", "iteraterefine", "refiner"}), + "refined": find({"refinedoutput"}), + "model": find({"modelm"}), + } + if any(not groups[key] for key in ("input", "generate", "initial", "feedback", "self_feedback", "refine", "refined")): + return {"applied": False, "removed_entities": [], "rebuilt_relations": []} + + def merge(group: list[tuple[str, dict[str, Any]]], name: str, role: str) -> dict[str, Any]: + canonical = group[0][1] + canonical["name"] = name + canonical["role"] = role + canonical["evidence_ids"] = list(dict.fromkeys(value for _, item in group for value in item.get("evidence_ids", []))) + return canonical + + input_item = merge(groups["input"], "input x", "input") + generate_item = merge(groups["generate"], "Generate", "generator") + initial_item = merge(groups["initial"], "Initial Output", "artifact") + feedback_item = merge(groups["feedback"], "FEEDBACK", "feedback module") + self_feedback_item = merge(groups["self_feedback"], "Self-Feedback", "feedback artifact") + refine_item = merge(groups["refine"], "REFINE", "refiner") + refined_item = merge(groups["refined"], "Refined output", "output") + model_item = merge(groups["model"], "Model M", "shared model") if groups["model"] else None + kept = [input_item, generate_item, initial_item, feedback_item, self_feedback_item, refine_item, refined_item] + if model_item: + kept.append(model_item) + keep_ids = {str(item.get("id")) for item in kept if item.get("id")} + removed_entities = [str(item.get("id")) for _, item in all_entities if item.get("id") and str(item.get("id")) not in keep_ids] + spec["inputs"] = [input_item] + spec["modules"] = [item for item in (model_item, generate_item, initial_item, feedback_item, self_feedback_item, refine_item) if item] + spec["outputs"] = [refined_item] + spec["innovations"] = [] + + def relation(source: dict[str, Any], target: dict[str, Any], relation_type: str) -> dict[str, Any]: + return { + "source": str(source.get("id")), + "target": str(target.get("id")), + "type": relation_type, + "label": "iterate" if relation_type == "feedback_loop" else "", + "evidence_ids": list(dict.fromkeys([*source.get("evidence_ids", []), *target.get("evidence_ids", [])])), + } + + relations = [ + relation(input_item, generate_item, "data_flow"), + relation(generate_item, initial_item, "data_flow"), + relation(initial_item, feedback_item, "evaluation"), + relation(feedback_item, self_feedback_item, "feedback"), + relation(initial_item, refine_item, "revision_input"), + relation(self_feedback_item, refine_item, "feedback"), + relation(refine_item, refined_item, "data_flow"), + relation(refined_item, feedback_item, "feedback_loop"), + ] + spec["relations"] = relations + spec["feedback_loops"] = [relations[-1]] + spec["topology"] = "feedback" + return { + "applied": True, + "removed_entities": removed_entities, + "rebuilt_relations": [f"{item['source']}->{item['target']}" for item in relations], + } + + +def _normalize_contract_relations(spec: dict[str, Any]) -> list[dict[str, Any]]: + aliases: dict[str, set[str]] = {} + endpoint_ids: set[str] = set() + for field in ("inputs", "modules", "outputs", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict) or not item.get("id"): + continue + item_id = str(item["id"]) + endpoint_ids.add(item_id) + for value in (item_id, _item_label(item)): + normalized = _normalized_label(str(value)) + if normalized: + aliases.setdefault(normalized, set()).add(item_id) + + def resolve(value: Any) -> str: + raw = str(value or "").strip() + if raw in endpoint_ids: + return raw + matches = aliases.get(_normalized_label(raw), set()) + return next(iter(matches)) if len(matches) == 1 else raw + + normalized_relations: list[dict[str, Any]] = [] + for raw in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(raw, dict): + continue + item = dict(raw) + source = resolve(item.get("source") or item.get("source_id")) + target = resolve(item.get("target") or item.get("target_id")) + if not source or not target: + continue + item["source"] = source + item["target"] = target + item["type"] = str(item.get("type") or item.get("relation_type") or "data_flow") + item["label"] = str(item.get("label") or "") + item["evidence_ids"] = list(item.get("evidence_ids")) if isinstance(item.get("evidence_ids"), list) else [] + normalized_relations.append(item) + return normalized_relations + + +def _repair_contract_relation_endpoints(spec: dict[str, Any]) -> dict[str, list[str]]: + endpoints = { + str(item.get("id")): item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + label_matches: dict[str, set[str]] = {} + for item_id, item in endpoints.items(): + normalized = _normalized_label(_item_label(item)) + if normalized: + label_matches.setdefault(normalized, set()).add(item_id) + + replacements: dict[str, str] = {} + repaired: list[str] = [] + relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] + for relation in relations: + target = str(relation.get("target") or "") + if target in endpoints: + continue + label = _normalized_label(str(relation.get("label") or "")) + matches = label_matches.get(label, set()) + if target and label and len(matches) == 1: + replacement = next(iter(matches)) + replacements[target] = replacement + relation["target"] = replacement + repaired.append(f"{target}->{replacement}") + + for relation in relations: + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + if source in replacements: + relation["source"] = replacements[source] + if target in replacements: + relation["target"] = replacements[target] + + deduplicated: list[dict[str, Any]] = [] + by_key: dict[tuple[str, str, str], dict[str, Any]] = {} + removed_duplicates: list[str] = [] + removed_unresolved: list[str] = [] + removed_self_relations: list[str] = [] + for relation in relations: + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + relation_type = str(relation.get("type") or "data_flow") + if source not in endpoints or target not in endpoints: + removed_unresolved.append(f"{source}->{target}") + continue + if source == target: + removed_self_relations.append(f"{source}->{target}:{relation_type}") + continue + key = (source, target, relation_type) + existing = by_key.get(key) + if existing: + existing["evidence_ids"] = list(dict.fromkeys(list(existing.get("evidence_ids", [])) + list(relation.get("evidence_ids", [])))) + if not existing.get("label") and relation.get("label"): + existing["label"] = relation["label"] + removed_duplicates.append(f"{source}->{target}:{relation_type}") + continue + by_key[key] = relation + deduplicated.append(relation) + spec["relations"] = deduplicated + return { + "repaired": list(dict.fromkeys(repaired)), + "removed_duplicates": list(dict.fromkeys(removed_duplicates)), + "removed_unresolved": list(dict.fromkeys(removed_unresolved)), + "removed_self_relations": list(dict.fromkeys(removed_self_relations)), + } + + +def _clear_unverbatim_relation_labels(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + evidence_by_id = { + str(item.get("id")): str(item.get("text") or "") + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + removed: list[str] = [] + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + label = str(relation.get("label") or "").strip() + if not label: + continue + cited_text = " ".join(evidence_by_id.get(str(value), "") for value in relation.get("evidence_ids", [])) + if _normalized_label(label) and _normalized_label(label) in _normalized_label(cited_text): + continue + removed.append(f"{relation.get('source')}->{relation.get('target')}:{label}") + relation["label"] = "" + return removed + + +def _repair_isolated_input_relations(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, list[str]]: + """Recover explicit data-source -> modality provenance from overview text. + + A common planner error is to flatten ``CFP and OCT from MEH-MIDAS and + public datasets`` into three peer inputs. That preserves all words while + changing the scientific meaning. This repair only fires for an explicit + ``using/with ... from ...`` clause in cited evidence. It reclassifies the + right-hand entities as data sources, creates a missing generic dataset + source when it is named verbatim, and connects sources to the left-hand + modalities before the existing modality-to-method flow. + """ + inputs = [item for item in spec.get("inputs", []) if isinstance(item, dict) and item.get("id")] + relations = spec.get("relations", []) if isinstance(spec.get("relations"), list) else [] + by_normalized = {_normalized_label(_item_label(item)): item for item in inputs if _normalized_label(_item_label(item))} + repaired: list[str] = [] + added_sources: list[str] = [] + reclassified: list[str] = [] + removed_shortcuts: list[str] = [] + removed_orphan_sources: list[str] = [] + collapsed_source_entities: list[str] = [] + existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + provenance_pattern = re.compile(r"\b(?:using|uses?|with)\s+(.{1,140}?)\s+from\s+(.{1,140}?)(?=[.;]|\bstage\s+(?:two|2)\b|$)", re.IGNORECASE) + source_term_pattern = re.compile(r"\b(?:datasets?|data|corpus|corpora|cohort|database|registry|biobank|repository)\b", re.IGNORECASE) + + for evidence in parsed.get("evidence", []): + if not isinstance(evidence, dict) or not evidence.get("id"): + continue + text = str(evidence.get("text") or "") + match = provenance_pattern.search(text) + if not match: + continue + modality_phrase, source_phrase = match.groups() + normalized_modality_phrase = _normalized_label(modality_phrase) + normalized_source_phrase = _normalized_label(source_phrase) + modalities = [ + item for item in inputs + if _normalized_label(_item_label(item)) + and _normalized_label(_item_label(item)) in normalized_modality_phrase + ] + if not modalities: + continue + sources = [ + item for item in inputs + if _normalized_label(_item_label(item)) + and _normalized_label(_item_label(item)) in normalized_source_phrase + ] + component_norms: list[str] = [] + component_labels: list[str] = [] + for raw_chunk in re.split(r"\s*(?:,|\band\b|\+)\s*", source_phrase, flags=re.IGNORECASE): + chunk = re.sub(r"^(?:the|a|an)\s+", "", raw_chunk.strip(" ()"), flags=re.IGNORECASE).strip() + normalized_chunk = _normalized_label(chunk) + if not chunk or not normalized_chunk: + continue + component_norms.append(normalized_chunk) + component_labels.append(chunk) + exact_existing = by_normalized.get(normalized_chunk) + if exact_existing: + if exact_existing not in sources: + sources.append(exact_existing) + continue + acronym_source = bool(re.fullmatch(r"[A-Z][A-Z0-9-]{2,}", chunk)) + if (not source_term_pattern.search(chunk) and not acronym_source) or len(chunk) > 50: + continue + source = { + "id": _stable_id("source", chunk, len(inputs)), + "name": chunk, + "role": "data_source", + "evidence_ids": [str(evidence["id"])], + } + inputs.append(source) + spec.setdefault("inputs", []).append(source) + by_normalized[normalized_chunk] = source + sources.append(source) + added_sources.append(str(source["id"])) + component_norms = list(dict.fromkeys(component_norms)) + if len(component_norms) >= 2: + composite = next( + ( + item for item in inputs + if isinstance(item, dict) + and _normalized_label(_item_label(item)) not in component_norms + and all(value in _normalized_label(_item_label(item)) for value in component_norms) + ), + None, + ) + if not composite: + composite_label = " + ".join(component_labels) + composite = { + "id": _stable_id("source", composite_label, len(inputs)), + "name": composite_label, + "role": "data_source", + "evidence_ids": [str(evidence["id"])], + } + inputs.append(composite) + spec.setdefault("inputs", []).append(composite) + by_normalized[_normalized_label(composite_label)] = composite + added_sources.append(str(composite["id"])) + component_ids = { + str(item.get("id")) + for item in inputs + if isinstance(item, dict) and _normalized_label(_item_label(item)) in component_norms + } + if component_ids: + collapsed_source_entities.extend(sorted(component_ids)) + inputs = [item for item in inputs if str(item.get("id")) not in component_ids] + spec["inputs"] = [item for item in spec.get("inputs", []) if not (isinstance(item, dict) and str(item.get("id")) in component_ids)] + relations = [ + item for item in relations + if not ( + isinstance(item, dict) + and (str(item.get("source")) in component_ids or str(item.get("target")) in component_ids) + ) + ] + existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + added_sources = [value for value in added_sources if value not in component_ids] + for value in component_norms: + by_normalized.pop(value, None) + sources = [composite] + if not sources: + continue + + evidence_id = str(evidence["id"]) + modality_ids = {str(item.get("id")) for item in modalities} + source_ids = {str(item.get("id")) for item in sources} + downstream_targets = { + str(relation.get("target")) + for relation in relations + if isinstance(relation, dict) and str(relation.get("source")) in modality_ids + } + kept_relations = [] + for relation in relations: + if not isinstance(relation, dict): + kept_relations.append(relation) + continue + pair = (str(relation.get("source")), str(relation.get("target"))) + if pair[0] in source_ids and pair[1] in downstream_targets: + removed_shortcuts.append(f"{pair[0]}->{pair[1]}") + existing_pairs.discard(pair) + continue + kept_relations.append(relation) + relations = kept_relations + + for modality in modalities: + if str(modality.get("role") or "").casefold() != "input modality": + modality["role"] = "input modality" + reclassified.append(str(modality.get("id"))) + for source in sources: + if str(source.get("role") or "").casefold() != "data_source": + source["role"] = "data_source" + reclassified.append(str(source.get("id"))) + source["evidence_ids"] = list(dict.fromkeys([*source.get("evidence_ids", []), evidence_id])) + for modality in modalities: + pair = (str(source.get("id")), str(modality.get("id"))) + if pair in existing_pairs: + continue + relations.append({ + "source": pair[0], + "target": pair[1], + "type": "data_source", + "label": "", + "evidence_ids": [evidence_id], + }) + existing_pairs.add(pair) + repaired.append(f"{pair[0]}->{pair[1]}") + connected_sources = { + str(item.get("source")) + for item in relations + if isinstance(item, dict) and str(item.get("type") or "").casefold() == "data_source" + } + retained_inputs = [] + for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: + item_id = str(item.get("id") or "") if isinstance(item, dict) else "" + role = str(item.get("role") or "").casefold() if isinstance(item, dict) else "" + if item_id.startswith("source_") and role == "data_source" and item_id not in connected_sources: + removed_orphan_sources.append(item_id) + continue + retained_inputs.append(item) + spec["inputs"] = retained_inputs + spec["relations"] = relations + return { + "repaired_relations": list(dict.fromkeys(repaired)), + "added_source_entities": list(dict.fromkeys(added_sources)), + "reclassified_entities": list(dict.fromkeys(reclassified)), + "removed_source_shortcuts": list(dict.fromkeys(removed_shortcuts)), + "removed_orphan_sources": list(dict.fromkeys(removed_orphan_sources)), + "collapsed_source_entities": list(dict.fromkeys(collapsed_source_entities)), + } + + +def _repair_explicit_evaluation_outputs(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, list[str]]: + modules = [item for item in spec.get("modules", []) if isinstance(item, dict) and item.get("id")] + if not modules: + return {"added_outputs": [], "added_relations": []} + source = next( + ( + item for item in reversed(modules) + if any(term in _item_label(item).casefold() for term in ("supervised", "fine-tun", "adapt")) + ), + modules[-1], + ) + outputs = spec.setdefault("outputs", []) + by_normalized = { + _normalized_label(_item_label(item)): item + for item in outputs if isinstance(item, dict) and _normalized_label(_item_label(item)) + } + relations = spec.setdefault("relations", []) + existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + added_outputs: list[str] = [] + added_relations: list[str] = [] + for evidence in parsed.get("evidence", []): + if not isinstance(evidence, dict) or not evidence.get("id"): + continue + text = str(evidence.get("text") or "") + section = str(evidence.get("section_hint") or "").casefold() + if str(evidence.get("kind") or "").casefold() != "caption" and "figure" not in section: + continue + if not re.search(r"\binternal\s+and\s+external\s+evaluation\b", text, re.IGNORECASE): + continue + evidence_id = str(evidence["id"]) + for label in ("internal evaluation", "external evaluation"): + normalized = _normalized_label(label) + output = by_normalized.get(normalized) + if not output: + output = { + "id": _stable_id("output", label, len(outputs)), + "name": label, + "role": "evaluation output", + "evidence_ids": [evidence_id], + } + outputs.append(output) + by_normalized[normalized] = output + added_outputs.append(str(output["id"])) + else: + output["evidence_ids"] = list(dict.fromkeys([*output.get("evidence_ids", []), evidence_id])) + pair = (str(source.get("id")), str(output.get("id"))) + if pair not in existing_pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "evaluation", "label": "", "evidence_ids": [evidence_id]}) + existing_pairs.add(pair) + added_relations.append(f"{pair[0]}->{pair[1]}") + break + return {"added_outputs": added_outputs, "added_relations": added_relations} + + +def _repair_generic_named_modules(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + generic_names = { + "model", "foundationmodel", "proposedmodel", "ourmodel", "method", "proposedmethod", "framework", "proposedframework", + } + evidence_by_id = { + str(item.get("id")): item + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + repaired: list[str] = [] + patterns = ( + re.compile(r"\bfoundation models?\s*\(\s*([A-Za-z][A-Za-z0-9_-]{2,40})\s*\)", re.IGNORECASE), + re.compile(r"\b(?:model|method|framework)\s+(?:called|named|denoted as)\s+([A-Za-z][A-Za-z0-9_-]{2,40})\b", re.IGNORECASE), + ) + for item in spec.get("modules", []) if isinstance(spec.get("modules"), list) else []: + if not isinstance(item, dict) or _normalized_label(_item_label(item)) not in generic_names: + continue + cited = [evidence_by_id.get(str(value)) for value in item.get("evidence_ids", [])] + candidates = [record for record in cited if record] + candidates.extend( + record for record in parsed.get("evidence", []) + if isinstance(record, dict) + and record not in candidates + and (str(record.get("kind") or "").casefold() == "caption" or int(record.get("page") or 9999) <= 3) + ) + replacement = None + replacement_evidence = None + for record in candidates: + text = str(record.get("text") or "") + match = next((pattern.search(text) for pattern in patterns if pattern.search(text)), None) + if not match: + continue + candidate = match.group(1).strip() + if _normalized_label(candidate) in {"ssl", "cnn", "transformer", "network"}: + continue + replacement = candidate + replacement_evidence = str(record.get("id") or "") + break + if not replacement: + continue + old_label = _item_label(item) + item["name"] = replacement + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} + for key, value in list(terminology.items()): + if _normalized_label(value) == _normalized_label(old_label) and _normalized_label(key) == _normalized_label(replacement): + terminology[key] = replacement + elif _normalized_label(key) == _normalized_label(old_label): + terminology.pop(key, None) + terminology[replacement] = replacement + if replacement_evidence: + item["evidence_ids"] = list(dict.fromkeys([*item.get("evidence_ids", []), replacement_evidence])) + repaired.append(f"{old_label}->{replacement}") + return repaired + + +def _repair_explicit_named_techniques(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + existing = { + _normalized_label(_item_label(item)): item + for item in spec.get("innovations", []) if isinstance(item, dict) and _normalized_label(_item_label(item)) + } + modules = [item for item in spec.get("modules", []) if isinstance(item, dict) and item.get("id")] + ssl_module = next((item for item in modules if _normalized_label(_item_label(item)) in {"ssl", "selfsupervisedlearning", "sslpretraining"}), None) + if not ssl_module: + return [] + repaired: list[str] = [] + pattern = re.compile(r"\b(?:ssl\s+)?(?:technique|method|approach)\s*\(\s*([A-Za-z][A-Za-z -]{2,50}?)\s*(?:\d+)?\s*\)", re.IGNORECASE) + for evidence in parsed.get("evidence", []): + if not isinstance(evidence, dict) or not evidence.get("id"): + continue + match = pattern.search(str(evidence.get("text") or "")) + if not match: + continue + label = re.sub(r"\s+", " ", match.group(1)).strip() + normalized = _normalized_label(label) + if not normalized or normalized in {"ssl", "learning", "training"}: + continue + innovation = existing.get(normalized) + if innovation: + innovation["evidence_ids"] = list(dict.fromkeys([*innovation.get("evidence_ids", []), str(evidence["id"])])) + innovation["must_appear_in_figure"] = True + innovation["visible_label_required"] = True + continue + innovation = { + "id": _stable_id("innovation", label, len(existing)), + "name": label, + "role": f"technique used by {_item_label(ssl_module)}", + "evidence_ids": [str(evidence["id"])], + "must_appear_in_figure": True, + "visible_label_required": True, + } + spec.setdefault("innovations", []).append(innovation) + existing[normalized] = innovation + repaired.append(str(innovation["id"])) + break + return repaired + + +def _repair_parallel_output_heads(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, list[str]]: + """Promote evidence-backed leaf branches to output-head roles. + + Branch planners often place classification, regression, mask, score, or + prediction heads in ``modules`` while placing only one sibling in + ``outputs``. That makes visually equivalent parallel heads receive + incompatible role instructions. Promotion is limited to leaf entities + sharing one upstream branch point, with at least two output-like siblings + and explicit branch/head/parallel evidence. + """ + if str(spec.get("topology") or "").casefold() != "branch": + return {"promoted_outputs": [], "reclassified_outputs": []} + modules = [item for item in spec.get("modules", []) if isinstance(item, dict) and item.get("id")] + outputs = [item for item in spec.get("outputs", []) if isinstance(item, dict) and item.get("id")] + relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] + if not modules or not relations: + return {"promoted_outputs": [], "reclassified_outputs": []} + + entity_by_id = {str(item["id"]): item for item in [*modules, *outputs]} + outgoing = {str(item.get("source")) for item in relations if item.get("source")} + incoming_by_source: dict[str, set[str]] = {} + relation_evidence: dict[tuple[str, str], list[str]] = {} + for relation in relations: + source, target = str(relation.get("source") or ""), str(relation.get("target") or "") + if not source or not target: + continue + incoming_by_source.setdefault(source, set()).add(target) + relation_evidence.setdefault((source, target), []).extend(str(value) for value in relation.get("evidence_ids", []) if value) + + evidence_by_id = { + str(item.get("id")): str(item.get("text") or "") + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + output_name_pattern = re.compile( + r"\b(?:class(?:ification|ifier)?|regression|mask|segmentation|detection|prediction|score|label|output|head)\b", + re.IGNORECASE, + ) + branch_evidence_pattern = re.compile(r"\b(?:parallel|branch(?:es)?|heads?|multi[- ]?task)\b", re.IGNORECASE) + promoted: list[str] = [] + reclassified: list[str] = [] + + for source_id, target_ids in incoming_by_source.items(): + leaf_targets = [ + entity_by_id[target_id] + for target_id in target_ids + if target_id in entity_by_id and target_id not in outgoing + ] + output_like = [item for item in leaf_targets if output_name_pattern.search(_item_label(item))] + if len(output_like) < 2: + continue + evidence_ids = list(dict.fromkeys( + value + for item in output_like + for value in [ + *item.get("evidence_ids", []), + *relation_evidence.get((source_id, str(item.get("id"))), []), + ] + if value + )) + evidence_text = " ".join(evidence_by_id.get(str(value), "") for value in evidence_ids) + if not branch_evidence_pattern.search(evidence_text): + continue + output_ids = {str(item.get("id")) for item in outputs} + for item in output_like: + item_id = str(item.get("id")) + if str(item.get("role") or "").casefold() != "output head": + item["role"] = "output head" + reclassified.append(item_id) + if item_id not in output_ids: + outputs.append(item) + output_ids.add(item_id) + promoted.append(item_id) + + if promoted: + promoted_set = set(promoted) + spec["modules"] = [item for item in modules if str(item.get("id")) not in promoted_set] + spec["outputs"] = outputs + return { + "promoted_outputs": list(dict.fromkeys(promoted)), + "reclassified_outputs": list(dict.fromkeys(reclassified)), + } + +def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: + summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} + spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} + spec["research_problem"] = _statement_object(spec.get("research_problem"), summary.get("research_problem")) + spec["central_claim"] = _statement_object(spec.get("central_claim"), summary.get("central_claim")) + if not isinstance(spec.get("inputs"), list) or not spec.get("inputs"): + spec["inputs"] = summary.get("inputs") if isinstance(summary.get("inputs"), list) else [] + if not isinstance(spec.get("outputs"), list) or not spec.get("outputs"): + spec["outputs"] = summary.get("outputs") if isinstance(summary.get("outputs"), list) else [] + if not isinstance(spec.get("modules"), list) or not spec.get("modules"): + summary_modules = summary.get("core_modules") if isinstance(summary.get("core_modules"), list) else summary.get("modules") + spec["modules"] = summary_modules if isinstance(summary_modules, list) else [] + for field in ("inputs", "modules", "outputs", "innovations"): + spec[field] = _normalize_contract_entities(spec.get(field), field) + entity_deduplication = _deduplicate_contract_entities(spec) + spec["relations"] = _normalize_contract_relations(spec) + panel_graph_report = _normalize_panel_graphs(spec, parsed) + completion_report = augment_contract_from_evidence(spec, parsed) + completion_report["panel_graphs"] = panel_graph_report + completion_report["panel_graph_completion"] = _complete_panel_graphs_from_inventory(spec, parsed) + completion_report["removed_duplicate_entities"] = entity_deduplication["removed"] + completion_report["duplicate_entity_replacements"] = entity_deduplication["replacements"] + plan["contract_completion_report"] = completion_report + if len(completion_report.get("added_entities", [])) >= 3: + fallback_ids = { + str(item.get("id")) + for item in (spec.get("modules", []) if isinstance(spec.get("modules"), list) else []) + if isinstance(item, dict) and str(item.get("role") or "") == "paper-derived stage requiring VLM verification" + } + if fallback_ids: + spec["modules"] = [item for item in spec.get("modules", []) if not (isinstance(item, dict) and str(item.get("id")) in fallback_ids)] + spec["relations"] = [ + item for item in spec.get("relations", []) if not ( + isinstance(item, dict) + and (str(item.get("source")) in fallback_ids or str(item.get("target")) in fallback_ids) + ) + ] + completion_report["removed_fallback_entities"] = sorted(fallback_ids) + _complete_from_overview_caption(spec, parsed) + endpoint_report = _repair_contract_relation_endpoints(spec) + completion_report["relation_endpoint_repairs"] = endpoint_report["repaired"] + completion_report["removed_duplicate_relations"] = endpoint_report["removed_duplicates"] + completion_report["removed_unresolved_relations"] = endpoint_report["removed_unresolved"] + completion_report["removed_self_relations"] = endpoint_report["removed_self_relations"] + provenance_report = _repair_isolated_input_relations(spec, parsed) + completion_report["repaired_isolated_inputs"] = provenance_report["repaired_relations"] + completion_report["repaired_data_provenance"] = provenance_report["repaired_relations"] + completion_report["added_data_sources"] = provenance_report["added_source_entities"] + completion_report["reclassified_data_entities"] = provenance_report["reclassified_entities"] + completion_report["removed_data_source_shortcuts"] = provenance_report["removed_source_shortcuts"] + completion_report["removed_orphan_data_sources"] = provenance_report["removed_orphan_sources"] + completion_report["collapsed_data_source_entities"] = provenance_report["collapsed_source_entities"] + evaluation_report = _repair_explicit_evaluation_outputs(spec, parsed) + completion_report["added_explicit_evaluation_outputs"] = evaluation_report["added_outputs"] + completion_report["added_explicit_evaluation_relations"] = evaluation_report["added_relations"] + completion_report["repaired_generic_named_modules"] = _repair_generic_named_modules(spec, parsed) + completion_report["added_explicit_named_techniques"] = _repair_explicit_named_techniques(spec, parsed) + output_head_report = _repair_parallel_output_heads(spec, parsed) + completion_report["promoted_parallel_output_heads"] = output_head_report["promoted_outputs"] + completion_report["reclassified_parallel_output_heads"] = output_head_report["reclassified_outputs"] + if endpoint_report["repaired"]: + secondary_completion = augment_contract_from_evidence(spec, parsed) + for key in ("added_entities", "upgraded_entities", "adopted_entities", "added_relations", "repaired_relations", "grounded_entities"): + completion_report[key] = list(dict.fromkeys(list(completion_report.get(key, [])) + list(secondary_completion.get(key, [])))) + modules = spec.get("modules") if isinstance(spec.get("modules"), list) else [] + encoder_modules = [item for item in modules if isinstance(item, dict) and "encoder" in _item_label(item).casefold() and "modality encoder" not in _item_label(item).casefold()] + if len(encoder_modules) >= 3: + group_id = "modality_encoders_group" + evidence_ids = list(dict.fromkeys(value for item in encoder_modules for value in item.get("evidence_ids", []))) + if not any(str(item.get("id")) == group_id for item in modules if isinstance(item, dict)): + modules.append({"id": group_id, "name": "Modality Encoders", "role": "group", "evidence_ids": evidence_ids}) + relations = spec.get("relations") if isinstance(spec.get("relations"), list) else [] + encoder_ids = {str(item.get("id")) for item in encoder_modules} + endpoints = modules + list(spec.get("outputs", []) if isinstance(spec.get("outputs"), list) else []) + list(spec.get("innovations", []) if isinstance(spec.get("innovations"), list) else []) + joint_items = [item for item in endpoints if isinstance(item, dict) and "joint" in _item_label(item).casefold() and ("embedding" in _item_label(item).casefold() or "representation" in _item_label(item).casefold())] + existing = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + for relation in list(relations): + if not isinstance(relation, dict) or str(relation.get("target")) not in encoder_ids: + continue + pair = (str(relation.get("source")), group_id) + if pair not in existing: + relations.append({"source": pair[0], "target": pair[1], "type": "encoding", "label": "", "evidence_ids": list(relation.get("evidence_ids", []))}) + existing.add(pair) + modality_terms = ("image", "text", "audio", "depth", "thermal", "imu", "video", "modalit") + for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: + if not isinstance(item, dict) or not any(term in _item_label(item).casefold() for term in modality_terms): + continue + pair = (str(item.get("id")), group_id) + if pair not in existing: + relations.append({"source": pair[0], "target": pair[1], "type": "encoding", "label": "", "evidence_ids": list(item.get("evidence_ids", []))}) + existing.add(pair) + for joint in joint_items: + if any(str(item.get("source")) in encoder_ids and str(item.get("target")) == str(joint.get("id")) for item in relations if isinstance(item, dict)): + pair = (group_id, str(joint.get("id"))) + if pair not in existing: + relations.append({"source": pair[0], "target": pair[1], "type": "alignment", "label": "", "evidence_ids": evidence_ids}) + existing.add(pair) + emergent_items = [ + item for item in endpoints + if isinstance(item, dict) + and (("emergent" in _item_label(item).casefold() and "align" in _item_label(item).casefold()) or "binding" in _item_label(item).casefold()) + ] + if not emergent_items: + emergent_relation = next((item for item in relations if isinstance(item, dict) and "emergent" in str(item.get("label") or "").casefold() and "align" in str(item.get("label") or "").casefold()), None) + if emergent_relation: + emergent = {"id": "emergent_alignment_concept", "statement": "Emergent Alignment", "role": "innovation", "evidence_ids": list(emergent_relation.get("evidence_ids", []))} + spec.setdefault("innovations", []).append(emergent) + emergent_items.append(emergent) + for joint in joint_items: + for emergent in emergent_items: + pair = (str(joint.get("id")), str(emergent.get("id"))) + if pair not in existing: + relations.append({"source": pair[0], "target": pair[1], "type": "enables", "label": "emergent alignment", "evidence_ids": list(emergent.get("evidence_ids", []))}) + existing.add(pair) + spec["relations"] = relations + spec["modules"] = modules + modules = spec.get("modules") if isinstance(spec.get("modules"), list) else [] + relations = spec.get("relations") if isinstance(spec.get("relations"), list) else [] + existing_labels = [_normalized_label(_item_label(item)) for field in ("inputs", "modules", "outputs", "innovations") for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) if isinstance(item, dict)] + component_terms = ("token", "embedding", "encoder", "decoder", "head", "branch", "prompt", "proposal", "mask", "engine", "dataset") + for index, item in enumerate(spec.get("must_show", []) if isinstance(spec.get("must_show"), list) else []): + if not isinstance(item, dict): + continue + text = _item_label(item) + normalized = _normalized_label(text) + if not text or len(text) > 80 or not any(term in text.casefold() for term in component_terms): + continue + if any(normalized and (normalized in label or label in normalized) for label in existing_labels if label): + continue + module = {"id": _stable_id("required", text, index), "name": text, "role": "required_component", "evidence_ids": list(item.get("evidence_ids", []))} + modules.append(module) + existing_labels.append(normalized) + spec["modules"] = modules + existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + transformer_target = next((item for item in modules if "transformer encoder" in _item_label(item).casefold()), None) + for item in modules: + label = _item_label(item).casefold() + normalized_component = _normalized_label(label) + if transformer_target and str(item.get("id")) != str(transformer_target.get("id")) and any(term in normalized_component for term in ("classtoken", "classificationtoken", "positionembedding")): + pair = (str(item.get("id")), str(transformer_target.get("id"))) + if pair not in existing_pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "conditioning", "label": "", "evidence_ids": list(item.get("evidence_ids", []))}) + existing_pairs.add(pair) + outgoing = {str(item.get("source")) for item in relations if isinstance(item, dict)} + incoming = {str(item.get("target")) for item in relations if isinstance(item, dict)} + def choose_input_target(label: str) -> dict[str, Any] | None: + preferences = [] + low = label.casefold() + if any(term in low for term in ("prompt", "提示", "プロンプト", "프롬프트")): + preferences = ["prompt encoder", "提示编码器", "提示編碼器", "プロンプトエンコーダ", "프롬프트 인코더", "encoder", "decoder"] + elif any(term in low for term in ("image", "video", "图像", "圖像", "影像", "视频", "影片", "画像", "動画", "이미지", "영상")): + preferences = ["patch", "image encoder", "图像编码器", "圖像編碼器", "画像エンコーダ", "이미지 인코더", "backbone", "encoder"] + elif any(term in low for term in ("text", "文本", "文字", "テキスト", "텍스트")): + preferences = ["text encoder", "文本编码器", "文本編碼器", "テキストエンコーダ", "텍스트 인코더", "modality encoders", "encoder"] + elif any(term in low for term in ("audio", "音频", "音訊", "音声", "오디오")): + preferences = ["audio encoder", "音频编码器", "音訊編碼器", "音声エンコーダ", "오디오 인코더", "modality encoders", "encoder"] + elif any(term in low for term in ("depth", "thermal", "imu", "深度", "热成像", "熱成像")): + preferences = ["modality encoders", "encoder", "编码器", "編碼器", "エンコーダ", "인코더"] + for preference in preferences: + match = next((module for module in modules if preference and preference in _item_label(module).casefold()), None) + if match: + return match + return None + for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: + source = str(item.get("id")) + if not source or source in outgoing: + continue + target = choose_input_target(_item_label(item)) + if target: + pair = (source, str(target.get("id"))) + if pair not in existing_pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "data_flow", "label": "", "evidence_ids": list(dict.fromkeys(list(item.get("evidence_ids", [])) + list(target.get("evidence_ids", []))))}) + existing_pairs.add(pair) + outgoing.add(source) + incoming.add(pair[1]) + + # When several peer inputs already converge on the same first module, + # connect any remaining declared input to that same target. This covers + # generic tensor/table inputs such as X_test without inventing a new stage. + input_ids = { + str(item.get("id")) + for item in spec.get("inputs", []) if isinstance(item, dict) and item.get("id") + } + input_target_counts: dict[str, int] = {} + for relation in relations: + if not isinstance(relation, dict) or str(relation.get("source") or "") not in input_ids: + continue + target_id = str(relation.get("target") or "") + if target_id: + input_target_counts[target_id] = input_target_counts.get(target_id, 0) + 1 + dominant_input_target = next( + (target_id for target_id, count in sorted(input_target_counts.items(), key=lambda item: (-item[1], item[0])) if count >= 2), + None, + ) + dominant_target_item = next((item for item in modules if str(item.get("id") or "") == dominant_input_target), None) + if dominant_target_item: + for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: + if not isinstance(item, dict): + continue + source = str(item.get("id") or "") + if not source or source in outgoing: + continue + pair = (source, str(dominant_target_item.get("id"))) + evidence_ids = list(dict.fromkeys([*item.get("evidence_ids", []), *dominant_target_item.get("evidence_ids", [])])) + if pair not in existing_pairs and evidence_ids: + relations.append({"source": pair[0], "target": pair[1], "type": "encoding", "label": "", "evidence_ids": evidence_ids}) + existing_pairs.add(pair) + outgoing.add(source) + incoming.add(pair[1]) + module_sources = {str(item.get("source")) for item in relations if isinstance(item, dict)} + completion_added_ids = {str(value) for value in completion_report.get("added_entities", []) if str(value)} + eligible_output_modules = [item for item in modules if str(item.get("id") or "") not in completion_added_ids] + sink_modules = [item for item in eligible_output_modules if str(item.get("id")) not in module_sources] + for item in spec.get("outputs", []) if isinstance(spec.get("outputs"), list) else []: + target_id = str(item.get("id")) + if not target_id or target_id in incoming: + continue + output_label = _item_label(item).casefold() + source = next((module for module in reversed(eligible_output_modules) if any(term in _item_label(module).casefold() for term in ("head", "decoder", "refine", "classifier", "predict"))), None) + if "mask" in output_label: + source = next((module for module in eligible_output_modules if "mask decoder" in _item_label(module).casefold() or "mask branch" in _item_label(module).casefold()), source) + elif "box" in output_label or "bounding" in output_label or "class" in output_label or "category" in output_label: + source = next((module for module in eligible_output_modules if "box branch" in _item_label(module).casefold() or "classification head" in _item_label(module).casefold()), source) + source = source or (sink_modules[-1] if sink_modules else eligible_output_modules[-1] if eligible_output_modules else None) + if source: + pair = (str(source.get("id")), target_id) + if pair not in existing_pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "data_flow", "label": "", "evidence_ids": list(dict.fromkeys(list(source.get("evidence_ids", [])) + list(item.get("evidence_ids", []))))}) + existing_pairs.add(pair) + incoming.add(target_id) + spec["relations"] = relations + completion_report["merged_architecture_entities"] = _merge_isolated_architecture_entities(spec, parsed) + completion_report["demoted_unconnected_completion_entities"] = _demote_unconnected_completion_entities(spec, completion_report, parsed) + feedback_report = _simplify_feedback_contract(spec) + completion_report["feedback_simplification_applied"] = feedback_report["applied"] + completion_report["feedback_removed_entities"] = feedback_report["removed_entities"] + completion_report["feedback_rebuilt_relations"] = feedback_report["rebuilt_relations"] + dense_report = _simplify_dense_multiframe_contract(spec) + completion_report["dense_multiframe_simplification_applied"] = dense_report["applied"] + completion_report["dense_multiframe_removed_entities"] = dense_report["removed_entities"] + completion_report["dense_multiframe_rebuilt_relations"] = dense_report["rebuilt_relations"] + multimodal_report = _simplify_multimodal_contract(spec) + completion_report["multimodal_simplification_applied"] = multimodal_report["applied"] + completion_report["multimodal_removed_entities"] = multimodal_report["removed_entities"] + completion_report["multimodal_rebuilt_relations"] = multimodal_report["rebuilt_relations"] + branch_report = _simplify_branch_contract(spec) + completion_report["branch_merged_entities"] = branch_report["merged_entities"] + completion_report["removed_cross_branch_relations"] = branch_report["removed_cross_branch_relations"] + completion_report["removed_branch_shortcuts"] = branch_report["removed_branch_shortcuts"] + spec.setdefault("training_flow", summary.get("training_flow") if isinstance(summary.get("training_flow"), list) else []) + spec.setdefault("inference_flow", summary.get("inference_flow") if isinstance(summary.get("inference_flow"), list) else []) + spec.setdefault("feedback_loops", [item for item in spec.get("relations", []) if isinstance(item, dict) and "feedback" in str(item.get("type") or "").casefold()]) + allowed_topologies = {"linear", "branch", "feedback", "multimodal", "dense_multiframe", "unknown"} + inferred_topology = _infer_topology(spec) + dense_overview = _detect_dense_overview_panels(parsed) + completion_report["dense_overview_detection"] = dense_overview + if dense_overview["detected"] and inferred_topology not in {"feedback", "multimodal"}: + inferred_topology = "dense_multiframe" + if str(spec.get("topology") or "") not in allowed_topologies or inferred_topology in {"feedback", "multimodal", "dense_multiframe"}: + spec["topology"] = inferred_topology + completion_report["exact_visible_entity_groundings"] = _ground_exact_visible_entities(spec, parsed) + completion_report["late_repaired_relation_evidence"] = _repair_relation_evidence_after_grounding(spec, parsed) + completion_report["removed_out_of_scope_entities"] = _prune_out_of_scope_entities(spec, parsed, completion_report) + completion_report["removed_ungrounded_innovations"] = _remove_ungrounded_innovations(spec) + completion_report["removed_input_shortcuts"] = _remove_input_shortcuts_through_intermediates(spec) + _repair_cross_script_terminology(spec, parsed) + completion_report["repaired_cross_script_entity_labels"] = _repair_cross_script_entity_labels(spec, parsed) + completion_report["removed_unverbatim_relation_labels"] = _clear_unverbatim_relation_labels(spec, parsed) + overview_panels = _extract_overview_panels(parsed, spec) + if overview_panels: + spec["overview_panels"] = overview_panels + elif not isinstance(spec.get("overview_panels"), list): + spec["overview_panels"] = [] + completion_report["overview_panels"] = spec["overview_panels"] + optional_labels: list[str] = [] + optional_normalized: set[str] = set() + def add_optional_label(value: object) -> None: + label = str(value or "").strip() + normalized = _normalized_label(label) + if label and normalized and normalized not in optional_normalized: + optional_labels.append(label) + optional_normalized.add(normalized) + for value in spec.get("optional_visible_labels", []) if isinstance(spec.get("optional_visible_labels"), list) else []: + add_optional_label(value) + for panel in spec.get("overview_panels", []) if isinstance(spec.get("overview_panels"), list) else []: + if isinstance(panel, dict): + add_optional_label(panel.get("panel_label")) + for item in spec.get("supporting_details", []) if isinstance(spec.get("supporting_details"), list) else []: + if isinstance(item, dict): + add_optional_label(item.get("optional_visible_label")) + spec["optional_visible_labels"] = optional_labels + labels = [] + normalized_labels: set[str] = set() + def add_required_label(value: object) -> None: + label = str(value or "").strip() + normalized = _normalized_label(label) + if label and normalized and normalized not in normalized_labels: + labels.append(label) + normalized_labels.add(normalized) + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} + active_panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + if len(active_panel_graphs) >= 2: + for panel in active_panel_graphs: + add_required_label(panel.get("label")) + for node in panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []: + if isinstance(node, dict): + add_required_label(_item_label(node)) + else: + for field in ("inputs", "modules", "outputs"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if isinstance(item, dict) and _item_label(item): + add_required_label(_item_label(item)) + for panel in spec.get("overview_panels", []) if isinstance(spec.get("overview_panels"), list) else []: + if isinstance(panel, dict): + add_required_label(panel.get("label")) + for value in terminology.values(): + add_required_label(value) + spec["required_labels"] = labels + repeatable_labels = [str(value).strip() for value in spec.get("repeatable_labels", []) if str(value).strip()] if isinstance(spec.get("repeatable_labels"), list) else [] + entity_by_id = { + str(item.get("id")): item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + panel_membership_counts: dict[str, int] = {} + for panel in spec.get("overview_panels", []) if isinstance(spec.get("overview_panels"), list) else []: + if not isinstance(panel, dict): + continue + for item_id in {str(value) for value in panel.get("entity_ids", []) if value}: + panel_membership_counts[item_id] = panel_membership_counts.get(item_id, 0) + 1 + for item_id, count in panel_membership_counts.items(): + item = entity_by_id.get(item_id) + label = _item_label(item) if item else "" + if count >= 2 and label and label not in repeatable_labels: + repeatable_labels.append(label) + panel_instance_label_counts: dict[str, tuple[str, int]] = {} + for panel in spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []: + if not isinstance(panel, dict): + continue + for node in panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []: + if not isinstance(node, dict): + continue + label = _item_label(node) + key = _normalized_label(label) + if not key: + continue + original, count = panel_instance_label_counts.get(key, (label, 0)) + panel_instance_label_counts[key] = (original, count + 1) + for label, count in panel_instance_label_counts.values(): + if count >= 2 and label not in repeatable_labels: + repeatable_labels.append(label) + central_claim_text = str(spec.get("central_claim", {}).get("text") or "") if isinstance(spec.get("central_claim"), dict) else str(spec.get("central_claim") or "") + shared_component_signal = bool(re.search(r"\b(?:same|shared|reused?|single underlying)\b", central_claim_text, re.IGNORECASE)) + if shared_component_signal: + for item in spec.get("modules", []) if isinstance(spec.get("modules"), list) else []: + if not isinstance(item, dict): + continue + label = _item_label(item) + role = str(item.get("role") or "").casefold() + if label and ("model" in label.casefold() or "shared" in role or "reused" in role) and label not in repeatable_labels: + repeatable_labels.append(label) + if len(active_panel_graphs) >= 2: + required_normalized = {_normalized_label(value) for value in labels if _normalized_label(value)} + repeatable_labels = [value for value in repeatable_labels if _normalized_label(value) in required_normalized] + spec["repeatable_labels"] = repeatable_labels + raw_uncertainties = list(summary.get("unknowns", []) if isinstance(summary.get("unknowns"), list) else []) + raw_uncertainties.extend( + f"Dropped unresolved relation {value} because its endpoint was not declared." + for value in completion_report.get("removed_unresolved_relations", []) + ) + uncertainties = [ + str(item.get("statement") or item.get("text") or item.get("id") or "unknown") if isinstance(item, dict) else str(item) + for item in raw_uncertainties + ] + evidence_by_id = {item["id"]: item for item in parsed.get("evidence", [])} + for field in ("inputs", "modules", "outputs", "relations", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + evidence = [evidence_by_id.get(value) for value in item.get("evidence_ids", [])] + valid_records = [record for record in evidence if record] + if valid_records and all(float(record.get("confidence", 1.0)) < 0.75 and str(record.get("source") or "").casefold() in {"easyocr", "paddle", "adapter"} for record in valid_records): + uncertainties.append(f"{_item_id(item, field, 0)} relies only on low-confidence OCR evidence") + if parsed.get("extraction_report", {}).get("semantic_scope") == "sampled_pages_only": + uncertainties.append("The source is a long scanned document; only scheduled sample pages were OCRed, so unprocessed pages may contain additional scientific details.") + spec["uncertainties"] = list(dict.fromkeys(str(item) for item in uncertainties if str(item).strip())) + spec.setdefault("forbidden_inventions", []) + plan["figure_specification"] = spec + return spec + + +def build_overlay_spec(plan: dict[str, Any]) -> dict[str, Any]: + spec = plan.get("figure_specification", {}) + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + if len(panel_graphs) >= 2: + labels = [] + connectors = [] + groups = [] + reading_order = [] + for panel in panel_graphs: + panel_id = str(panel.get("id") or "") + groups.append({ + "id": panel_id, + "panel_label": str(panel.get("panel_label") or ""), + "label": str(panel.get("label") or ""), + "member_ids": [str(item.get("instance_id") or item.get("id") or "") for item in panel.get("nodes", []) if isinstance(item, dict)], + }) + node_ids = set() + for node in panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []: + if not isinstance(node, dict): + continue + instance_id = str(node.get("instance_id") or node.get("id") or "") + text_value = _item_label(node) + if not instance_id or not text_value: + continue + node_ids.add(instance_id) + reading_order.append(instance_id) + labels.append({ + "target_id": instance_id, + "text": text_value, + "role": str(node.get("role") or "module"), + "panel_id": panel_id, + "bbox_percent": node.get("bbox_percent"), + }) + for item in panel.get("relations", []) if isinstance(panel.get("relations"), list) else []: + if not isinstance(item, dict): + continue + source = str(item.get("source") or "") + target = str(item.get("target") or "") + if source not in node_ids or target not in node_ids or source == target: + continue + connectors.append({ + "source": source, + "target": target, + "type": str(item.get("type") or "data_flow"), + "label": str(item.get("label") or ""), + "panel_id": panel_id, + }) + return { + "summary": "Exact editable panel-instance labels and local directed connectors derived from the paper contract.", + "labels": labels, + "connectors": connectors, + "groups": groups, + "reading_order": reading_order, + } + labels = [] + visible_targets: dict[str, str] = {} + target_aliases: dict[str, str] = {} + for field in ("inputs", "modules", "outputs", "innovations"): + for index, item in enumerate(spec.get(field, []) if isinstance(spec.get(field), list) else []): + if not isinstance(item, dict): + continue + text = _item_label(item) + normalized = _normalized_label(text) + target_id = _item_id(item, field, index) + if text and normalized not in visible_targets: + labels.append({"target_id": target_id, "text": text, "role": field.rstrip("s")}) + visible_targets[normalized] = target_id + elif text and normalized: + target_aliases[target_id] = visible_targets[normalized] + connectors = [] + connector_keys: set[tuple[str, str, str, str]] = set() + for item in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if isinstance(item, dict): + source = target_aliases.get(str(item.get("source") or item.get("source_id") or ""), str(item.get("source") or item.get("source_id") or "")) + target = target_aliases.get(str(item.get("target") or item.get("target_id") or ""), str(item.get("target") or item.get("target_id") or "")) + relation_type = str(item.get("type") or item.get("relation_type") or "data_flow") + label = str(item.get("label") or "") + key = (source, target, relation_type, label) + if not source or not target or source == target or key in connector_keys: + continue + connectors.append({"source": source, "target": target, "type": relation_type, "label": label}) + connector_keys.add(key) + return { + "summary": "Exact editable labels and directed connectors derived from the paper contract.", + "labels": labels, + "connectors": connectors, + "groups": list(plan.get("design_plan", {}).get("groups", []) if isinstance(plan.get("design_plan"), dict) else []), + "reading_order": list(plan.get("design_plan", {}).get("reading_order", []) if isinstance(plan.get("design_plan"), dict) else []), + } + + +def synchronize_plan_to_contract(plan: dict[str, Any]) -> None: + spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + entities = [ + item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + required_labels = [str(value).strip() for value in spec.get("required_labels", []) if str(value).strip()] + allowed = {_normalized_label(value) for value in [*required_labels, *spec.get("repeatable_labels", [])] if _normalized_label(value)} + visible_entities = [item for item in entities if _normalized_label(_item_label(item)) in allowed] if allowed else entities + panel_nodes = [node for panel in panel_graphs for node in panel.get("nodes", []) if isinstance(node, dict)] + entity_ids = ( + [str(item.get("instance_id") or item.get("id") or "") for item in panel_nodes if str(item.get("instance_id") or item.get("id") or "")] + if len(panel_graphs) >= 2 else + [str(item.get("id") or "") for item in visible_entities if str(item.get("id") or "")] + ) + labels = required_labels or [_item_label(item) for item in visible_entities if _item_label(item)] + spec["storyline"] = labels + spec["must_show"] = labels + topology = str(spec.get("topology") or "unknown") + if topology == "multimodal": + spec["training_flow"] = [] + spec["inference_flow"] = [] + plan["figure_specification"] = spec + plan["design_plan"] = { + "summary": "Information narrative synchronized to the normalized scientific contract.", + "reading_order": entity_ids, + "groups": [], + "innovation_emphasis": [_item_label(item) for item in spec.get("innovations", []) if isinstance(item, dict) and _normalized_label(_item_label(item)) in allowed], + "preserve": labels, + "remove": list(spec.get("forbidden_inventions", []) if isinstance(spec.get("forbidden_inventions"), list) else []), + } + layout = plan.get("layout_intent") if isinstance(plan.get("layout_intent"), dict) else {} + layout.update({ + "summary": "Layout intent synchronized to the normalized scientific topology.", + "pattern": {"feedback": "loop", "multimodal": "hub_and_spoke", "dense_multiframe": "stacked", "branch": "two_stage"}.get(topology, "left_to_right"), + "flow_description": "Render only declared entities and directed relations; do not add implementation details outside the visible-label whitelist.", + }) + plan["layout_intent"] = layout + plan["visual_metaphors"] = { + "summary": "Paper-grounded visual metaphors synchronized to the normalized contract.", + "items": [ + {"module_id": str(item.get("instance_id") or item.get("id") or ""), "metaphor": _item_label(item), "must_show": [], "avoid_showing": list(spec.get("forbidden_inventions", []))} + for item in (panel_nodes if len(panel_graphs) >= 2 else visible_entities) + ], + } + + +def merge_review_grounded_contract(primary: dict[str, Any], fallback: dict[str, Any]) -> dict[str, Any]: + primary_spec = primary.get("figure_specification") if isinstance(primary.get("figure_specification"), dict) else {} + fallback_spec = fallback.get("figure_specification") if isinstance(fallback.get("figure_specification"), dict) else {} + endpoint_remap: dict[str, str] = {} + for field in ("inputs", "modules", "outputs", "innovations", "must_show", "relations"): + combined = list(primary_spec.get(field, []) if isinstance(primary_spec.get(field), list) else []) + if field == "relations": + keys = {(str(item.get("source")), str(item.get("target")), str(item.get("type"))) for item in combined if isinstance(item, dict)} + for item in fallback_spec.get(field, []) if isinstance(fallback_spec.get(field), list) else []: + if not isinstance(item, dict): + continue + merged_relation = dict(item) + merged_relation["source"] = endpoint_remap.get(str(item.get("source")), str(item.get("source"))) + merged_relation["target"] = endpoint_remap.get(str(item.get("target")), str(item.get("target"))) + key = (str(merged_relation.get("source")), str(merged_relation.get("target")), str(merged_relation.get("type"))) + if key and key not in keys: + combined.append(merged_relation) + keys.add(key) + else: + by_id = {_item_id(item, field, index).casefold(): _item_id(item, field, index) for index, item in enumerate(combined) if isinstance(item, dict)} + by_label = {_item_label(item).casefold(): _item_id(item, field, index) for index, item in enumerate(combined) if isinstance(item, dict) and _item_label(item)} + for index, item in enumerate(fallback_spec.get(field, []) if isinstance(fallback_spec.get(field), list) else []): + if not isinstance(item, dict): + continue + raw_item_id = _item_id(item, field, index) + item_id = raw_item_id.casefold() + label = _item_label(item).casefold() + if item_id in by_id: + endpoint_remap[raw_item_id] = by_id[item_id] + elif label and label in by_label: + endpoint_remap[raw_item_id] = by_label[label] + else: + combined.append(item) + endpoint_remap[raw_item_id] = raw_item_id + by_id[item_id] = raw_item_id + if label: + by_label[label] = raw_item_id + primary_spec[field] = combined + terminology = dict(fallback_spec.get("terminology") if isinstance(fallback_spec.get("terminology"), dict) else {}) + terminology.update(primary_spec.get("terminology") if isinstance(primary_spec.get("terminology"), dict) else {}) + primary_spec["terminology"] = terminology + primary_spec["forbidden_inventions"] = list(dict.fromkeys( + list(primary_spec.get("forbidden_inventions", []) if isinstance(primary_spec.get("forbidden_inventions"), list) else []) + + list(fallback_spec.get("forbidden_inventions", []) if isinstance(fallback_spec.get("forbidden_inventions"), list) else []) + )) + primary["figure_specification"] = primary_spec + primary_design = primary.get("design_plan") if isinstance(primary.get("design_plan"), dict) else {} + fallback_order = fallback.get("design_plan", {}).get("reading_order", []) if isinstance(fallback.get("design_plan"), dict) else [] + primary_design["reading_order"] = list(dict.fromkeys(list(primary_design.get("reading_order", [])) + list(fallback_order))) + primary["design_plan"] = primary_design + return primary + + +def _paper_review_from_plan(plan: dict[str, Any], selected_domain: dict[str, Any]) -> dict[str, Any]: + summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} + spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} + def fact(item: Any, item_id: str) -> dict[str, Any]: + if not isinstance(item, dict): + return {"id": item_id, "statement": str(item or "unknown"), "evidence_ids": [], "status": "unknown"} + return { + "id": str(item.get("id") or item_id), + "statement": str(item.get("statement") or item.get("text") or item.get("name") or "unknown"), + "visible_label": str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or ""), + "evidence_ids": list(item.get("evidence_ids", [])), + "status": str(item.get("status") or "required"), + "importance": str(item.get("importance") or "high"), + "confidence": float(item.get("confidence", 1.0) or 0.0), + "must_appear_in_figure": True, + "visual_role": str(item.get("role") or "module"), + } + return { + "summary": "Compact paper review derived from the fast VLM figure contract.", + "schema_version": "2.0-fast", + "paper_identity": {"title": summary.get("title"), "paper_type": summary.get("paper_type", "unknown")}, + "domain_profile": selected_domain.get("id"), + "research_questions": [fact(spec.get("research_problem") or summary.get("research_problem"), "research_problem")], + "central_claims": [fact(spec.get("central_claim") or summary.get("central_claim"), "central_claim")], + "inputs": [fact(item, f"input_{index}") for index, item in enumerate(spec.get("inputs", [])) if isinstance(item, dict)], + "outputs": [fact(item, f"output_{index}") for index, item in enumerate(spec.get("outputs", [])) if isinstance(item, dict)], + "research_objects": [], + "concepts": [], + "modules": [fact(item, f"module_{index}") for index, item in enumerate(spec.get("modules", [])) if isinstance(item, dict)], + "relations": [ + {**fact(item, f"relation_{index}"), "source_id": item.get("source"), "target_id": item.get("target"), "relation_type": item.get("type", "data_flow")} + for index, item in enumerate(spec.get("relations", [])) if isinstance(item, dict) + ], + "contributions": [], + "innovations": [fact(item, f"innovation_{index}") for index, item in enumerate(spec.get("innovations", [])) if isinstance(item, dict)], + "workflows": { + "training": [fact(item, f"training_{index}") for index, item in enumerate(spec.get("training_flow", []))], + "inference": [fact(item, f"inference_{index}") for index, item in enumerate(spec.get("inference_flow", []))], + }, + "experiments": {}, + "assumptions": [], + "limitations": [], + "results": [], + "terminology": [], + "forbidden_inventions": [{"id": f"forbidden_{index}", "statement": str(value), "evidence_ids": []} for index, value in enumerate(spec.get("forbidden_inventions", []))], + "unknowns": list(spec.get("uncertainties", [])), + } + + +def _fast_cache_path(parsed: dict[str, Any], model: str, domain_profile: str) -> Path: + signature = json.dumps({"version": 41, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "domain_profile": domain_profile}, sort_keys=True).encode("utf-8") + variant = hashlib.sha256(signature).hexdigest()[:16] + root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) + return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" + + +def _semantic_cache_safe(parsed: dict[str, Any]) -> bool: + report = parsed.get("extraction_report", {}) + return bool(report.get("scientific_scope_complete", True) and report.get("ocr_run_complete", True)) + + +def prepare_paper_figure_contract( + paper: str | Path, + out: str | Path, + deadline_seconds: int = 180, + planner_mode: str = "vlm", + planner_model: str | None = None, + ocr_engine: str = "auto", + ocr_lang: str = "en_ch", + preferences_path: str | Path | None = None, + positive_references: list[str] | None = None, + negative_references: list[str] | None = None, + aspect_ratio: str | None = None, + language: str | None = None, + domain_profile: str = "auto", + ocr_adapter: Callable | None = None, + fast_mode: bool = False, +) -> dict[str, Any]: + started = time.monotonic() + root = ensure_dir(out).resolve() + inputs = ensure_dir(root / "inputs") + positive_dir = ensure_dir(inputs / "positive_references") + negative_dir = ensure_dir(inputs / "negative_references") + archived_paper = _archive_file(paper, inputs, f"paper{Path(paper).suffix.lower()}") + archived_positive = [_archive_file(path, positive_dir) for path in (positive_references or [])] + archived_negative = [_archive_file(path, negative_dir) for path in (negative_references or [])] + raw_preferences = _load_preferences(preferences_path) + preferences = merge_preferences(raw_preferences, aspect_ratio=aspect_ratio, language=language) + preferences["positive_references"] = archived_positive + preferences["negative_references"] = archived_negative + write_json(root / "preferences.json", preferences) + write_json(root / "input_manifest.json", { + "summary": "Archived inputs for paper figure contract preparation.", + "paper_original": str(Path(paper).resolve()), + "paper_archived": archived_paper, + "preferences_original": str(Path(preferences_path).resolve()) if preferences_path else None, + "positive_references": archived_positive, + "negative_references": archived_negative, + }) + + deadline = max(30, int(deadline_seconds)) + deadline_at = started + deadline + provider_deadline_at = deadline_at - 15.0 + ocr_rescue_min_remaining = 90.0 if fast_mode and planner_mode == "vlm" and deadline <= 240 else 45.0 + parsed = None if ocr_adapter else read_document_cache(archived_paper, ocr_engine=ocr_engine, ocr_lang=ocr_lang) + document_cache_hit = parsed is not None + if parsed is None: + parsed = parse_paper(archived_paper, deadline_at=deadline_at, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, ocr_rescue_min_remaining=ocr_rescue_min_remaining) + if not ocr_adapter: + write_document_cache(archived_paper, parsed, ocr_engine=ocr_engine, ocr_lang=ocr_lang) + parsed["document_cache_version"] = DOCUMENT_CACHE_VERSION + document_preparation_seconds = round(time.monotonic() - started, 3) + write_json(root / "document_model.json", parsed) + write_json(root / "extraction_report.json", parsed["extraction_report"]) + write_json(root / "document_index.json", parsed["document_index"]) + write_json(root / "section_index.json", parsed["document_index"]) + write_json(root / "evidence_map.json", {"summary": "Page-aware evidence map.", "source_path": parsed["source_path"], "page_count": parsed["page_count"], "char_count": parsed["char_count"], "headings": parsed["headings"], "evidence": parsed["evidence"]}) + write_text(root / "paper.md", paper_markdown(parsed)) + write_text(root / "section_summary.md", _section_summary_markdown(parsed)) + + if parsed["extraction_report"].get("status") == "fail": + result = { + "summary": "Paper extraction failed the minimum readable-page gate.", + "ok": False, + "status": "extraction_failed", + "out_dir": str(root), + "paper": archived_paper, + "elapsed_seconds": round(time.monotonic() - started, 3), + "errors": parsed["extraction_report"].get("warnings", []), + } + write_json(root / "run_report.json", result) + return {**result, "root": root, "parsed": parsed, "preferences": preferences} + + selected_domain = detect_domain_profile(parsed, explicit=domain_profile) + write_json(root / "domain_profile.json", selected_domain) + if fast_mode: + remaining = max(0, int(provider_deadline_at - time.monotonic())) + effective_mode = planner_mode if remaining >= 25 else "heuristic" + cache_path = _fast_cache_path(parsed, str(planner_model or ""), str(selected_domain.get("id") or "general")) + if cache_path.exists() and effective_mode == "vlm" and _semantic_cache_safe(parsed): + plan = read_json(cache_path) + planner_metadata = {"requested_mode": "vlm", "mode": "vlm", "model": planner_model, "warning": None, "prompt": "", "cached": True} + else: + plan, planner_metadata = plan_fast_paper_contract( + parsed, + preferences, + mode=effective_mode, + model=planner_model, + timeout_seconds=min(35, remaining), + retries=2, + evidence_max_chars=36000, + deadline_at=provider_deadline_at, + ) + if planner_metadata.get("mode") == "vlm": + paper_review = _paper_review_from_plan(plan, selected_domain) + review_metadata = {"summary": "Paper review derived from the successful fast VLM plan.", "requested_mode": planner_mode, "mode": "derived_from_vlm_plan", "model": planner_metadata.get("model"), "warning": None, "prompt": ""} + else: + remaining = max(0, int(provider_deadline_at - time.monotonic())) + review_mode = planner_mode if remaining >= 25 else "heuristic" + paper_review, review_metadata = build_paper_review( + parsed, + selected_domain, + mode=review_mode, + model=planner_model, + timeout_seconds=max(1, min(35, remaining)), + retries=1, + evidence_max_chars=36000, + deadline_at=provider_deadline_at, + ) + if review_metadata.get("mode") == "vlm": + plan = build_review_grounded_plan(parsed, preferences, paper_review) + planner_metadata = {**planner_metadata, "mode": "review_grounded_vlm", "model": review_metadata.get("model"), "warning": planner_metadata.get("warning")} + else: + remaining = max(0, int(provider_deadline_at - time.monotonic())) + effective_mode = planner_mode if remaining >= 30 else "heuristic" + paper_review, review_metadata = build_paper_review(parsed, selected_domain, mode=effective_mode, model=planner_model, timeout_seconds=max(1, min(35, remaining)), retries=1, deadline_at=provider_deadline_at) + remaining = max(0, int(provider_deadline_at - time.monotonic())) + planning_mode = effective_mode if remaining >= 25 else "heuristic" + plan, planner_metadata = plan_paper_image( + parsed, + preferences, + mode=planning_mode, + model=planner_model, + reference_images=archived_positive + archived_negative, + paper_review=paper_review, + timeout_seconds=max(1, min(35, remaining)), + retries=1, + deadline_at=provider_deadline_at, + ) + fallback_plan = build_review_grounded_plan(parsed, preferences, paper_review) + if review_metadata.get("mode") == "vlm": + plan = merge_review_grounded_contract(plan, fallback_plan) + review_prompt = review_metadata.pop("prompt", "") + write_text(root / "prompts" / "paper_review_prompt.txt", review_prompt) + write_json(root / "paper_review_metadata.json", review_metadata) + write_json(root / "paper_review.json", paper_review) + coverage = validate_review_coverage(paper_review, parsed, selected_domain, strict=False) + write_json(root / "review_coverage_report.json", coverage) + planning_prompt = planner_metadata.pop("prompt") + write_text(root / "prompts" / "planning_prompt.txt", planning_prompt) + write_json(root / "planning_metadata.json", {"summary": "Planner execution metadata.", **planner_metadata}) + expand_plan_evidence(plan, paper_review, parsed) + normalize_figure_contract(plan, parsed) + synchronize_plan_to_contract(plan) + completion_report = plan.get("contract_completion_report") if isinstance(plan.get("contract_completion_report"), dict) else {} + write_json(root / "contract_completion_report.json", completion_report) + for name in ("paper_summary", "figure_specification", "design_plan", "layout_intent", "visual_metaphors", "style_plan"): + write_json(root / f"{name}.json", plan[name]) + planning_validation = validate_plan_grounding(plan, parsed) + write_json(root / "planning_validation_report.json", planning_validation) + if fast_mode and planning_validation.get("ok") and planner_metadata.get("mode") in {"vlm", "review_grounded_vlm"} and _semantic_cache_safe(parsed): + write_json(cache_path, plan) + overlay = build_overlay_spec(plan) + write_json(root / "overlay_spec.json", overlay) + final_prompt = compile_image_prompt(plan, preferences, candidate_variant=1, deterministic_overlay=True) + write_text(root / "image_prompt.md", final_prompt) + write_text(root / "image_prompt.txt", final_prompt) + referenced_ids = _collect_evidence_ids({"paper_review": paper_review, "figure_specification": plan["figure_specification"]}) + provisional_ground_truth = compile_review_ground_truth(plan, parsed["evidence"]) + review_evidence_ids = { + str(evidence_id) + for priority in provisional_ground_truth.get("priorities", []) + for evidence_id in priority.get("evidence_ids", []) + if str(evidence_id).strip() + } + referenced_ids.update(review_evidence_ids) + key_evidence = [ + item for item in parsed["evidence"] + if str(item.get("id") or "") in referenced_ids or str(item.get("legacy_id") or "") in referenced_ids + ] + write_json(root / "key_evidence.json", { + "summary": "Evidence records referenced by the paper review and figure contract.", + "evidence": key_evidence, + }) + review_ground_truth = compile_review_ground_truth(plan, key_evidence) + plan["review_ground_truth"] = review_ground_truth + write_json(root / "review_ground_truth.json", review_ground_truth) + + elapsed = round(time.monotonic() - started, 3) + deadline_reached = time.monotonic() >= deadline_at + scientific_scope_complete = bool(parsed["extraction_report"].get("scientific_scope_complete", True)) + production_ready = bool((review_metadata.get("mode") == "vlm" or planner_metadata.get("mode") == "vlm") and planning_validation.get("ok") and not deadline_reached and scientific_scope_complete) + has_warning = bool(planner_metadata.get("warning") or review_metadata.get("warning") or parsed["extraction_report"].get("status") == "warning") + status = "complete" if production_ready and not has_warning else "completed_with_warnings" + contract_source = "cache" if planner_metadata.get("cached") else "vlm" if planner_metadata.get("mode") == "vlm" else "vlm_review_deterministic_compile" if planner_metadata.get("mode") == "review_grounded_vlm" else "deterministic_evidence_rules" + source_extraction_seconds = float(parsed["extraction_report"].get("elapsed_seconds") or 0.0) + extraction_seconds = 0.0 if document_cache_hit else source_extraction_seconds + planner_provider = planner_metadata.get("provider") if isinstance(planner_metadata.get("provider"), dict) else {} + review_provider = review_metadata.get("provider") if isinstance(review_metadata.get("provider"), dict) else {} + provider_calls = [item for item in (planner_provider, review_provider) if item.get("attempts")] + provider_summary = { + "planner": planner_provider, + "review": review_provider, + "attempts": sum(int(item.get("attempts") or 0) for item in provider_calls), + "retries_used": sum(int(item.get("retries_used") or 0) for item in provider_calls), + "success": any(bool(item.get("success")) for item in provider_calls) if provider_calls else None, + "successful_call_count": sum(bool(item.get("success")) for item in provider_calls), + "call_count": len(provider_calls), + "elapsed_seconds": round(sum(float(item.get("elapsed_seconds") or 0.0) for item in provider_calls), 3), + "failure_categories": list(dict.fromkeys(category for item in provider_calls for category in (item.get("failure_categories") or []))), + } + result = { + "summary": "Fast paper-to-framework contract preparation completed.", + "ok": bool(planning_validation.get("ok")), + "status": status, + "production_ready": production_ready, + "engineering_only": not production_ready, + "out_dir": str(root), + "paper": archived_paper, + "source_sha256": parsed["source_sha256"], + "deadline_seconds": deadline, + "deadline_reached": deadline_reached, + "elapsed_seconds": elapsed, + "planner_mode": planner_metadata.get("mode"), + "paper_review_mode": review_metadata.get("mode"), + "contract_source": contract_source, + "cache_hit": bool(planner_metadata.get("cached")), + "document_cache_hit": document_cache_hit, + "extraction_quality": { + "pdf_type": parsed["extraction_report"].get("pdf_type"), + "page_count": parsed["extraction_report"].get("page_count"), + "readable_page_ratio": parsed["extraction_report"].get("readable_page_ratio"), + "semantic_scope": parsed["extraction_report"].get("semantic_scope", "full_document"), + "scientific_scope_complete": scientific_scope_complete, + "sampled_scan_ready": parsed["extraction_report"].get("sampled_scan_ready", False), + "evidence_page_coverage_ratio": parsed["extraction_report"].get("evidence_page_coverage_ratio"), + "evidence_char_count": parsed["extraction_report"].get("evidence_char_count"), + "max_column_count": parsed["extraction_report"].get("max_column_count"), + "multi_column_page_count": parsed["extraction_report"].get("multi_column_page_count"), + "rotated_pages": parsed["extraction_report"].get("rotated_pages", []), + "section_count": parsed["extraction_report"].get("section_count", 0), + "typographic_heading_count": parsed["extraction_report"].get("typographic_heading_count", 0), + "merged_heading_line_count": parsed["extraction_report"].get("merged_heading_line_count", 0), + "figure_caption_count": parsed["extraction_report"].get("figure_caption_count", 0), + "table_caption_count": parsed["extraction_report"].get("table_caption_count", 0), + "formula_count": parsed["extraction_report"].get("formula_count", 0), + "section_coverage": parsed["extraction_report"].get("section_coverage", {}), + "missing_priority_sections": [name for name, present in parsed["extraction_report"].get("section_coverage", {}).items() if not present], + "ocr_candidate_count": len(parsed["extraction_report"].get("ocr_candidate_pages", [])), + "ocr_scheduled_count": len(parsed["extraction_report"].get("ocr_priority_pages", [])), + "ocr_completed_count": len(parsed["extraction_report"].get("ocr_pages", [])), + "ocr_attempted_count": len(parsed["extraction_report"].get("ocr_attempted_pages", [])), + "ocr_schedule_complete": parsed["extraction_report"].get("ocr_schedule_complete", True), + "ocr_run_complete": parsed["extraction_report"].get("ocr_run_complete", True), + "ocr_worker_count": parsed["extraction_report"].get("ocr_worker_count", 1), + "ocr_rescue_min_remaining_seconds": parsed["extraction_report"].get("ocr_rescue_min_remaining_seconds", 45.0), + "repeated_margin_noise_removed_count": parsed["extraction_report"].get("repeated_margin_noise_removed_count", 0), + "native_hyphenation_repair_count": parsed["extraction_report"].get("native_hyphenation_repair_count", 0), + "ocr_margin_noise_removed_count": parsed["extraction_report"].get("ocr_margin_noise_removed_count", 0), + "ocr_spacing_repair_count": parsed["extraction_report"].get("ocr_spacing_repair_count", 0), + "ocr_latin_lexical_gate_applied": parsed["extraction_report"].get("ocr_latin_lexical_gate_applied", False), + "ocr_latin_known_word_ratio": parsed["extraction_report"].get("ocr_latin_known_word_ratio"), + }, + "provider": provider_summary, + "contract_completion": { + "overview_term_coverage": completion_report.get("overview_term_coverage"), + "added_entity_count": len(completion_report.get("added_entities", [])), + "added_relation_count": len(completion_report.get("added_relations", [])), + }, + "planner_warning": planner_metadata.get("warning"), + "review_warning": review_metadata.get("warning"), + "topology": plan["figure_specification"].get("topology"), + "module_count": len(plan["figure_specification"].get("modules", [])), + "relation_count": len(plan["figure_specification"].get("relations", [])), + "stage_timings": { + "document_preparation_seconds": document_preparation_seconds, + "document_extraction_seconds": extraction_seconds, + "cached_source_extraction_seconds": source_extraction_seconds if document_cache_hit else None, + "semantic_compilation_seconds": round(max(0.0, elapsed - document_preparation_seconds), 3), + "total_seconds": elapsed, + }, + "uncertainties": plan["figure_specification"].get("uncertainties", []), + "artifacts": ["paper.md", "document_model.json", "extraction_report.json", "section_index.json", "section_summary.md", "key_evidence.json", "review_ground_truth.json", "paper_review.json", "figure_specification.json", "contract_completion_report.json", "planning_validation_report.json", "image_prompt.md", "overlay_spec.json", "run_report.json"], + } + write_json(root / "run_report.json", result) + return { + **result, + "root": root, + "parsed": parsed, + "preferences": preferences, + "paper_review": paper_review, + "review_metadata": review_metadata, + "selected_domain": selected_domain, + "plan": plan, + "planner_metadata": planner_metadata, + "archived_positive": archived_positive, + "archived_negative": archived_negative, + "planning_validation": planning_validation, + } + + +def run_fast_framework_prompt(**kwargs: Any) -> dict[str, Any]: + editable_ppt = bool(kwargs.pop("editable_ppt", False)) + verify_ppt_roundtrip = bool(kwargs.pop("verify_ppt_roundtrip", False)) + editable_ppt = editable_ppt or verify_ppt_roundtrip + if not kwargs.get("planner_model"): + kwargs["planner_model"] = os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" + kwargs["fast_mode"] = True + prepared = prepare_paper_figure_contract(**kwargs) + if editable_ppt and prepared.get("ok"): + from .editable_ppt import compile_semantic_ppt + + ppt_started = time.monotonic() + paper_summary = prepared.get("plan", {}).get("paper_summary", {}) + title = str(paper_summary.get("title") or paper_summary.get("title_guess") or "").strip() or None + ppt_result = compile_semantic_ppt( + prepared["plan"]["figure_specification"], + prepared["root"], + aspect_ratio=prepared.get("preferences", {}).get("aspect_ratio") or "16:9", + title=title, + ) + prepared.update({ + "pptx_generated": bool(ppt_result.get("ok")), + "pptx": ppt_result.get("pptx"), + "semantic_ppt": ppt_result, + }) + ppt_seconds = round(time.monotonic() - ppt_started, 3) + prepared["ppt_compile_seconds"] = ppt_seconds + prepared["elapsed_seconds"] = round(float(prepared.get("elapsed_seconds") or 0.0) + ppt_seconds, 3) + if isinstance(prepared.get("stage_timings"), dict): + prepared["stage_timings"]["ppt_compile_seconds"] = ppt_seconds + prepared["stage_timings"]["total_seconds"] = prepared["elapsed_seconds"] + prepared["artifacts"] = [*prepared.get("artifacts", []), "figure_program.json", "editable_composition.pptx", "semantic_ppt_report.json", "composition_quality_report.json"] + if verify_ppt_roundtrip and ppt_result.get("ok"): + from .ppt_roundtrip import evaluate_ppt_roundtrip + + roundtrip_started = time.monotonic() + roundtrip = evaluate_ppt_roundtrip( + read_json(Path(prepared["root"]) / "figure_program.json"), + str(ppt_result["pptx"]), + prepared["root"], + render=True, + ) + roundtrip_seconds = round(time.monotonic() - roundtrip_started, 3) + prepared["ppt_roundtrip"] = roundtrip + prepared["ppt_roundtrip_seconds"] = roundtrip_seconds + prepared["ok"] = bool(prepared.get("ok") and roundtrip.get("ok")) + prepared["production_ready"] = bool(prepared.get("production_ready") and roundtrip.get("ok")) + if not roundtrip.get("ok"): + prepared["status"] = "completed_with_warnings" + prepared["elapsed_seconds"] = round(float(prepared.get("elapsed_seconds") or 0.0) + roundtrip_seconds, 3) + if isinstance(prepared.get("stage_timings"), dict): + prepared["stage_timings"]["ppt_roundtrip_seconds"] = roundtrip_seconds + prepared["stage_timings"]["total_seconds"] = prepared["elapsed_seconds"] + prepared["artifacts"] = [*prepared.get("artifacts", []), "ppt_roundtrip_report.json", "review.pdf", "final_600dpi.png"] + write_json(Path(prepared["root"]) / "run_report.json", {key: value for key, value in prepared.items() if key not in {"root", "parsed", "preferences", "paper_review", "review_metadata", "selected_domain", "plan", "planner_metadata", "archived_positive", "archived_negative", "planning_validation"}}) + internal = {"root", "parsed", "preferences", "paper_review", "review_metadata", "selected_domain", "plan", "planner_metadata", "archived_positive", "archived_negative", "planning_validation"} + return {key: value for key, value in prepared.items() if key not in internal} diff --git a/rfs/paper_to_image/review.py b/rfs/paper_to_image/review.py index 1c17128..2033f59 100644 --- a/rfs/paper_to_image/review.py +++ b/rfs/paper_to_image/review.py @@ -68,7 +68,7 @@ def detect_domain_profile(parsed: dict, explicit: str = "auto") -> dict: return {"summary": profile["summary"], "id": selected, "selection": "automatic", "scores": scores, **profile} -def _review_prompt(parsed: dict, domain_profile: dict) -> str: +def _review_prompt(parsed: dict, domain_profile: dict, evidence_max_chars: int = 65000) -> str: return f""" # Summary @@ -143,7 +143,7 @@ def _review_prompt(parsed: dict, domain_profile: dict) -> str: - Short visible labels should normally be at most 32 characters. Paper evidence: -{evidence_excerpt(parsed, max_chars=65000)} +{evidence_excerpt(parsed, max_chars=evidence_max_chars)} """.strip() @@ -235,7 +235,9 @@ def normalize_review(raw: dict, domain_profile: dict) -> dict: def _expand_evidence(review: dict, parsed: dict) -> None: evidence = {item["id"]: item for item in parsed.get("evidence", [])} + aliases = {item.get("legacy_id"): item["id"] for item in parsed.get("evidence", []) if item.get("legacy_id")} def expand(item: dict) -> None: + item["evidence_ids"] = list(dict.fromkeys(aliases.get(str(value), str(value)) for value in item.get("evidence_ids", []))) item["evidence_refs"] = [{"evidence_id": value, "page": evidence[value].get("page"), "section": evidence[value].get("section_hint"), "quote": evidence[value].get("text", "")[:600]} for value in item.get("evidence_ids", []) if value in evidence] for field in ["research_questions", "central_claims", "inputs", "outputs", "research_objects", "concepts", "modules", "relations", "contributions", "innovations", "assumptions", "limitations", "results", "terminology", "forbidden_inventions", "unknowns"]: for item in review.get(field, []): @@ -272,7 +274,13 @@ def validate_review_coverage(review: dict, parsed: dict, domain_profile: dict, s grounded += 1 elif item.get("status") in {"required", "optional"}: errors.append(f"{field}:{item.get('id')} lacks evidence") - module_ids = {item.get("id") for item in review.get("modules", [])} | {item.get("id") for item in review.get("research_objects", [])} + endpoint_fields = ("inputs", "outputs", "research_objects", "concepts", "modules", "innovations") + module_ids = { + item.get("id") + for field in endpoint_fields + for item in review.get(field, []) + if isinstance(item, dict) and item.get("id") + } for relation in review.get("relations", []): source = relation.get("source_id") or relation.get("source") target = relation.get("target_id") or relation.get("target") @@ -320,13 +328,13 @@ def validate_review_coverage(review: dict, parsed: dict, domain_profile: dict, s } -def build_paper_review(parsed: dict, domain_profile: dict, mode: str = "vlm", model: str | None = None) -> tuple[dict, dict]: - prompt = _review_prompt(parsed, domain_profile) - metadata = {"summary": "Paper-review execution metadata.", "requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt} +def build_paper_review(parsed: dict, domain_profile: dict, mode: str = "vlm", model: str | None = None, timeout_seconds: int = 300, retries: int = 1, evidence_max_chars: int = 65000, deadline_at: float | None = None) -> tuple[dict, dict]: + prompt = _review_prompt(parsed, domain_profile, evidence_max_chars=evidence_max_chars) + metadata = {"summary": "Paper-review execution metadata.", "requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt, "provider": {}} if mode == "vlm" and vlm_credentials_available(): resolved = resolve_vlm_model("RFS_PAPER_REVIEW_MODEL", "RFS_PAPER_TO_IMAGE_MODEL", explicit_model=model) try: - raw = call_vlm_json(prompt, [], model=resolved, timeout=300, retries=1) + raw = call_vlm_json(prompt, [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries)), call_metadata=metadata["provider"], deadline_at=deadline_at) metadata["model"] = resolved review = normalize_review(raw, domain_profile) _expand_evidence(review, parsed) diff --git a/rfs/paper_to_image/review_contract.py b/rfs/paper_to_image/review_contract.py new file mode 100644 index 0000000..2109a83 --- /dev/null +++ b/rfs/paper_to_image/review_contract.py @@ -0,0 +1,340 @@ +from __future__ import annotations + +import re +from typing import Any + + +def _text(value: Any) -> str: + if isinstance(value, dict): + for key in ("text", "statement", "name", "label", "title", "description", "claim", "step"): + text = str(value.get(key) or "").strip() + if text: + return text + return "" + return str(value or "").strip() + + +def _list(value: Any) -> list[Any]: + return value if isinstance(value, list) else [] + + +def _slug(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "_", value.casefold()).strip("_") + return slug[:48] or "item" + + +def _normalized(value: str) -> str: + return re.sub(r"[^a-z0-9\u4e00-\u9fff]+", "", value.casefold()) + + +def _excerpt(value: Any, limit: int) -> str: + text = re.sub(r"\s+", " ", str(value or "")).strip() + return text if len(text) <= limit else f"{text[: limit - 1].rstrip()}…" + + +def _tokens(value: Any) -> set[str]: + stop = {"the", "and", "for", "with", "from", "into", "that", "this", "are", "was", "were", "has", "have", "via", "only", "shown", "show", "high", "level"} + return {token for token in re.findall(r"[a-z0-9\u4e00-\u9fff]+", str(value or "").casefold()) if len(token) >= 3 and token not in stop} + + +def _repair_owner(issue_type: str, instruction: str) -> str: + structural_prefixes = ( + "ocr_", + "topology_", + "scientific_missing_relations", + "scientific_reversed_relations", + "scientific_containment_mismatches", + ) + if issue_type.startswith(structural_prefixes): + return "overlay_compiler" + normalized = instruction.casefold() + structural_terms = ( + "arrow", + "connector", + "edge direction", + "reversed relation", + "missing relation", + "missing label", + "misspelled", + "duplicate label", + "forbidden label", + "unexpected label", + "ocr", + "text spelling", + ) + return "overlay_compiler" if any(term in normalized for term in structural_terms) else "image2_visual_substrate" + + +def compile_review_ground_truth(plan: dict, evidence_records: list[dict] | None = None) -> dict: + """Compile a compact, evidence-bearing contract used only for image review. + + The figure specification remains the semantic source of truth. This view + adds the original evidence excerpts needed by a visual critic to check the + candidate against the paper instead of judging only a summarized plan. + """ + + specification = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} + evidence_records = [item for item in (evidence_records or []) if isinstance(item, dict)] + evidence_index: dict[str, dict] = {} + for item in evidence_records: + for key in (item.get("id"), item.get("legacy_id")): + if key: + evidence_index[str(key)] = item + searchable_evidence = [ + (item, _normalized(str(item.get("text") or ""))) + for item in evidence_records + if _normalized(str(item.get("text") or "")) + ] + + entity_labels: dict[str, str] = {} + for field in ("inputs", "modules", "outputs", "innovations"): + for item in _list(specification.get(field)): + if not isinstance(item, dict): + continue + item_id = str(item.get("id") or "").strip() + label = _text(item) + if item_id and label: + entity_labels[item_id] = label + + priorities: list[dict] = [] + seen: set[tuple[str, str]] = set() + + def add_priority(kind: str, value: Any, *, importance: str = "required", visual_expectation: str | None = None) -> None: + source_id = "" + target_id = "" + if isinstance(value, dict): + label = _text(value) + evidence_ids = [str(item) for item in _list(value.get("evidence_ids")) if str(item).strip()] + source_id = str(value.get("source") or "").strip() + target_id = str(value.get("target") or "").strip() + if kind in {"relation", "feedback_loop", "training_flow", "inference_flow"} and (source_id or target_id): + source = entity_labels.get(source_id, source_id) + target = entity_labels.get(target_id, target_id) + relation_type = str(value.get("type") or value.get("relation") or "directed relation").strip() + label = f"{source} -> {target} [{relation_type}]" + item_id = str(value.get("id") or "").strip() + status = str(value.get("status") or importance).strip().casefold() + explicitly_required = bool(value.get("required") is True or value.get("must_appear_in_figure") or value.get("visible_label_required")) + explicitly_optional = bool(value.get("required") is False or value.get("optional") or status in {"optional", "unknown", "uncertain"}) + required = explicitly_required or (not explicitly_optional and importance != "optional") + else: + label = _text(value) + evidence_ids = [] + item_id = "" + required = importance != "optional" + if not label: + return + semantic_kind = "relation" if kind in {"relation", "feedback_loop"} else "flow" if kind in {"training_flow", "inference_flow"} and (source_id or target_id) else "concept" + key = (semantic_kind, _normalized(label)) + if key in seen: + return + seen.add(key) + excerpts = [] + for evidence_id in evidence_ids: + record = evidence_index.get(evidence_id) + if not record: + continue + excerpts.append({ + "evidence_id": str(record.get("id") or evidence_id), + "page": record.get("page"), + "section": record.get("section_hint") or record.get("section"), + "text": _excerpt(record.get("text"), 240), + "source": record.get("source"), + "confidence": record.get("confidence"), + }) + link_method = "explicit" if excerpts else "none" + if not excerpts: + needle = _normalized(label) + if len(needle) >= 3: + matches = [] + needle_tokens = _tokens(label) + for item, haystack in searchable_evidence: + reverse_fragment = len(haystack) >= 8 and len(haystack) / len(needle) >= 0.45 and haystack in needle + evidence_tokens = _tokens(item.get("text")) + token_coverage = len(needle_tokens & evidence_tokens) / max(1, len(needle_tokens)) + if needle == haystack: + rank = 0 + elif needle in haystack: + rank = 1 + elif reverse_fragment: + rank = 2 + elif len(needle_tokens) >= 3 and token_coverage >= 0.8: + rank = 3 + else: + continue + matches.append((rank, len(str(item.get("text") or "")), int(item.get("page") or 0), item)) + matches.sort(key=lambda match: match[:3]) + for _, _, _, record in matches[:2]: + record_id = str(record.get("id") or record.get("legacy_id") or "").strip() + if record_id and record_id not in evidence_ids: + evidence_ids.append(record_id) + excerpts.append({ + "evidence_id": record_id, + "page": record.get("page"), + "section": record.get("section_hint") or record.get("section"), + "text": _excerpt(record.get("text"), 240), + "source": record.get("source"), + "confidence": record.get("confidence"), + }) + if excerpts: + link_method = "deterministic_text_match" + priorities.append({ + "id": item_id or f"GT_{kind.upper()}_{_slug(label)}", + "kind": kind, + "label": label, + "importance": "critical" if importance == "critical" else "required" if required else "optional", + "required": required, + "visual_expectation": visual_expectation or ("visible and scientifically correct" if required else "may be visible when useful"), + "evidence_ids": evidence_ids, + "evidence": excerpts, + "evidence_link_method": link_method, + }) + + add_priority("research_problem", specification.get("research_problem"), importance="critical", visual_expectation="the problem context is visually recognizable") + add_priority("central_claim", specification.get("central_claim"), importance="critical", visual_expectation="the figure's main visual story supports this claim") + for field, kind, expectation in ( + ("inputs", "input", "shown with the correct semantic role as an input or source"), + ("modules", "module", "shown as the named paper-specific component"), + ("relations", "relation", "shown with the correct source, target, and direction"), + ("feedback_loops", "feedback_loop", "shown as a closed and correctly directed loop"), + ("outputs", "output", "shown as an output rather than an internal module"), + ("innovations", "innovation", "the paper's novelty is visually identifiable"), + ("must_show", "must_show", "explicitly visible in the figure"), + ): + for item in _list(specification.get(field)): + add_priority(kind, item, importance="critical" if kind in {"module", "relation", "feedback_loop"} else "required", visual_expectation=expectation) + for item in _list(specification.get("training_flow")): + add_priority("training_flow", item, importance="optional", visual_expectation="when visible, shown only in the training path") + for item in _list(specification.get("inference_flow")): + add_priority("inference_flow", item, importance="optional", visual_expectation="when visible, shown only in the inference path") + + for label in _list(specification.get("required_labels")): + add_priority("required_label", label, importance="required", visual_expectation="appears as exact visible text") + for label in _list(specification.get("optional_visible_labels")): + add_priority("optional_label", label, importance="optional", visual_expectation="allowed but not required") + + forbidden = [] + for index, item in enumerate(_list(specification.get("forbidden_inventions")), start=1): + label = _text(item) + if label: + forbidden.append({"id": f"FORBIDDEN_{index:03d}", "label": label, "rule": "must not be asserted or visually implied"}) + + unknown = [] + for index, item in enumerate(_list(specification.get("uncertainties")), start=1): + label = _text(item) + if label: + unknown.append({"id": f"UNKNOWN_{index:03d}", "label": label, "rule": "do not resolve by invention"}) + + evidence_catalog: dict[str, dict] = {} + for priority in priorities: + for item in priority.get("evidence", []): + evidence_id = str(item.get("evidence_id") or "").strip() + record = evidence_index.get(evidence_id) + if evidence_id and record and evidence_id not in evidence_catalog: + evidence_catalog[evidence_id] = { + "evidence_id": evidence_id, + "page": record.get("page"), + "section": record.get("section_hint") or record.get("section"), + "text": _excerpt(record.get("text"), 900), + "source": record.get("source"), + "confidence": record.get("confidence"), + } + required_priorities = [item for item in priorities if item["required"]] + grounded_required = [item for item in required_priorities if item["evidence"]] + return { + "summary": "Paper-evidence ground truth for visual candidate review.", + "source": "figure_specification + key_evidence", + "evidence_available": bool(grounded_required), + "strict_evidence_required": bool(evidence_records and required_priorities), + "required_priority_count": len(required_priorities), + "grounded_required_priority_count": len(grounded_required), + "priorities": priorities, + "evidence_catalog": list(evidence_catalog.values()), + "forbidden": forbidden, + "unknown": unknown, + } + + +def build_repair_plan(review: dict, source_candidate_id: str | None = None, round_index: int | None = None) -> dict: + """Turn critic output into a deterministic, auditable Image2 edit plan.""" + + items: list[dict] = [] + seen: set[tuple[str, str]] = set() + preserve = [str(item).strip() for item in review.get("preserve", []) if str(item).strip()] + + def add(issue_type: str, issue: Any, severity: str, expected_check: str, evidence_ids: list[str] | None = None) -> None: + if isinstance(issue, dict): + instruction = str(issue.get("repair") or issue.get("instruction") or issue.get("reason") or issue.get("label") or issue).strip() + region = issue.get("region") or issue.get("visible_region") or issue.get("target") or "auto-localize" + ids = [str(item) for item in issue.get("evidence_ids", []) if str(item).strip()] + else: + instruction = str(issue).strip() + region = "auto-localize" + ids = [] + if not instruction: + return + key = (issue_type, instruction.casefold()) + if key in seen: + return + seen.add(key) + items.append({ + "id": f"RP_{len(items) + 1:03d}", + "region": region, + "issue_type": issue_type, + "severity": severity, + "owner": _repair_owner(issue_type, instruction), + "evidence_ids": list(dict.fromkeys([*(evidence_ids or []), *ids])), + "preserve": preserve, + "instruction": instruction, + "expected_check": expected_check, + }) + + alignment = review.get("paper_alignment", {}) if isinstance(review.get("paper_alignment"), dict) else {} + for issue in alignment.get("missing_priorities", []): + ids = issue.get("evidence_ids", []) if isinstance(issue, dict) else [] + add("paper_missing_priority", issue, "critical", "paper_alignment.missing_priorities no longer contains this priority", ids) + for issue in alignment.get("evidence_conflicts", []): + ids = issue.get("evidence_ids", []) if isinstance(issue, dict) else [] + add("paper_evidence_conflict", issue, "critical", "paper_alignment.evidence_conflicts is empty for this claim", ids) + for issue in alignment.get("unsupported_visual_claims", []): + add("unsupported_visual_claim", issue, "critical", "paper_alignment.unsupported_visual_claims no longer contains this claim") + + scientific = review.get("scientific", {}) if isinstance(review.get("scientific"), dict) else {} + for field in ("missing_modules", "missing_relations", "reversed_relations", "invented_items", "role_mismatches", "containment_mismatches"): + for issue in scientific.get(field, []): + add(f"scientific_{field}", issue, "critical", f"scientific.{field} is empty for this issue") + + topology = review.get("topology", {}) if isinstance(review.get("topology"), dict) else {} + for field in ("missing_relations", "reversed_relations", "bypassed_relations", "invented_relations"): + for issue in topology.get(field, []): + add(f"topology_{field}", issue, "critical", f"topology.{field} is empty for this connector") + + ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} + unexpected_field = "unsupported_unexpected_labels" if "unsupported_unexpected_labels" in ocr else "unexpected_labels" + for field in ("missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found", unexpected_field): + for issue in ocr.get(field, []): + add(f"ocr_{field}", issue, "high", f"ocr.{field} is empty for this label") + + for instruction in review.get("repair", []): + add("critic_instruction", instruction, "medium", "the originating critic issue is resolved") + for issue in review.get("hard_errors", []): + add("hard_error", issue, "critical", "the hard error is absent after full re-review") + + template = review.get("template", {}) if isinstance(review.get("template"), dict) else {} + if not template.get("passed", False): + add("template_alignment", "Improve macro-panel alignment, reading order, connector rhythm, and density without changing scientific content.", "medium", "template.passed is true") + aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} + if not aesthetic.get("passed", False): + add("aesthetic_quality", "Improve hierarchy, balance, whitespace, color consistency, mechanism visualization, and publication polish while preserving every correct scientific element.", "medium", "aesthetic.passed is true") + basic = review.get("basic", {}) if isinstance(review.get("basic"), dict) else {} + if not basic.get("visual_enrichment_passed", True): + add("visual_enrichment", "Add paper-grounded mechanism cues and scientific visual detail inside existing regions; do not alter labels or connector topology.", "high", "basic.visual_enrichment_passed is true") + + return { + "summary": "Machine-executable repair plan compiled from all candidate critics.", + "source_candidate_id": source_candidate_id, + "round_index": round_index, + "item_count": len(items), + "preserve": preserve, + "items": items, + } diff --git a/rfs/paper_to_image/semantic_blueprint.py b/rfs/paper_to_image/semantic_blueprint.py new file mode 100644 index 0000000..a8a4f37 --- /dev/null +++ b/rfs/paper_to_image/semantic_blueprint.py @@ -0,0 +1,894 @@ +from __future__ import annotations + +from collections import defaultdict, deque +from math import hypot +from typing import Any + + +_SUPPORTED_TOPOLOGIES = {"linear", "branch", "multimodal", "feedback", "dense_multiframe"} +_FEEDBACK_TYPES = {"feedback_loop", "iteration", "iterate", "return_flow"} + + +def _label(item: dict[str, Any]) -> str: + return str(item.get("name") or item.get("label") or item.get("title") or "").strip() + + +def _normalized(value: Any) -> str: + return "".join(char for char in str(value or "").casefold() if char.isalnum()) + + +def _visible_entities(specification: dict[str, Any]) -> list[dict[str, Any]]: + required = { + _normalized(value) + for value in specification.get("required_labels", []) + if str(value).strip() + } + entities: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + seen_labels: set[str] = set() + for field in ("inputs", "modules", "outputs", "innovations"): + values = specification.get(field, []) + if not isinstance(values, list): + continue + for index, item in enumerate(values): + if not isinstance(item, dict): + continue + label = _label(item) + entity_id = str(item.get("id") or f"{field}_{index + 1}").strip() + normalized_label = _normalized(label) + if not label or not entity_id or entity_id in seen_ids or normalized_label in seen_labels: + continue + if required and normalized_label not in required: + continue + seen_ids.add(entity_id) + seen_labels.add(normalized_label) + entities.append({ + "id": entity_id, + "label": label, + "field": field, + "role": str(item.get("role") or "").strip(), + "evidence_ids": [str(value) for value in item.get("evidence_ids", []) if value], + "order": len(entities), + }) + return entities + + +def _dense_overview_panels(specification: dict[str, Any]) -> list[dict[str, Any]]: + values = specification.get("overview_panels", []) + if not isinstance(values, list): + return [] + panels: list[dict[str, Any]] = [] + for index, item in enumerate(values): + if not isinstance(item, dict): + continue + label = str(item.get("label") or item.get("title") or "").strip() + panel_id = str(item.get("id") or f"panel_{index + 1}").strip() + if not label or not panel_id: + continue + panels.append({ + "id": panel_id, + "panel_label": str(item.get("panel_label") or chr(ord("a") + index)).strip(), + "label": label, + "description": str(item.get("description") or "").strip(), + "evidence_ids": [str(value) for value in item.get("evidence_ids", []) if value], + "entity_ids": [str(value) for value in item.get("entity_ids", []) if value], + "order": index, + }) + return panels + + +def _layout_dense_panels(panels: list[dict[str, Any]]) -> list[dict[str, Any]]: + count = len(panels) + positioned: list[dict[str, Any]] = [] + if count == 3: + boxes = [ + {"x": 0.04, "y": 0.05, "w": 0.92, "h": 0.42}, + {"x": 0.04, "y": 0.53, "w": 0.44, "h": 0.42}, + {"x": 0.52, "y": 0.53, "w": 0.44, "h": 0.42}, + ] + else: + columns = 2 if count <= 4 else 3 + rows = (count + columns - 1) // columns + gap_x, gap_y = 0.04, 0.06 + width = (0.92 - gap_x * (columns - 1)) / columns + height = (0.90 - gap_y * (rows - 1)) / rows + boxes = [ + { + "x": round(0.04 + (index % columns) * (width + gap_x), 6), + "y": round(0.05 + (index // columns) * (height + gap_y), 6), + "w": round(width, 6), + "h": round(height, 6), + } + for index in range(count) + ] + for panel, box in zip(panels, boxes): + positioned.append({**panel, "bbox_percent": box}) + return positioned + + +def _layout_nodes_in_dense_panels( + nodes: list[dict[str, Any]], + relations: list[dict[str, str]], + panels: list[dict[str, Any]], + global_ranks: dict[str, int], +) -> list[dict[str, Any]]: + if not panels: + return _layout_nodes(nodes, _rank_nodes(nodes, relations)) + panel_by_entity = { + entity_id: panel["id"] + for panel in panels + for entity_id in panel.get("entity_ids", []) + } + panel_by_evidence = { + evidence_id: panel["id"] + for panel in panels + for evidence_id in panel.get("evidence_ids", []) + } + default_panel = panels[0]["id"] + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for node in nodes: + panel_id = panel_by_entity.get(node["id"]) + if not panel_id: + panel_id = next((panel_by_evidence[value] for value in node.get("evidence_ids", []) if value in panel_by_evidence), None) + grouped[panel_id or default_panel].append(node) + by_panel = {panel["id"]: panel for panel in panels} + positioned: list[dict[str, Any]] = [] + for panel_id, panel_nodes in grouped.items(): + panel = by_panel.get(panel_id) or panels[0] + node_ids = {item["id"] for item in panel_nodes} + local_relations = [ + item for item in relations + if item["source"] in node_ids and item["target"] in node_ids + ] + local_layout = _layout_nodes(panel_nodes, _rank_nodes(panel_nodes, local_relations)) + box = panel["bbox_percent"] + inner_x = box["x"] + box["w"] * 0.035 + inner_y = box["y"] + box["h"] * 0.24 + inner_w = box["w"] * 0.93 + inner_h = box["h"] * 0.70 + for item in local_layout: + local_box = item["bbox_percent"] + positioned.append({ + **item, + "panel_id": panel["id"], + "global_rank": int(global_ranks.get(item["id"], item.get("rank", 0))), + "bbox_percent": { + "x": round(inner_x + local_box["x"] * inner_w, 6), + "y": round(inner_y + local_box["y"] * inner_h, 6), + "w": round(local_box["w"] * inner_w, 6), + "h": round(local_box["h"] * inner_h, 6), + }, + }) + return positioned + + +def _visible_relations(specification: dict[str, Any], node_ids: set[str]) -> list[dict[str, str]]: + relations: list[dict[str, str]] = [] + seen: set[tuple[str, str]] = set() + values = specification.get("relations", []) + if not isinstance(values, list): + return relations + type_priority = { + "feedback_loop": 0, + "branch": 1, + "conditioning": 2, + "feature_flow": 3, + "data_flow": 4, + } + candidates: dict[tuple[str, str], list[dict[str, str]]] = defaultdict(list) + for item in values: + if not isinstance(item, dict): + continue + source = str(item.get("source") or "").strip() + target = str(item.get("target") or "").strip() + if not source or not target or source == target or source not in node_ids or target not in node_ids: + continue + candidates[(source, target)].append({ + "source": source, + "target": target, + "type": str(item.get("type") or "data_flow").strip(), + "label": str(item.get("label") or "").strip(), + }) + for pair, items in candidates.items(): + if pair in seen: + continue + selected = sorted(items, key=lambda item: type_priority.get(item["type"], 9))[0] + labels = [item["label"] for item in items if item["label"]] + if labels: + selected["label"] = labels[0] + relations.append(selected) + seen.add(pair) + return relations + + +def _rank_nodes(nodes: list[dict[str, Any]], relations: list[dict[str, str]]) -> dict[str, int]: + node_ids = [item["id"] for item in nodes] + incoming: dict[str, set[str]] = {node_id: set() for node_id in node_ids} + outgoing: dict[str, set[str]] = {node_id: set() for node_id in node_ids} + forward_relations = [ + item for item in relations + if item["type"] not in _FEEDBACK_TYPES + ] + for item in forward_relations: + incoming[item["target"]].add(item["source"]) + outgoing[item["source"]].add(item["target"]) + + indegree = {node_id: len(incoming[node_id]) for node_id in node_ids} + order_lookup = {item["id"]: item["order"] for item in nodes} + queue = deque(sorted((node_id for node_id in node_ids if indegree[node_id] == 0), key=order_lookup.get)) + ranks = {node_id: 0 for node_id in node_ids} + visited: set[str] = set() + while queue: + source = queue.popleft() + visited.add(source) + for target in sorted(outgoing[source], key=order_lookup.get): + ranks[target] = max(ranks[target], ranks[source] + 1) + indegree[target] -= 1 + if indegree[target] == 0: + queue.append(target) + + if len(visited) != len(node_ids): + for _ in range(len(node_ids)): + changed = False + for item in forward_relations: + candidate = min(len(node_ids) - 1, ranks[item["source"]] + 1) + if candidate > ranks[item["target"]]: + ranks[item["target"]] = candidate + changed = True + if not changed: + break + + input_ids = {item["id"] for item in nodes if item["field"] == "inputs"} + for node_id in input_ids: + if not incoming[node_id]: + ranks[node_id] = 0 + relations_by_source: dict[str, list[dict[str, str]]] = defaultdict(list) + for relation in forward_relations: + relations_by_source[relation["source"]].append(relation) + for node in nodes: + node_id = node["id"] + outgoing_relations = relations_by_source.get(node_id, []) + if incoming[node_id] or not outgoing_relations: + continue + if all(str(item.get("type") or "").casefold() == "conditioning" for item in outgoing_relations): + target_rank = min(ranks[item["target"]] for item in outgoing_relations) + ranks[node_id] = max(0, target_rank - 1) + return ranks + + +def _layout_nodes(nodes: list[dict[str, Any]], ranks: dict[str, int]) -> list[dict[str, Any]]: + layers: dict[int, list[dict[str, Any]]] = defaultdict(list) + for item in nodes: + layers[ranks[item["id"]]].append(item) + compact_ranks = {rank: index for index, rank in enumerate(sorted(layers))} + layer_count = max(1, len(compact_ranks)) + node_width = min(0.16, max(0.115, 0.72 / layer_count)) + x_start, x_end = 0.07, 0.93 + positioned: list[dict[str, Any]] = [] + for original_rank in sorted(layers): + layer_index = compact_ranks[original_rank] + items = sorted(layers[original_rank], key=lambda item: (item["field"] == "innovations", item["field"] == "outputs", item["order"])) + count = len(items) + center_x = (x_start + x_end) / 2 if layer_count == 1 else x_start + layer_index * (x_end - x_start) / (layer_count - 1) + node_height = min(0.16, max(0.075, 0.66 / max(count, 1))) + if count == 1: + centers_y = [0.50] + else: + y_start, y_end = 0.15, 0.85 + centers_y = [y_start + index * (y_end - y_start) / (count - 1) for index in range(count)] + for item, center_y in zip(items, centers_y): + x = max(0.015, min(0.985 - node_width, center_x - node_width / 2)) + positioned.append({ + **item, + "rank": layer_index, + "bbox_percent": { + "x": round(x, 6), + "y": round(max(0.06, center_y - node_height / 2), 6), + "w": round(node_width, 6), + "h": round(node_height, 6), + }, + }) + return positioned + + +def _connector_path( + source: dict[str, Any], + target: dict[str, Any], + relation_type: str, + source_port_fraction: float = 0.5, + target_port_fraction: float = 0.5, +) -> tuple[list[list[float]], str]: + source_box = source["bbox_percent"] + target_box = target["bbox_percent"] + source_center = (source_box["x"] + source_box["w"] / 2, source_box["y"] + source_box["h"] / 2) + target_center = (target_box["x"] + target_box["w"] / 2, target_box["y"] + target_box["h"] / 2) + source_rank = int(source.get("global_rank", source.get("rank", 0))) + target_rank = int(target.get("global_rank", target.get("rank", 0))) + is_return = relation_type in _FEEDBACK_TYPES or target_rank <= source_rank + if is_return: + route_y = 0.95 if source_center[1] >= 0.42 or target_center[1] >= 0.42 else 0.05 + source_anchor_y = source_box["y"] + source_box["h"] if route_y > 0.5 else source_box["y"] + target_anchor_y = target_box["y"] + target_box["h"] if route_y > 0.5 else target_box["y"] + return ([ + [round(source_center[0], 6), round(source_anchor_y, 6)], + [round(source_center[0], 6), route_y], + [round(target_center[0], 6), route_y], + [round(target_center[0], 6), round(target_anchor_y, 6)], + ], "outer_feedback") + if source_rank == target_rank: + if source_center[1] <= target_center[1]: + start = [source_center[0], source_box["y"] + source_box["h"]] + end = [target_center[0], target_box["y"]] + else: + start = [source_center[0], source_box["y"]] + end = [target_center[0], target_box["y"] + target_box["h"]] + return ([[round(value, 6) for value in start], [round(value, 6) for value in end]], "vertical") + if target_center[0] >= source_center[0]: + start = [source_box["x"] + source_box["w"], source_box["y"] + source_port_fraction * source_box["h"]] + end = [target_box["x"], target_box["y"] + target_port_fraction * target_box["h"]] + else: + start = [source_box["x"], source_box["y"] + source_port_fraction * source_box["h"]] + end = [target_box["x"] + target_box["w"], target_box["y"] + target_port_fraction * target_box["h"]] + if target_rank - source_rank > 1: + lane_y = max(0.035, min(0.965, min(source_box["y"], target_box["y"]) - 0.045)) + if lane_y < 0.055: + lane_y = min(0.965, max(source_box["y"] + source_box["h"], target_box["y"] + target_box["h"]) + 0.045) + return ([ + [round(value, 6) for value in start], + [round(start[0] + 0.025, 6), round(start[1], 6)], + [round(start[0] + 0.025, 6), round(lane_y, 6)], + [round(end[0] - 0.025, 6), round(lane_y, 6)], + [round(end[0] - 0.025, 6), round(end[1], 6)], + [round(value, 6) for value in end], + ], "bypass_orthogonal") + if abs(start[1] - end[1]) < 0.025: + return ([[round(value, 6) for value in start], [round(value, 6) for value in end]], "straight") + lane_fraction = 0.20 + 0.60 * (0.65 * target_port_fraction + 0.35 * source_port_fraction) + mid_x = start[0] + (end[0] - start[0]) * lane_fraction + return ([ + [round(value, 6) for value in start], + [round(mid_x, 6), round(start[1], 6)], + [round(mid_x, 6), round(end[1], 6)], + [round(value, 6) for value in end], + ], "orthogonal") + + +def _segments(path: list[list[float]]) -> list[tuple[tuple[float, float], tuple[float, float]]]: + return [ + ((float(start[0]), float(start[1])), (float(end[0]), float(end[1]))) + for start, end in zip(path[:-1], path[1:]) + if start != end + ] + + +def _segment_length(segment: tuple[tuple[float, float], tuple[float, float]]) -> float: + return hypot(segment[1][0] - segment[0][0], segment[1][1] - segment[0][1]) + + +def _path_length(path: list[list[float]]) -> float: + return sum(_segment_length(segment) for segment in _segments(path)) + + +def _point_on_segment( + point: tuple[float, float], + segment: tuple[tuple[float, float], tuple[float, float]], + *, + tolerance: float = 1e-7, +) -> bool: + (x, y), ((x1, y1), (x2, y2)) = point, segment + cross = (x - x1) * (y2 - y1) - (y - y1) * (x2 - x1) + if abs(cross) > tolerance: + return False + return min(x1, x2) - tolerance <= x <= max(x1, x2) + tolerance and min(y1, y2) - tolerance <= y <= max(y1, y2) + tolerance + + +def _segment_crosses( + first: tuple[tuple[float, float], tuple[float, float]], + second: tuple[tuple[float, float], tuple[float, float]], +) -> bool: + (a1, a2), (b1, b2) = first, second + first_vertical = abs(a1[0] - a2[0]) < 1e-7 + second_vertical = abs(b1[0] - b2[0]) < 1e-7 + if first_vertical == second_vertical: + return False + vertical, horizontal = (first, second) if first_vertical else (second, first) + intersection = (vertical[0][0], horizontal[0][1]) + if not _point_on_segment(intersection, vertical) or not _point_on_segment(intersection, horizontal): + return False + endpoints = {vertical[0], vertical[1], horizontal[0], horizontal[1]} + return intersection not in endpoints + + +def _shared_segment_overlap( + first: tuple[tuple[float, float], tuple[float, float]], + second: tuple[tuple[float, float], tuple[float, float]], + *, + lane_tolerance: float = 0.006, +) -> float: + first_vertical = abs(first[0][0] - first[1][0]) < 1e-7 + second_vertical = abs(second[0][0] - second[1][0]) < 1e-7 + if first_vertical != second_vertical: + return 0.0 + if first_vertical: + if abs(first[0][0] - second[0][0]) > lane_tolerance: + return 0.0 + first_range = sorted((first[0][1], first[1][1])) + second_range = sorted((second[0][1], second[1][1])) + else: + if abs(first[0][1] - second[0][1]) > lane_tolerance: + return 0.0 + first_range = sorted((first[0][0], first[1][0])) + second_range = sorted((second[0][0], second[1][0])) + return max(0.0, min(first_range[1], second_range[1]) - max(first_range[0], second_range[0])) + + +def _segment_intersects_box( + segment: tuple[tuple[float, float], tuple[float, float]], + box: dict[str, Any], + *, + padding: float = 0.006, +) -> bool: + left = float(box["x"]) - padding + right = float(box["x"]) + float(box["w"]) + padding + top = float(box["y"]) - padding + bottom = float(box["y"]) + float(box["h"]) + padding + (x1, y1), (x2, y2) = segment + if abs(x1 - x2) < 1e-7: + return left < x1 < right and max(min(y1, y2), top) < min(max(y1, y2), bottom) + if abs(y1 - y2) < 1e-7: + return top < y1 < bottom and max(min(x1, x2), left) < min(max(x1, x2), right) + return False + + +def _compact_path(path: list[list[float]]) -> list[list[float]]: + compact: list[list[float]] = [] + for point in path: + normalized = [round(float(point[0]), 6), round(float(point[1]), 6)] + if compact and normalized == compact[-1]: + continue + compact.append(normalized) + while len(compact) >= 3: + a, b, c = compact[-3:] + if (abs(a[0] - b[0]) < 1e-7 and abs(b[0] - c[0]) < 1e-7) or (abs(a[1] - b[1]) < 1e-7 and abs(b[1] - c[1]) < 1e-7): + compact.pop(-2) + else: + break + return compact + + +def _node_center_y(node: dict[str, Any]) -> float: + box = node["bbox_percent"] + return float(box["y"]) + float(box["h"]) / 2 + + +def _relation_ports( + relations: list[dict[str, str]], + by_id: dict[str, dict[str, Any]], +) -> dict[int, tuple[float, float]]: + incoming: dict[str, list[dict[str, str]]] = defaultdict(list) + outgoing: dict[str, list[dict[str, str]]] = defaultdict(list) + for relation in relations: + incoming[relation["target"]].append(relation) + outgoing[relation["source"]].append(relation) + ports: dict[int, tuple[float, float]] = {} + for relation in relations: + target_items = sorted( + incoming[relation["target"]], + key=lambda item: (_node_center_y(by_id[item["source"]]), relations.index(item)), + ) + source_items = sorted( + outgoing[relation["source"]], + key=lambda item: (_node_center_y(by_id[item["target"]]), relations.index(item)), + ) + ports[id(relation)] = ( + (source_items.index(relation) + 1) / (len(source_items) + 1), + (target_items.index(relation) + 1) / (len(target_items) + 1), + ) + return ports + + +def _forward_endpoints( + source: dict[str, Any], + target: dict[str, Any], + source_fraction: float, + target_fraction: float, +) -> tuple[list[float], list[float]]: + source_box = source["bbox_percent"] + target_box = target["bbox_percent"] + source_center_x = source_box["x"] + source_box["w"] / 2 + target_center_x = target_box["x"] + target_box["w"] / 2 + if target_center_x >= source_center_x: + return ( + [source_box["x"] + source_box["w"], source_box["y"] + source_fraction * source_box["h"]], + [target_box["x"], target_box["y"] + target_fraction * target_box["h"]], + ) + return ( + [source_box["x"], source_box["y"] + source_fraction * source_box["h"]], + [target_box["x"] + target_box["w"], target_box["y"] + target_fraction * target_box["h"]], + ) + + +def _route_candidates( + source: dict[str, Any], + target: dict[str, Any], + relation_type: str, + source_fraction: float, + target_fraction: float, +) -> list[dict[str, Any]]: + base_path, base_style = _connector_path( + source, + target, + relation_type, + source_port_fraction=source_fraction, + target_port_fraction=target_fraction, + ) + candidates = [{"path": base_path, "route_style": base_style, "lane_assignment": "local_default"}] + source_rank = int(source.get("global_rank", source.get("rank", 0))) + target_rank = int(target.get("global_rank", target.get("rank", 0))) + is_return = relation_type in _FEEDBACK_TYPES or target_rank <= source_rank + if is_return: + source_box, target_box = source["bbox_percent"], target["bbox_percent"] + source_center_x = source_box["x"] + source_box["w"] / 2 + target_center_x = target_box["x"] + target_box["w"] / 2 + for lane_name, lane_y in (("outer_top", 0.035), ("outer_bottom", 0.965)): + source_anchor_y = source_box["y"] if lane_y < 0.5 else source_box["y"] + source_box["h"] + target_anchor_y = target_box["y"] if lane_y < 0.5 else target_box["y"] + target_box["h"] + candidates.append({ + "path": [[source_center_x, source_anchor_y], [source_center_x, lane_y], [target_center_x, lane_y], [target_center_x, target_anchor_y]], + "route_style": "outer_feedback", + "lane_assignment": lane_name, + }) + return candidates + if source_rank == target_rank: + return candidates + start, end = _forward_endpoints(source, target, source_fraction, target_fraction) + direction = 1.0 if end[0] >= start[0] else -1.0 + if target_rank - source_rank > 1: + source_box = source["bbox_percent"] + target_box = target["bbox_percent"] + source_center_x = source_box["x"] + source_box["w"] / 2 + target_center_x = target_box["x"] + target_box["w"] / 2 + for lane_index, lane_y in enumerate((0.035, 0.055, 0.945, 0.965), start=1): + source_x = start[0] + direction * (0.020 + 0.008 * (lane_index % 2)) + target_x = end[0] - direction * (0.020 + 0.008 * ((lane_index + 1) % 2)) + candidates.append({ + "path": [start, [source_x, start[1]], [source_x, lane_y], [target_x, lane_y], [target_x, end[1]], end], + "route_style": "bypass_orthogonal", + "lane_assignment": ("outer_top" if lane_y < 0.5 else "outer_bottom") + f"_{lane_index}", + }) + source_edge_y = source_box["y"] if lane_y < 0.5 else source_box["y"] + source_box["h"] + target_edge_y = target_box["y"] if lane_y < 0.5 else target_box["y"] + target_box["h"] + candidates.append({ + "path": [[source_center_x, source_edge_y], [source_center_x, lane_y], [target_center_x, lane_y], [target_center_x, target_edge_y]], + "route_style": "bypass_orthogonal", + "lane_assignment": ("outer_top_edge" if lane_y < 0.5 else "outer_bottom_edge") + f"_{lane_index}", + }) + return candidates + if abs(start[1] - end[1]) >= 0.025: + for lane_index, fraction in enumerate((0.24, 0.38, 0.50, 0.62, 0.76), start=1): + lane_x = start[0] + (end[0] - start[0]) * fraction + candidates.append({ + "path": [start, [lane_x, start[1]], [lane_x, end[1]], end], + "route_style": "orthogonal", + "lane_assignment": f"inter_rank_{lane_index}", + }) + return candidates + + +def _candidate_metrics( + candidate_path: list[list[float]], + relation: dict[str, str], + nodes: list[dict[str, Any]], + accepted_paths: list[list[list[float]]], +) -> dict[str, float | int]: + candidate_segments = _segments(candidate_path) + obstacle_intersections = 0 + for node in nodes: + if node["id"] in {relation["source"], relation["target"]}: + continue + if any(_segment_intersects_box(segment, node["bbox_percent"]) for segment in candidate_segments): + obstacle_intersections += 1 + crossings = 0 + shared_overlap = 0.0 + for accepted_path in accepted_paths: + accepted_segments = _segments(accepted_path) + crossings += sum(1 for first in candidate_segments for second in accepted_segments if _segment_crosses(first, second)) + shared_overlap += sum(_shared_segment_overlap(first, second) for first in candidate_segments for second in accepted_segments) + return { + "node_obstacle_intersections": obstacle_intersections, + "crossing_count": crossings, + "shared_segment_overlap": round(shared_overlap, 6), + "bend_count": max(0, len(candidate_path) - 2), + "path_length": round(_path_length(candidate_path), 6), + } + + +def _route_connectors( + positioned_nodes: list[dict[str, Any]], + relations: list[dict[str, str]], +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + by_id = {item["id"]: item for item in positioned_nodes} + ports = _relation_ports(relations, by_id) + indexed_relations = list(enumerate(relations)) + + def priority(item: tuple[int, dict[str, str]]) -> tuple[int, int, int]: + index, relation = item + source = by_id[relation["source"]] + target = by_id[relation["target"]] + source_rank = int(source.get("global_rank", source.get("rank", 0))) + target_rank = int(target.get("global_rank", target.get("rank", 0))) + is_return = relation["type"] in _FEEDBACK_TYPES or target_rank <= source_rank + return (2 if is_return else 1 if target_rank - source_rank > 1 else 0, abs(target_rank - source_rank), index) + + accepted_paths: list[list[list[float]]] = [] + routed: dict[int, dict[str, Any]] = {} + for original_index, relation in sorted(indexed_relations, key=priority): + source_fraction, target_fraction = ports[id(relation)] + candidates = _route_candidates( + by_id[relation["source"]], + by_id[relation["target"]], + relation["type"], + source_fraction, + target_fraction, + ) + scored = [] + for candidate_index, candidate in enumerate(candidates): + path = _compact_path(candidate["path"]) + metrics = _candidate_metrics(path, relation, positioned_nodes, accepted_paths) + score = ( + int(metrics["node_obstacle_intersections"]) * 10000 + + float(metrics["shared_segment_overlap"]) * 4000 + + int(metrics["crossing_count"]) * 250 + + int(metrics["bend_count"]) * 0.7 + + float(metrics["path_length"]) + + candidate_index * 1e-5 + ) + scored.append((score, path, candidate, metrics)) + _, path, selected, metrics = min(scored, key=lambda item: item[0]) + accepted_paths.append(path) + routed[original_index] = { + "id": f"semantic_connector_{original_index + 1:02d}", + **relation, + "path_percent": path, + "route_style": selected["route_style"], + "lane_assignment": selected["lane_assignment"], + "routing_metrics": metrics, + } + + connectors = [routed[index] for index in range(len(relations))] + crossing_count = 0 + shared_overlap = 0.0 + for first_index, first in enumerate(connectors): + for second in connectors[first_index + 1:]: + first_segments = _segments(first["path_percent"]) + second_segments = _segments(second["path_percent"]) + crossing_count += sum(1 for a in first_segments for b in second_segments if _segment_crosses(a, b)) + shared_overlap += sum(_shared_segment_overlap(a, b) for a in first_segments for b in second_segments) + obstacle_intersections = sum(int(item["routing_metrics"]["node_obstacle_intersections"]) for item in connectors) + quality = { + "routing_algorithm": "global_candidate_lane_router_v1", + "crossing_count": crossing_count, + "shared_segment_overlap": round(shared_overlap, 6), + "node_obstacle_intersections": obstacle_intersections, + "lane_assignments": {item["id"]: item["lane_assignment"] for item in connectors}, + "passed": obstacle_intersections == 0 and crossing_count == 0 and shared_overlap < 0.01, + } + return connectors, quality + + +def _compile_panel_graph_blueprint( + specification: dict[str, Any], + *, + max_nodes: int, +) -> dict[str, Any] | None: + raw_panels = specification.get("panel_graphs", []) + if not isinstance(raw_panels, list) or len(raw_panels) < 2: + return None + panels: list[dict[str, Any]] = [] + panel_nodes: dict[str, list[dict[str, Any]]] = {} + panel_relations: dict[str, list[dict[str, str]]] = {} + panel_groups: dict[str, list[dict[str, Any]]] = {} + total_nodes = 0 + total_relations = 0 + for panel_index, raw_panel in enumerate(raw_panels): + if not isinstance(raw_panel, dict): + continue + panel_id = str(raw_panel.get("id") or f"panel_{panel_index + 1}").strip() + label = str(raw_panel.get("label") or raw_panel.get("description") or "").strip() + if not panel_id or not label: + continue + nodes: list[dict[str, Any]] = [] + groups: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + for node_index, raw_node in enumerate(raw_panel.get("nodes", []) if isinstance(raw_panel.get("nodes"), list) else []): + if not isinstance(raw_node, dict): + continue + instance_id = str(raw_node.get("instance_id") or raw_node.get("id") or "").strip() + node_label = _label(raw_node) + if not instance_id or not node_label or instance_id in seen_ids: + continue + role = str(raw_node.get("role") or "module").strip() + role_key = role.casefold() + if role_key == "group": + groups.append({ + "id": instance_id, + "label": node_label, + "role": role, + "evidence_ids": [str(value) for value in raw_node.get("evidence_ids", []) if value], + }) + seen_ids.add(instance_id) + continue + field = "inputs" if "input" in role_key or "dataset" in role_key else "outputs" if any(term in role_key for term in ("output", "prediction", "objective", "loss")) else "modules" + nodes.append({ + "id": instance_id, + "semantic_id": str(raw_node.get("semantic_id") or "").strip() or None, + "label": node_label, + "field": field, + "role": role, + "evidence_ids": [str(value) for value in raw_node.get("evidence_ids", []) if value], + "order": node_index, + "panel_id": panel_id, + }) + seen_ids.add(instance_id) + operational_ids = {item["id"] for item in nodes} + relations: list[dict[str, str]] = [] + for raw_relation in raw_panel.get("relations", []) if isinstance(raw_panel.get("relations"), list) else []: + if not isinstance(raw_relation, dict): + continue + source = str(raw_relation.get("source") or "").strip() + target = str(raw_relation.get("target") or "").strip() + if not source or not target or source == target or source not in operational_ids or target not in operational_ids: + continue + relations.append({ + "source": source, + "target": target, + "type": str(raw_relation.get("type") or "data_flow").strip(), + "label": str(raw_relation.get("label") or "").strip(), + }) + panels.append({ + "id": panel_id, + "panel_label": str(raw_panel.get("panel_label") or chr(ord("a") + panel_index)).strip(), + "label": label, + "description": str(raw_panel.get("description") or label).strip(), + "evidence_ids": [str(value) for value in raw_panel.get("evidence_ids", []) if value], + "entity_ids": [item["id"] for item in nodes], + "groups": groups, + "order": panel_index, + }) + panel_nodes[panel_id] = nodes + panel_relations[panel_id] = relations + panel_groups[panel_id] = groups + total_nodes += len(nodes) + total_relations += len(relations) + if len(panels) < 2 or total_nodes < 2 or total_relations < 1: + return None + panel_node_limit = max(24, int(max_nodes)) + if total_nodes > panel_node_limit: + return { + "summary": "Panel-local semantic blueprint skipped because the instance graph is too dense for one raster guide.", + "applied": False, + "topology": "dense_multiframe", + "nodes": [], + "connectors": [], + "reason": "too_many_nodes", + "node_count": total_nodes, + "source": "panel_graphs", + } + + positioned_panels = _layout_dense_panels(panels) + by_panel = {item["id"]: item for item in positioned_panels} + positioned_nodes: list[dict[str, Any]] = [] + connectors: list[dict[str, Any]] = [] + isolated_node_ids: list[str] = [] + connector_index = 1 + panel_route_quality: list[dict[str, Any]] = [] + for panel in positioned_panels: + panel_id = panel["id"] + nodes = panel_nodes.get(panel_id, []) + relations = panel_relations.get(panel_id, []) + ranks = _rank_nodes(nodes, relations) + local_layout = _layout_nodes(nodes, ranks) + box = by_panel[panel_id]["bbox_percent"] + inner_x = box["x"] + box["w"] * 0.035 + inner_y = box["y"] + box["h"] * 0.22 + inner_w = box["w"] * 0.93 + inner_h = box["h"] * 0.72 + local_positioned: list[dict[str, Any]] = [] + for item in local_layout: + local_box = item["bbox_percent"] + local_positioned.append({ + **item, + "panel_id": panel_id, + "global_rank": int(ranks.get(item["id"], item.get("rank", 0))), + "bbox_percent": { + "x": round(inner_x + local_box["x"] * inner_w, 6), + "y": round(inner_y + local_box["y"] * inner_h, 6), + "w": round(local_box["w"] * inner_w, 6), + "h": round(local_box["h"] * inner_h, 6), + }, + }) + positioned_nodes.extend(local_positioned) + connected_ids = {value for relation in relations for value in (relation["source"], relation["target"])} + isolated_node_ids.extend(item["id"] for item in nodes if item["id"] not in connected_ids) + local_connectors, local_quality = _route_connectors(local_positioned, relations) + panel_route_quality.append({"panel_id": panel_id, **local_quality}) + for connector in local_connectors: + connectors.append({ + **connector, + "id": f"semantic_connector_{connector_index:02d}", + "panel_id": panel_id, + }) + connector_index += 1 + route_quality = { + "routing_algorithm": "panel_local_global_candidate_lane_router_v1", + "crossing_count": sum(int(item["crossing_count"]) for item in panel_route_quality), + "shared_segment_overlap": round(sum(float(item["shared_segment_overlap"]) for item in panel_route_quality), 6), + "node_obstacle_intersections": sum(int(item["node_obstacle_intersections"]) for item in panel_route_quality), + "panel_reports": panel_route_quality, + "passed": all(bool(item["passed"]) for item in panel_route_quality), + } + return { + "summary": "Panel-local paper semantic graphs compiled into normalized node and connector geometry.", + "applied": True, + "topology": "dense_multiframe", + "node_count": len(positioned_nodes), + "connector_count": len(connectors), + "isolated_node_ids": isolated_node_ids, + "panels": positioned_panels, + "nodes": positioned_nodes, + "connectors": connectors, + "route_quality": route_quality, + "source": "panel_graphs", + } + + +def compile_semantic_blueprint( + specification: dict[str, Any] | None, + *, + max_nodes: int = 16, +) -> dict[str, Any]: + specification = specification if isinstance(specification, dict) else {} + topology = str(specification.get("topology") or "unknown") + if topology == "dense_multiframe": + panel_graph_blueprint = _compile_panel_graph_blueprint(specification, max_nodes=max_nodes) + if panel_graph_blueprint is not None: + return panel_graph_blueprint + nodes = _visible_entities(specification) + if topology not in _SUPPORTED_TOPOLOGIES: + return {"summary": "Generic semantic blueprint not applicable to this topology.", "applied": False, "topology": topology, "nodes": [], "connectors": [], "reason": "unsupported_topology"} + if len(nodes) < 2: + return {"summary": "Generic semantic blueprint requires at least two visible entities.", "applied": False, "topology": topology, "nodes": [], "connectors": [], "reason": "too_few_nodes"} + if len(nodes) > max(2, int(max_nodes)): + return {"summary": "Generic semantic blueprint skipped because the visible contract is too dense for one raster guide.", "applied": False, "topology": topology, "nodes": [], "connectors": [], "reason": "too_many_nodes", "node_count": len(nodes)} + + node_ids = {item["id"] for item in nodes} + relations = _visible_relations(specification, node_ids) + if not relations: + return {"summary": "Generic semantic blueprint requires at least one declared relation.", "applied": False, "topology": topology, "nodes": [], "connectors": [], "reason": "no_relations"} + + connected_ids = {value for item in relations for value in (item["source"], item["target"])} + isolated_node_ids = [item["id"] for item in nodes if item["id"] not in connected_ids] + connected_nodes = [item for item in nodes if item["id"] in connected_ids] + if len(connected_nodes) >= 2: + nodes = connected_nodes + + global_ranks = _rank_nodes(nodes, relations) + dense_panels = _layout_dense_panels(_dense_overview_panels(specification)) if topology == "dense_multiframe" else [] + if len(dense_panels) >= 2: + positioned = _layout_nodes_in_dense_panels(nodes, relations, dense_panels, global_ranks) + else: + positioned = _layout_nodes(nodes, global_ranks) + connectors, route_quality = _route_connectors(positioned, relations) + return { + "summary": "Paper-grounded semantic graph compiled into normalized node and connector geometry.", + "applied": True, + "topology": topology, + "node_count": len(positioned), + "connector_count": len(connectors), + "isolated_node_ids": isolated_node_ids, + "panels": dense_panels, + "nodes": positioned, + "connectors": connectors, + "route_quality": route_quality, + "source": "figure_specification", + } diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index 316d3d2..933e6a4 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -5,13 +5,84 @@ from pathlib import Path from typing import Any -from PIL import Image, ImageDraw +from PIL import Image, ImageDraw, ImageFont from ..utils import ensure_dir, write_json from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available +from .semantic_blueprint import compile_semantic_blueprint BUILTIN_TEMPLATES: dict[str, dict[str, Any]] = { + "dense-multiframe": { + "summary": "Dense three-panel overview combining task semantics, model internals, and a staged data engine.", + "template_id": "dense-multiframe", + "name": "Task, model, and data-engine overview", + "aspect_ratio": 1.78, + "topology": ["three_macro_panels", "nested_model_flow", "staged_data_engine", "model_to_data_feedback"], + "ideal_module_range": [8, 16], + "visual_density": "very_high", + "panels": [ + {"id": "task", "role": "promptable_task", "bbox_percent": {"x": 0.02, "y": 0.08, "w": 0.27, "h": 0.84}}, + {"id": "model", "role": "model_architecture", "bbox_percent": {"x": 0.34, "y": 0.08, "w": 0.34, "h": 0.84}}, + {"id": "data", "role": "data_engine", "bbox_percent": {"x": 0.73, "y": 0.08, "w": 0.25, "h": 0.84}}, + ], + "connectors": ["image_to_image_encoder", "prompt_to_prompt_encoder", "encoders_to_decoder", "decoder_to_mask", "three_stage_data_engine", "model_to_data_engine_support"], + "style": {"background": "white", "panel_fill": "soft_tint", "stroke": "muted_blue", "accent": ["blue", "green", "orange"], "corners": "rounded", "shadow": "soft_cards"}, + }, + "multimodal": { + "summary": "Multiple modality inputs converge through modality encoders into one joint embedding space and an emergent alignment output.", + "template_id": "multimodal", + "name": "Multimodal convergence into a shared space", + "aspect_ratio": 1.78, + "topology": ["many_inputs", "encoder_bank", "many_to_one", "joint_embedding", "emergent_alignment"], + "ideal_module_range": [2, 5], + "visual_density": "high", + "panels": [ + {"id": "modalities", "role": "stacked_modality_inputs", "bbox_percent": {"x": 0.02, "y": 0.06, "w": 0.25, "h": 0.88}}, + {"id": "encoders", "role": "modality_encoder_bank", "bbox_percent": {"x": 0.34, "y": 0.18, "w": 0.20, "h": 0.64}}, + {"id": "embedding", "role": "joint_embedding_space", "bbox_percent": {"x": 0.61, "y": 0.23, "w": 0.18, "h": 0.54}}, + {"id": "alignment", "role": "emergent_alignment_output", "bbox_percent": {"x": 0.84, "y": 0.28, "w": 0.14, "h": 0.44}}, + ], + "connectors": ["each_modality_to_encoder_bank", "encoder_bank_to_joint_embedding", "joint_embedding_to_emergent_alignment"], + "style": {"background": "white", "panel_fill": "near_white", "stroke": "muted_indigo", "accent": ["blue", "violet", "orange", "teal"], "corners": "rounded", "shadow": "soft_cards"}, + }, + "branch": { + "summary": "Shared-trunk architecture with proposal convergence and parallel output heads.", + "template_id": "branch", + "name": "Shared trunk with parallel heads", + "aspect_ratio": 1.78, + "topology": ["left_to_right", "shared_backbone", "convergence", "three_way_branch"], + "ideal_module_range": [4, 10], + "visual_density": "high", + "panels": [ + {"id": "input", "role": "input", "bbox_percent": {"x": 0.02, "y": 0.28, "w": 0.13, "h": 0.44}}, + {"id": "backbone", "role": "shared_backbone", "bbox_percent": {"x": 0.19, "y": 0.22, "w": 0.18, "h": 0.56}}, + {"id": "proposals", "role": "proposal_path", "bbox_percent": {"x": 0.40, "y": 0.10, "w": 0.16, "h": 0.28}}, + {"id": "alignment", "role": "feature_proposal_convergence", "bbox_percent": {"x": 0.43, "y": 0.48, "w": 0.17, "h": 0.25}}, + {"id": "heads", "role": "parallel_output_heads", "bbox_percent": {"x": 0.68, "y": 0.14, "w": 0.28, "h": 0.72}}, + ], + "connectors": ["input_to_backbone", "backbone_to_proposals", "backbone_features_to_alignment", "proposals_to_alignment", "alignment_three_way_split"], + "style": {"background": "white", "panel_fill": "near_white", "stroke": "muted_blue", "accent": ["blue", "orange", "green"], "corners": "rounded", "shadow": "soft_cards"}, + }, + "feedback": { + "summary": "Compact two-row iterative generation, feedback, and refinement template.", + "template_id": "feedback", + "name": "Feedback refinement loop", + "aspect_ratio": 1.78, + "topology": ["feedback", "iteration", "two_row_loop", "shared_model"], + "ideal_module_range": [4, 9], + "visual_density": "high", + "panels": [ + {"id": "task_input", "role": "input", "bbox_percent": {"x": 0.02, "y": 0.12, "w": 0.12, "h": 0.24}}, + {"id": "generation", "role": "initial_generation", "bbox_percent": {"x": 0.18, "y": 0.10, "w": 0.20, "h": 0.28}}, + {"id": "initial_output", "role": "initial_output", "bbox_percent": {"x": 0.42, "y": 0.10, "w": 0.18, "h": 0.28}}, + {"id": "feedback", "role": "self_feedback", "bbox_percent": {"x": 0.70, "y": 0.10, "w": 0.22, "h": 0.36}}, + {"id": "refinement", "role": "refinement", "bbox_percent": {"x": 0.42, "y": 0.62, "w": 0.20, "h": 0.25}}, + {"id": "refined_output", "role": "refined_output", "bbox_percent": {"x": 0.68, "y": 0.62, "w": 0.20, "h": 0.25}}, + ], + "connectors": ["input_to_generation", "generation_to_initial_output", "initial_output_to_feedback", "initial_output_to_refinement", "feedback_to_refinement", "refinement_to_refined_output", "refined_output_to_feedback_loop"], + "style": {"background": "white", "panel_fill": "near_white", "stroke": "muted_blue", "accent": ["navy", "teal", "orange"], "corners": "rounded", "shadow": "soft_cards"}, + }, "arbor": { "summary": "Tree-centered iterative research template.", "template_id": "arbor", @@ -169,20 +240,24 @@ def build_template_profiles(reference_paths: list[str], out_dir: str | Path, mod def _paper_features(review: dict) -> dict: - statements = " ".join(str(item.get("statement", "")) for field in ["research_questions", "central_claims", "modules", "relations", "innovations"] for item in review.get(field, [])).lower() + statements = " ".join(str(item.get("statement", "")) for field in ["research_questions", "central_claims", "inputs", "modules", "relations", "innovations"] for item in review.get(field, [])).lower() relation_types = {str(item.get("relation_type") or item.get("type") or "").lower() for item in review.get("relations", [])} workflow = review.get("workflows", {}) + input_labels = " ".join(str(item.get("visible_label") or item.get("statement") or "").lower() for item in review.get("inputs", [])) + modality_count = sum(term in input_labels for term in ("image", "text", "audio", "video", "depth", "thermal", "imu")) return { "module_count": len(review.get("modules", [])), "input_count": len(review.get("inputs", [])), "has_loop": bool(workflow.get("feedback")) or "feedback" in relation_types or any(token in statements for token in ["iterative", "loop", "tree search", "backpropagate"]), - "has_tree": any(token in statements for token in ["tree", "branch", "prune", "search"]), - "has_multimodal": any(token in statements for token in ["multimodal", "multi-modal", "image", "video", "audio", "document"]), + "has_tree": any(token in statements for token in ["tree", "prune", "tree search", "search tree"]), + "has_branch": any(token in statements for token in ["branch", "parallel head", "parallel output", "multi-head", "multihead"]), + "modality_count": modality_count, + "has_multimodal": modality_count >= 3 or any(token in statements for token in ["multimodal", "multi-modal", "cross-modal", "cross modal"]), "has_retrieval": any(token in statements for token in ["retrieval", "retrieve", "index", "rag", "knowledge graph"]), } -def select_template(profiles: list[dict], review: dict, requested: str = "auto", target_ratio: str = "auto") -> dict: +def select_template(profiles: list[dict], review: dict, requested: str = "auto", target_ratio: str = "auto", contract_topology: str | None = None) -> dict: if requested != "auto": matches = [profile for profile in profiles if profile.get("template_id") == requested or profile.get("profile_id") == requested] if not matches: @@ -193,6 +268,23 @@ def select_template(profiles: list[dict], review: dict, requested: str = "auto", selected = dict(matches[0]) selected["selection"] = {"mode": "explicit", "score": 1.0, "reasons": ["explicit CLI selection"]} return selected + topology_template = { + "dense_multiframe": "dense-multiframe", + "branch": "branch", + "feedback": "feedback", + "multimodal": "multimodal", + }.get(str(contract_topology or "").casefold()) + if topology_template: + match = next((profile for profile in profiles if profile.get("template_id") == topology_template), None) + if match: + selected = dict(match) + selected["selection"] = { + "mode": "contract_topology", + "score": 1.0, + "reasons": [f"normalized scientific contract topology={contract_topology}"], + "paper_features": _paper_features(review), + } + return selected features = _paper_features(review) requested_ratio = None if target_ratio != "auto": @@ -208,9 +300,15 @@ def select_template(profiles: list[dict], review: dict, requested: str = "auto", module_fit = 1.0 if low <= features["module_count"] <= high else max(0.0, 1.0 - min(abs(features["module_count"] - low), abs(features["module_count"] - high)) / 10) topology = 0.0 reasons = [f"module_fit={module_fit:.2f}"] - if template_id == "arbor" and (features["has_loop"] or features["has_tree"]): - topology += 1.0; reasons.append("loop/tree topology") - if template_id == "linear" and not features["has_loop"] and 4 <= features["module_count"] <= 8: + if template_id == "feedback" and features["has_loop"] and not features["has_tree"]: + topology += 1.0; reasons.append("explicit feedback-loop topology") + if template_id == "branch" and features["has_branch"] and not features["has_loop"] and not features["has_tree"] and not features["has_multimodal"] and not features["has_retrieval"]: + topology += 1.0; reasons.append("shared-trunk parallel-branch topology") + if template_id == "multimodal" and features["has_multimodal"] and features["modality_count"] >= 3 and features["module_count"] < 8: + topology += 1.1; reasons.append("multiple modalities converge into a shared representation") + if template_id == "arbor" and features["has_tree"]: + topology += 1.0; reasons.append("tree/branch topology") + if template_id == "linear" and not features["has_loop"] and not features["has_tree"] and not features["has_branch"] and not features["has_retrieval"] and 2 <= features["module_count"] <= 8: topology += 0.9; reasons.append("sequential stage count") if template_id == "tripanel" and features["has_retrieval"]: topology += 0.9; reasons.append("index/retrieval topology") @@ -244,12 +342,117 @@ def _canvas_size(profile: dict, target_ratio: str) -> tuple[int, int, float]: return width, height, ratio -def render_layout_blueprint(profile: dict, out_path: str | Path, target_ratio: str = "auto") -> dict: +def render_layout_blueprint( + profile: dict, + out_path: str | Path, + target_ratio: str = "auto", + figure_specification: dict[str, Any] | None = None, +) -> dict: width, height, ratio = _canvas_size(profile, target_ratio) image = Image.new("RGB", (width, height), (250, 250, 249)) draw = ImageDraw.Draw(image) panels = profile.get("panels", []) + + def normalized(value: Any) -> str: + return "".join(char for char in str(value or "").casefold() if char.isalnum()) + + exact_labels: dict[str, str] = {} + if figure_specification: + for field in ("inputs", "modules", "outputs", "innovations"): + for item in figure_specification.get(field, []) if isinstance(figure_specification.get(field), list) else []: + if not isinstance(item, dict): + continue + label = str(item.get("name") or item.get("label") or item.get("title") or "").strip() + if label: + exact_labels[normalized(label)] = label + for label in figure_specification.get("required_labels", []) if isinstance(figure_specification.get("required_labels"), list) else []: + if str(label).strip(): + exact_labels.setdefault(normalized(label), str(label).strip()) + + def exact(canonical: str) -> str: + return exact_labels.get(normalized(canonical), canonical) + + feedback_required = { + normalized(value) + for value in ("input x", "Generate", "Initial Output", "FEEDBACK", "Self-Feedback", "REFINE", "Refined output", "Model M") + } + semantic_feedback = profile.get("template_id") == "feedback" and feedback_required.issubset(exact_labels) + dense_required = { + normalized(value) + for value in ( + "Promptable Segmentation Task", "Segment Anything Model", "Data Engine", "Image", "Prompt", + "Image Encoder", "Prompt Encoder", "Mask Decoder", "Valid Segmentation Mask", + "Assisted-manual", "Semi-automatic", "Fully Automatic", "SA-1B", + ) + } + semantic_dense = profile.get("template_id") == "dense-multiframe" and dense_required.issubset(exact_labels) + generic_semantic_plan = compile_semantic_blueprint(figure_specification) + semantic_generic = bool(generic_semantic_plan.get("applied")) and not semantic_feedback and not semantic_dense + semantic_blueprint = semantic_feedback or semantic_dense or semantic_generic + if semantic_generic: + panels = [] + + def font(size: int, bold: bool = False): + candidates = ["arialbd.ttf", "DejaVuSans-Bold.ttf"] if bold else ["arial.ttf", "DejaVuSans.ttf"] + for candidate in candidates: + try: + return ImageFont.truetype(candidate, max(12, size)) + except OSError: + continue + return ImageFont.load_default() + + def centered_text(box: tuple[int, int, int, int], value: str, size: int, fill=(42, 72, 91), bold: bool = True) -> None: + fitted_size = max(12, size) + selected_font = font(fitted_size, bold=bold) + bounds = draw.textbbox((0, 0), value, font=selected_font) + available_width = max(8, box[2] - box[0] - 12) + while bounds[2] - bounds[0] > available_width and fitted_size > 12: + fitted_size -= 1 + selected_font = font(fitted_size, bold=bold) + bounds = draw.textbbox((0, 0), value, font=selected_font) + text_width = bounds[2] - bounds[0] + text_height = bounds[3] - bounds[1] + x = box[0] + ((box[2] - box[0]) - text_width) / 2 + y = box[1] + ((box[3] - box[1]) - text_height) / 2 - bounds[1] + draw.text((x, y), value, font=selected_font, fill=fill) + + def centered_wrapped_text(box: tuple[int, int, int, int], value: str, size: int, fill=(42, 72, 91), bold: bool = True) -> None: + max_width = max(20, box[2] - box[0] - 14) + max_height = max(20, box[3] - box[1] - 10) + tokens = value.split() if " " in value.strip() else list(value.strip()) + selected_font = font(size, bold=bold) + selected_lines = [value] + for fitted_size in range(max(10, size), 9, -1): + selected_font = font(fitted_size, bold=bold) + lines: list[str] = [] + current = "" + separator = " " if " " in value.strip() else "" + for token in tokens: + candidate = token if not current else f"{current}{separator}{token}" + if draw.textlength(candidate, font=selected_font) <= max_width or not current: + current = candidate + else: + lines.append(current) + current = token + if current: + lines.append(current) + if len(lines) > 3: + continue + text_value = "\n".join(lines) + bounds = draw.multiline_textbbox((0, 0), text_value, font=selected_font, spacing=3, align="center") + if bounds[2] - bounds[0] <= max_width and bounds[3] - bounds[1] <= max_height: + selected_lines = lines + break + text_value = "\n".join(selected_lines) + bounds = draw.multiline_textbbox((0, 0), text_value, font=selected_font, spacing=3, align="center") + text_width = bounds[2] - bounds[0] + text_height = bounds[3] - bounds[1] + x = box[0] + ((box[2] - box[0]) - text_width) / 2 + y = box[1] + ((box[3] - box[1]) - text_height) / 2 - bounds[1] + draw.multiline_text((x, y), text_value, font=selected_font, fill=fill, spacing=3, align="center") + centers = [] + panel_boxes = [] for index, panel in enumerate(panels): bbox = panel["bbox_percent"] x0, y0 = int(bbox["x"] * width), int(bbox["y"] * height) @@ -268,22 +471,254 @@ def render_layout_blueprint(profile: dict, out_path: str | Path, target_ratio: s else: draw.rounded_rectangle((x0, y0, x1, y1), radius=max(10, round(height * 0.025)), fill=fill, outline=(104, 132, 151), width=3) centers.append(((x0 + x1) // 2, (y0 + y1) // 2)) - slots = max(2, min(6, round((x1 - x0) / max(width * 0.12, 1)))) - for slot_index in range(slots): - sx0 = x0 + int((slot_index + 0.2) / slots * (x1 - x0)) - sx1 = x0 + int((slot_index + 0.8) / slots * (x1 - x0)) - sy0 = y0 + int(0.28 * (y1 - y0)) - sy1 = y0 + int(0.70 * (y1 - y0)) - draw.rounded_rectangle((sx0, sy0, sx1, sy1), radius=8, outline=(178, 190, 198), width=2) - for index in range(len(centers) - 1): - start = centers[index] - end = centers[index + 1] - draw.line((start, end), fill=(80, 111, 134), width=4) - angle = math.atan2(end[1] - start[1], end[0] - start[0]) + panel_boxes.append((x0, y0, x1, y1)) + if not semantic_blueprint: + slots = max(2, min(6, round((x1 - x0) / max(width * 0.12, 1)))) + for slot_index in range(slots): + sx0 = x0 + int((slot_index + 0.2) / slots * (x1 - x0)) + sx1 = x0 + int((slot_index + 0.8) / slots * (x1 - x0)) + sy0 = y0 + int(0.28 * (y1 - y0)) + sy1 = y0 + int(0.70 * (y1 - y0)) + draw.rounded_rectangle((sx0, sy0, sx1, sy1), radius=8, outline=(178, 190, 198), width=2) + def draw_arrow_path(points: list[tuple[int, int]], color: tuple[int, int, int] = (80, 111, 134)) -> None: + draw.line(points, fill=color, width=4, joint="curve") + if len(points) < 2: + return + tail, end = points[-2], points[-1] + angle = math.atan2(end[1] - tail[1], end[0] - tail[0]) arrow = 12 points = [end, (end[0] - arrow * math.cos(angle - 0.45), end[1] - arrow * math.sin(angle - 0.45)), (end[0] - arrow * math.cos(angle + 0.45), end[1] - arrow * math.sin(angle + 0.45))] - draw.polygon(points, fill=(80, 111, 134)) + draw.polygon(points, fill=color) + + def region(box, x0: float, y0: float, x1: float, y1: float) -> tuple[int, int, int, int]: + return ( + round(box[0] + x0 * (box[2] - box[0])), + round(box[1] + y0 * (box[3] - box[1])), + round(box[0] + x1 * (box[2] - box[0])), + round(box[1] + y1 * (box[3] - box[1])), + ) + + def draw_dashed_arrow(start: tuple[int, int], end: tuple[int, int], color: tuple[int, int, int] = (177, 92, 33)) -> None: + length = max(1, end[0] - start[0]) + dash = max(10, round(width * 0.008)) + gap = max(7, round(width * 0.005)) + cursor = 0 + while cursor < length - 16: + draw.line((start[0] + cursor, start[1], min(end[0] - 16, start[0] + cursor + dash), end[1]), fill=color, width=4) + cursor += dash + gap + draw_arrow_path([(end[0] - 18, end[1]), end], color) + + if profile.get("template_id") == "feedback" and len(panel_boxes) >= 6: + def left(box): return (box[0], (box[1] + box[3]) // 2) + def right(box): return (box[2], (box[1] + box[3]) // 2) + def top_at(box, fraction): return (round(box[0] + fraction * (box[2] - box[0])), box[1]) + def bottom_at(box, fraction): return (round(box[0] + fraction * (box[2] - box[0])), box[3]) + input_box, generation_box, initial_box, feedback_box, refinement_box, refined_box = panel_boxes[:6] + draw_arrow_path([right(input_box), left(generation_box)]) + draw_arrow_path([right(generation_box), left(initial_box)]) + draw_arrow_path([right(initial_box), left(feedback_box)]) + initial_start, initial_end = bottom_at(initial_box, 0.45), top_at(refinement_box, 0.35) + initial_mid_y = round((initial_start[1] + initial_end[1]) / 2) + draw_arrow_path([initial_start, (initial_start[0], initial_mid_y), (initial_end[0], initial_mid_y), initial_end], (31, 117, 126)) + feedback_source_box = region(feedback_box, 0.13, 0.60, 0.87, 0.88) if semantic_feedback else feedback_box + feedback_start, feedback_end = bottom_at(feedback_source_box, 0.50), top_at(refinement_box, 0.72) + feedback_mid_y = round((feedback_start[1] + feedback_end[1]) / 2) + draw_arrow_path([feedback_start, (feedback_start[0], feedback_mid_y), (feedback_end[0], feedback_mid_y), feedback_end], (31, 117, 126)) + draw_arrow_path([right(refinement_box), left(refined_box)], (31, 117, 126)) + loop_start = right(refined_box) + loop_end = (feedback_box[2], round(feedback_box[1] + 0.70 * (feedback_box[3] - feedback_box[1]))) + loop_x = min(width - 18, max(loop_start[0], loop_end[0]) + round(width * 0.045)) + draw_arrow_path([loop_start, (loop_x, loop_start[1]), (loop_x, loop_end[1]), loop_end], (31, 117, 126)) + if semantic_feedback: + input_box, generation_box, initial_box, feedback_box, refinement_box, refined_box = panel_boxes[:6] + title_size = max(16, round(height * 0.025)) + badge_size = max(14, round(height * 0.020)) + centered_text(region(input_box, 0.05, 0.30, 0.95, 0.70), exact("input x"), title_size) + centered_text(region(generation_box, 0.05, 0.06, 0.95, 0.35), exact("Generate"), title_size) + centered_text(region(initial_box, 0.04, 0.20, 0.96, 0.52), exact("Initial Output"), title_size) + centered_text(region(feedback_box, 0.05, 0.04, 0.95, 0.25), exact("FEEDBACK"), title_size, fill=(177, 92, 33)) + centered_text(region(refinement_box, 0.05, 0.06, 0.95, 0.34), exact("REFINE"), title_size) + centered_text(region(refined_box, 0.04, 0.20, 0.96, 0.52), exact("Refined output"), title_size) + + for badge_box in ( + region(generation_box, 0.24, 0.48, 0.76, 0.76), + region(feedback_box, 0.30, 0.27, 0.70, 0.47), + region(refinement_box, 0.24, 0.48, 0.76, 0.76), + ): + draw.rounded_rectangle(badge_box, radius=10, fill=(235, 243, 248), outline=(126, 157, 177), width=2) + centered_text(badge_box, exact("Model M"), badge_size, bold=False) + + self_feedback_box = feedback_source_box + draw.rounded_rectangle(self_feedback_box, radius=10, fill=(238, 249, 248), outline=(31, 117, 126), width=2) + centered_text(self_feedback_box, exact("Self-Feedback"), badge_size) + internal_start = (round((feedback_box[0] + feedback_box[2]) / 2), region(feedback_box, 0, 0.47, 1, 0.47)[1]) + internal_end = (round((self_feedback_box[0] + self_feedback_box[2]) / 2), self_feedback_box[1]) + draw_arrow_path([internal_start, internal_end], (31, 117, 126)) + + iterate_box = (loop_x - round(width * 0.08), round((loop_start[1] + loop_end[1]) / 2) - 22, loop_x - 4, round((loop_start[1] + loop_end[1]) / 2) + 22) + centered_text(iterate_box, exact("iterate"), badge_size, fill=(31, 117, 126), bold=False) + elif semantic_dense and len(panel_boxes) >= 3: + task_box, model_box, data_box = panel_boxes[:3] + title_size = max(15, round(height * 0.022)) + node_size = max(13, round(height * 0.017)) + + image_box = region(task_box, 0.12, 0.23, 0.88, 0.37) + prompt_box = region(task_box, 0.12, 0.57, 0.88, 0.71) + image_encoder_box = region(model_box, 0.06, 0.23, 0.43, 0.37) + prompt_encoder_box = region(model_box, 0.06, 0.57, 0.43, 0.71) + mask_decoder_box = region(model_box, 0.56, 0.38, 0.94, 0.54) + valid_mask_box = region(model_box, 0.56, 0.69, 0.94, 0.84) + assisted_box = region(data_box, 0.12, 0.22, 0.88, 0.34) + semi_box = region(data_box, 0.12, 0.41, 0.88, 0.53) + fully_box = region(data_box, 0.12, 0.60, 0.88, 0.72) + sa1b_box = region(data_box, 0.22, 0.80, 0.78, 0.91) + + def left(box): return (box[0], (box[1] + box[3]) // 2) + def right(box): return (box[2], (box[1] + box[3]) // 2) + def top(box): return ((box[0] + box[2]) // 2, box[1]) + def bottom(box): return ((box[0] + box[2]) // 2, box[3]) + + draw_arrow_path([right(image_box), left(image_encoder_box)], (42, 99, 161)) + draw_arrow_path([right(prompt_box), left(prompt_encoder_box)], (31, 117, 126)) + decoder_upper = (mask_decoder_box[0], round(mask_decoder_box[1] + 0.32 * (mask_decoder_box[3] - mask_decoder_box[1]))) + decoder_lower = (mask_decoder_box[0], round(mask_decoder_box[1] + 0.70 * (mask_decoder_box[3] - mask_decoder_box[1]))) + draw_arrow_path([right(image_encoder_box), decoder_upper], (42, 99, 161)) + draw_arrow_path([right(prompt_encoder_box), decoder_lower], (31, 117, 126)) + draw_arrow_path([bottom(mask_decoder_box), top(valid_mask_box)], (80, 111, 134)) + draw_arrow_path([bottom(assisted_box), top(semi_box)], (177, 92, 33)) + draw_arrow_path([bottom(semi_box), top(fully_box)], (177, 92, 33)) + draw_arrow_path([bottom(fully_box), top(sa1b_box)], (177, 92, 33)) + support_y = round(model_box[1] + 0.17 * (model_box[3] - model_box[1])) + draw_dashed_arrow((model_box[2], support_y), (data_box[0], support_y), (177, 92, 33)) + + node_specs = [ + (image_box, "Image", (237, 245, 252), (42, 99, 161)), + (prompt_box, "Prompt", (237, 249, 247), (31, 117, 126)), + (image_encoder_box, "Image Encoder", (237, 245, 252), (42, 99, 161)), + (prompt_encoder_box, "Prompt Encoder", (237, 249, 247), (31, 117, 126)), + (mask_decoder_box, "Mask Decoder", (244, 242, 251), (98, 84, 147)), + (valid_mask_box, "Valid Segmentation Mask", (242, 248, 243), (66, 126, 80)), + (assisted_box, "Assisted-manual", (252, 244, 235), (177, 92, 33)), + (semi_box, "Semi-automatic", (252, 244, 235), (177, 92, 33)), + (fully_box, "Fully Automatic", (252, 244, 235), (177, 92, 33)), + (sa1b_box, "SA-1B", (247, 239, 229), (150, 76, 30)), + ] + for node_box, label, fill, outline in node_specs: + draw.rounded_rectangle(node_box, radius=10, fill=fill, outline=outline, width=2) + centered_text(node_box, exact(label), node_size, fill=outline) + + centered_text(region(task_box, 0.03, 0.03, 0.97, 0.15), exact("Promptable Segmentation Task"), title_size) + centered_text(region(model_box, 0.03, 0.03, 0.97, 0.15), exact("Segment Anything Model"), title_size) + centered_text(region(data_box, 0.03, 0.03, 0.97, 0.15), exact("Data Engine"), title_size, fill=(177, 92, 33)) + elif semantic_generic: + for panel in generic_semantic_plan.get("panels", []): + bbox = panel.get("bbox_percent", {}) + panel_box = ( + round(float(bbox.get("x", 0)) * width), + round(float(bbox.get("y", 0)) * height), + round(float(bbox.get("x", 0) + bbox.get("w", 0)) * width), + round(float(bbox.get("y", 0) + bbox.get("h", 0)) * height), + ) + draw.rounded_rectangle( + panel_box, + radius=max(10, round(height * 0.018)), + fill=(247, 249, 249), + outline=(104, 132, 151), + width=3, + ) + title_box = region(panel_box, 0.025, 0.025, 0.975, 0.20) + panel_marker = str(panel.get("panel_label") or "").strip() + panel_title = str(panel.get("label") or "").strip() + visible_title = f"{panel_marker} {panel_title}" if panel_marker else panel_title + centered_wrapped_text(title_box, visible_title, max(13, round(height * 0.020)), fill=(42, 72, 91)) + groups = [item for item in panel.get("groups", []) if isinstance(item, dict) and str(item.get("label") or "").strip()] + for group_index, group in enumerate(groups[:2]): + group_box = region(panel_box, 0.18, 0.165 + group_index * 0.055, 0.82, 0.215 + group_index * 0.055) + draw.rounded_rectangle(group_box, radius=max(5, round(height * 0.008)), fill=(244, 242, 251), outline=(98, 84, 147), width=2) + centered_wrapped_text(group_box, str(group.get("label") or ""), max(11, round(height * 0.014)), fill=(98, 84, 147), bold=False) + node_boxes: dict[str, tuple[int, int, int, int]] = {} + node_styles: dict[str, tuple[tuple[int, int, int], tuple[int, int, int]]] = {} + for node in generic_semantic_plan.get("nodes", []): + bbox = node["bbox_percent"] + node_box = ( + round(bbox["x"] * width), + round(bbox["y"] * height), + round((bbox["x"] + bbox["w"]) * width), + round((bbox["y"] + bbox["h"]) * height), + ) + field = str(node.get("field") or "modules") + role = str(node.get("role") or "").casefold() + if "data_source" in role or role in {"dataset", "source dataset"}: + fill, outline = (248, 244, 236), (137, 101, 56) + elif field == "inputs": + fill, outline = (237, 245, 252), (42, 99, 161) + elif field == "outputs": + fill, outline = (242, 248, 243), (66, 126, 80) + elif field == "innovations": + fill, outline = (252, 244, 235), (177, 92, 33) + elif "shared" in role or "joint" in role or "fusion" in role: + fill, outline = (244, 242, 251), (98, 84, 147) + else: + fill, outline = (237, 249, 247), (31, 117, 126) + node_boxes[node["id"]] = node_box + node_styles[node["id"]] = (fill, outline) + draw.rounded_rectangle(node_box, radius=max(8, round(height * 0.014)), fill=fill, outline=outline, width=3) + + relation_colors = { + "feedback_loop": (177, 92, 33), + "branch": (98, 84, 147), + "conditioning": (31, 117, 126), + "feature_flow": (42, 99, 161), + } + for connector in generic_semantic_plan.get("connectors", []): + points = [(round(point[0] * width), round(point[1] * height)) for point in connector.get("path_percent", [])] + color = relation_colors.get(str(connector.get("type") or ""), (80, 111, 134)) + draw_arrow_path(points, color) + label = str(connector.get("label") or "").strip() + if label and len(points) >= 2: + middle = points[len(points) // 2] + label_box = (middle[0] - 58, middle[1] - 48, middle[0] + 58, middle[1] - 18) + draw.rounded_rectangle(label_box, radius=7, fill=(250, 250, 249), outline=color, width=1) + centered_wrapped_text(label_box, label, max(11, round(height * 0.014)), fill=color, bold=False) + + for node in generic_semantic_plan.get("nodes", []): + node_box = node_boxes[node["id"]] + _, outline = node_styles[node["id"]] + centered_wrapped_text(node_box, str(node.get("label") or node["id"]), max(13, round(height * 0.019)), fill=outline) + else: + for index in range(len(centers) - 1): + draw_arrow_path([centers[index], centers[index + 1]]) path = Path(out_path) path.parent.mkdir(parents=True, exist_ok=True) image.save(path) - return {"summary": "Content-free layout blueprint rendered from selected template.", "path": str(path), "width": width, "height": height, "aspect_ratio_decimal": round(ratio, 6), "template_id": profile["template_id"], "contains_reference_text": False, "contains_reference_objects": False} + semantic_label_order = [] + if semantic_feedback: + semantic_label_order = ["input x", "Generate", "Initial Output", "FEEDBACK", "Self-Feedback", "REFINE", "Refined output", "Model M", "iterate"] + elif semantic_dense: + semantic_label_order = [ + "Promptable Segmentation Task", "Segment Anything Model", "Data Engine", "Image", "Prompt", + "Image Encoder", "Prompt Encoder", "Mask Decoder", "Valid Segmentation Mask", + "Assisted-manual", "Semi-automatic", "Fully Automatic", "SA-1B", + ] + elif semantic_generic: + semantic_label_order = [str(item.get("label") or "") for item in generic_semantic_plan.get("panels", [])] + semantic_label_order.extend( + str(group.get("label") or "") + for panel in generic_semantic_plan.get("panels", []) if isinstance(panel, dict) + for group in panel.get("groups", []) if isinstance(group, dict) + ) + semantic_label_order.extend(str(item.get("label") or "") for item in generic_semantic_plan.get("nodes", [])) + semantic_label_order.extend(str(item.get("label") or "") for item in generic_semantic_plan.get("connectors", []) if str(item.get("label") or "").strip()) + return { + "summary": "Paper-semantic layout blueprint rendered from the normalized contract." if semantic_blueprint else "Content-free layout blueprint rendered from selected template.", + "path": str(path), + "width": width, + "height": height, + "aspect_ratio_decimal": round(ratio, 6), + "template_id": profile["template_id"], + "contains_reference_text": False, + "contains_reference_objects": False, + "contains_paper_semantic_labels": semantic_blueprint, + "semantic_labels": [exact(value) for value in semantic_label_order], + "semantic_plan": generic_semantic_plan if semantic_generic else None, + } diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 35c9a4c..cf01473 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -1,41 +1,19 @@ from __future__ import annotations -import json -import shutil +import os import time from pathlib import Path from typing import Any -from ..utils import ensure_dir, write_json, write_text -from .analyzer import parse_paper -from .generator import generate_and_select -from .planner import compile_image_prompt, merge_preferences, plan_paper_image, validate_plan_grounding -from .review import build_paper_review, detect_domain_profile, validate_review_coverage +from ..utils import read_json, write_json, write_text +from .generator import generate_and_select, native_image2_aspect_ratio +from .overlay_renderer import render_visual_substrate_blueprint +from .planner import compile_image_prompt +from .preparation import prepare_paper_figure_contract +from .review import validate_review_coverage from .templates import build_template_profiles, render_layout_blueprint, select_template -def _load_preferences(path: str | Path | None) -> dict: - if not path: - return {} - source = Path(path).resolve() - if not source.exists(): - raise FileNotFoundError(f"Preferences file does not exist: {source}") - raw = json.loads(source.read_text(encoding="utf-8")) - if not isinstance(raw, dict): - raise ValueError("Preferences file must contain a JSON object") - return raw - - -def _archive_file(source: str | Path, target_dir: Path, target_name: str | None = None) -> str: - path = Path(source).resolve() - if not path.exists(): - raise FileNotFoundError(f"Input file does not exist: {path}") - target_dir.mkdir(parents=True, exist_ok=True) - target = target_dir / (target_name or path.name) - shutil.copyfile(path, target) - return str(target) - - def run_paper_to_image( paper: str | Path, out: str | Path, @@ -49,101 +27,129 @@ def run_paper_to_image( aspect_ratio: str | None = None, language: str | None = None, image_model: str | None = None, - image_retries: int = 2, + image_retries: int = 1, review_mode: str = "heuristic", review_model: str | None = None, domain_profile: str = "auto", template: str = "auto", - repair_rounds: int = 1, + repair_rounds: int = 3, + repair_source: str | Path | None = None, ocr_engine: str = "auto", ocr_lang: str = "en_ch", ocr_adapter=None, critic_adapter=None, + topology_adapter=None, + resume_candidates: bool = False, + require_visual_enrichment: bool = True, + deterministic_overlay: bool = True, + editable_overlay_ppt: bool = False, ) -> dict: started = time.time() - root = ensure_dir(out).resolve() - inputs = ensure_dir(root / "inputs") - positive_dir = ensure_dir(inputs / "positive_references") - negative_dir = ensure_dir(inputs / "negative_references") - - archived_paper = _archive_file(paper, inputs, f"paper{Path(paper).suffix.lower()}") - archived_positive = [_archive_file(path, positive_dir) for path in (positive_references or [])] - archived_negative = [_archive_file(path, negative_dir) for path in (negative_references or [])] - raw_preferences = _load_preferences(preferences_path) - preferences = merge_preferences(raw_preferences, aspect_ratio=aspect_ratio, language=language) - preferences["positive_references"] = archived_positive - preferences["negative_references"] = archived_negative - write_json(root / "preferences.json", preferences) - input_manifest = { - "summary": "Archived inputs for the paper-to-image run.", - "paper_original": str(Path(paper).resolve()), - "paper_archived": archived_paper, - "preferences_original": str(Path(preferences_path).resolve()) if preferences_path else None, - "positive_references": archived_positive, - "negative_references": archived_negative, - } - write_json(root / "input_manifest.json", input_manifest) - - parsed = parse_paper(archived_paper) - evidence_map = { - "summary": "Page-aware evidence map used by paper summary and figure planning.", - "source_path": parsed["source_path"], - "page_count": parsed["page_count"], - "char_count": parsed["char_count"], - "headings": parsed["headings"], - "evidence": parsed["evidence"], - } - write_json(root / "evidence_map.json", evidence_map) - write_json(root / "document_index.json", parsed["document_index"]) - - selected_domain = detect_domain_profile(parsed, explicit=domain_profile) - write_json(root / "domain_profile.json", selected_domain) - paper_review, review_metadata = build_paper_review(parsed, selected_domain, mode=planner_mode, model=planner_model) - write_text(root / "prompts" / "paper_review_prompt.txt", review_metadata.pop("prompt")) - write_json(root / "paper_review_metadata.json", review_metadata) - write_json(root / "paper_review.json", paper_review) + effective_planner_model = planner_model or os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" + prepared = prepare_paper_figure_contract( + paper=paper, + out=out, + deadline_seconds=180, + planner_mode=planner_mode, + planner_model=effective_planner_model, + ocr_engine=ocr_engine if ocr_engine in {"auto", "paddle", "easyocr", "off"} else "auto", + ocr_lang=ocr_lang, + preferences_path=preferences_path, + positive_references=positive_references, + negative_references=negative_references, + aspect_ratio=aspect_ratio, + language=language, + domain_profile=domain_profile, + ocr_adapter=ocr_adapter, + fast_mode=True, + ) + if not prepared.get("ok"): + raise ValueError(f"Paper contract preparation failed: {prepared.get('errors') or prepared.get('planning_validation', {}).get('errors')}") + root = prepared["root"] + archived_paper = prepared["paper"] + archived_positive = prepared["archived_positive"] + archived_negative = prepared["archived_negative"] + preferences = prepared["preferences"] + parsed = prepared["parsed"] + selected_domain = prepared["selected_domain"] + paper_review = prepared["paper_review"] + review_metadata = prepared["review_metadata"] + plan = prepared["plan"] + planner_metadata = prepared["planner_metadata"] + planning_validation = prepared["planning_validation"] production_mode = asset_mode == "image2" - coverage = validate_review_coverage(paper_review, parsed, selected_domain, strict=production_mode) + if production_mode and not parsed.get("extraction_report", {}).get("scientific_scope_complete", True): + raise RuntimeError("Production Image2 generation requires full-document scientific scope; sampled scanned-paper contracts are engineering-only") + review_is_fast_vlm_derivative = review_metadata.get("mode") == "derived_from_vlm_plan" + coverage = validate_review_coverage(paper_review, parsed, selected_domain, strict=production_mode and not review_is_fast_vlm_derivative) write_json(root / "review_coverage_report.json", coverage) - if production_mode and review_metadata.get("mode") != "vlm": + semantic_vlm_ready = bool( + review_metadata.get("mode") in {"vlm", "derived_from_vlm_plan"} + or planner_metadata.get("mode") in {"vlm", "review_grounded_vlm"} + ) + if production_mode and not semantic_vlm_ready: raise RuntimeError("Production Image2 generation requires successful VLM paper review") if not coverage["ok"]: raise ValueError(f"Paper review failed coverage validation: {coverage['errors']}") + if production_mode: + requested_ratio = str(preferences.get("aspect_ratio") or "auto") + if requested_ratio != "auto": + native_ratio = native_image2_aspect_ratio(requested_ratio) + if native_ratio != requested_ratio: + preferences["requested_aspect_ratio"] = requested_ratio + preferences["aspect_ratio"] = native_ratio + preferences["generation_ratio_policy"] = "nearest_native_image2_canvas; preserve_internal_geometry; no_semantic_crop" + write_json(root / "preferences.json", preferences) + template_profiles = build_template_profiles(archived_positive, root / "template_profiles", mode=planner_mode, model=planner_model) - selected_template = select_template(template_profiles, paper_review, requested=template, target_ratio=str(preferences.get("aspect_ratio") or "auto")) + selected_template = select_template( + template_profiles, + paper_review, + requested=template, + target_ratio=str(preferences.get("aspect_ratio") or "auto"), + contract_topology=str(plan.get("figure_specification", {}).get("topology") or "unknown"), + ) if str(preferences.get("aspect_ratio") or "auto") == "auto": ratio = float(selected_template.get("source_aspect_ratio") or selected_template.get("aspect_ratio") or 16 / 9) - preferences["aspect_ratio"] = f"{ratio:.3f}:1.000" + preferences["template_source_aspect_ratio"] = round(ratio, 6) + preferences["aspect_ratio"] = "3:2" if ratio >= 1.25 else "2:3" if ratio <= 0.80 else "1:1" + preferences["generation_ratio_policy"] = "nearest_native_image2_canvas; preserve_template_internal_geometry; no_semantic_crop" write_json(root / "preferences.json", preferences) write_json(root / "selected_template.json", {"summary": "Automatically or explicitly selected content-free architecture template.", **selected_template}) - blueprint_report = render_layout_blueprint(selected_template, root / "layout_blueprint.png", target_ratio=preferences["aspect_ratio"]) + blueprint_report = render_layout_blueprint( + selected_template, + root / "layout_blueprint.png", + target_ratio=preferences["aspect_ratio"], + figure_specification=plan.get("figure_specification"), + ) write_json(root / "layout_blueprint.json", blueprint_report) + if isinstance(blueprint_report.get("semantic_plan"), dict) and blueprint_report["semantic_plan"].get("applied"): + selected_template["semantic_plan"] = blueprint_report["semantic_plan"] + overlay_spec = read_json(root / "overlay_spec.json") + overlay_enabled = bool(deterministic_overlay and selected_template.get("semantic_plan", {}).get("applied") and not repair_source) + if editable_overlay_ppt and not overlay_enabled: + raise RuntimeError("Editable visual-substrate PPT requires an applied deterministic semantic overlay and cannot use an external baked repair source") + generation_blueprint = root / "layout_blueprint.png" + if overlay_enabled: + generation_blueprint = root / "visual_substrate_blueprint.png" + visual_blueprint_report = render_visual_substrate_blueprint( + selected_template["semantic_plan"], + generation_blueprint, + width=int(blueprint_report.get("width") or 1536), + height=int(blueprint_report.get("height") or 1024), + ) + write_json(root / "visual_substrate_blueprint.json", visual_blueprint_report) - references = archived_positive + archived_negative - plan, planner_metadata = plan_paper_image( - parsed, - preferences, - mode=planner_mode, - model=planner_model, - reference_images=references, - paper_review=paper_review, - ) - write_text(root / "prompts" / "planning_prompt.txt", planner_metadata.pop("prompt")) - write_json(root / "planning_metadata.json", {"summary": "Planner execution metadata.", **planner_metadata}) - artifact_names = ["paper_summary", "figure_specification", "design_plan", "layout_intent", "visual_metaphors", "style_plan"] - for name in artifact_names: - write_json(root / f"{name}.json", plan[name]) plan["style_plan"]["selected_template_id"] = selected_template.get("profile_id") plan["style_plan"]["template_style"] = selected_template.get("style", {}) plan["style_plan"]["template_palette"] = selected_template.get("palette", []) write_json(root / "style_plan.json", plan["style_plan"]) - planning_validation = validate_plan_grounding(plan, parsed) write_json(root / "planning_validation_report.json", planning_validation) if not planning_validation["ok"]: raise ValueError(f"Paper-to-image planning failed scientific grounding validation: {planning_validation['errors']}") - final_prompt = compile_image_prompt(plan, preferences, candidate_variant=1, selected_template=selected_template) + final_prompt = compile_image_prompt(plan, preferences, candidate_variant=1, selected_template=selected_template, deterministic_overlay=overlay_enabled) write_text(root / "image_prompt.txt", final_prompt) generation_parameters = { "summary": "Image generation parameters.", @@ -157,9 +163,15 @@ def run_paper_to_image( "review_model": review_model, "domain_profile": selected_domain["id"], "template": selected_template["template_id"], - "repair_rounds": max(0, min(1, int(repair_rounds))), + "repair_rounds": max(0, min(4, int(repair_rounds))), + "repair_source": str(Path(repair_source).resolve()) if repair_source else None, + "resume_candidates": bool(resume_candidates), "ocr_engine": ocr_engine, "ocr_lang": ocr_lang, + "deterministic_overlay": overlay_enabled, + "visual_substrate_geometry_normalization": overlay_enabled, + "editable_overlay_ppt": bool(editable_overlay_ppt), + "generation_blueprint": str(generation_blueprint), } write_json(root / "generation_parameters.json", generation_parameters) @@ -176,14 +188,46 @@ def run_paper_to_image( review_mode=review_mode, review_model=review_model, repair_rounds=repair_rounds, + repair_source=repair_source, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, + topology_adapter=topology_adapter, + resume_candidates=resume_candidates, + require_visual_enrichment=require_visual_enrichment, + generation_blueprint_path=generation_blueprint, + overlay_spec=overlay_spec, + deterministic_overlay=overlay_enabled, ) + editable_overlay = None + if editable_overlay_ppt: + selected_id = generation.get("selected_candidate_id") + candidates_by_id = { + str(item.get("candidate_id")): item + for item in generation.get("candidates", []) + if isinstance(item, dict) and item.get("candidate_id") + } + selected_record = candidates_by_id.get(str(selected_id)) if selected_id else None + if selected_record is None and candidates_by_id: + selected_record = max(candidates_by_id.values(), key=lambda item: float(item.get("score") or 0.0)) + substrate_path = Path(str(selected_record.get("visual_substrate_path") or "")).resolve() if isinstance(selected_record, dict) and selected_record.get("visual_substrate_path") else Path() + if not substrate_path.is_file(): + raise RuntimeError("Editable visual-substrate PPT requires the selected candidate normalized visual substrate") + from .editable_ppt import compile_visual_substrate_ppt + + editable_overlay = compile_visual_substrate_ppt( + plan["figure_specification"], + substrate_path, + root, + aspect_ratio=preferences["aspect_ratio"], + title=str(plan.get("paper_summary", {}).get("title") or "").strip() or None, + semantic_plan=selected_template["semantic_plan"], + overlay_spec=overlay_spec, + ) elapsed = round(time.time() - started, 3) run_summary: dict[str, Any] = { - "summary": "Paper-to-image workflow completed without PPTX generation.", + "summary": "Paper-to-image workflow completed with a geometry-normalized visual substrate and optional native editable PPT structure layer.", "ok": True, "out_dir": str(root), "paper": archived_paper, @@ -200,19 +244,30 @@ def run_paper_to_image( "engineering_preview": generation.get("engineering_preview"), "selected_candidate_id": generation["selected_candidate_id"], "selected_passed_all_checks": generation["selected_passed_all_checks"], + "repair_candidates": generation.get("repair_candidates", 0), + "repair_stop_reason": generation.get("repair_stop_reason"), + "repair_source": str(Path(repair_source).resolve()) if repair_source else None, + "resume_candidates": bool(resume_candidates), + "deterministic_overlay": overlay_enabled, + "editable_overlay_ppt": bool(editable_overlay_ppt), + "editable_composition": editable_overlay.get("pptx") if isinstance(editable_overlay, dict) else None, "elapsed_seconds": elapsed, - "pptx_generated": False, + "pptx_generated": bool(editable_overlay and editable_overlay.get("ok")), "artifacts": [ "input_manifest.json", "preferences.json", "evidence_map.json", "document_index.json", "paper_review.json", + "key_evidence.json", + "review_ground_truth.json", "review_coverage_report.json", "domain_profile.json", "template_profiles/", "selected_template.json", "layout_blueprint.png", + *( ["visual_substrate_blueprint.png", "visual_substrate_blueprint.json", "geometry/", "overlays/"] if overlay_enabled else [] ), + *( ["figure_program.json", "editable_composition.pptx", "editable_overlay_report.json", "composition_quality_report.json"] if editable_overlay else [] ), "paper_summary.json", "figure_specification.json", "design_plan.json", @@ -224,12 +279,19 @@ def run_paper_to_image( "generation_parameters.json", "image2_request_manifest.json", "candidate_review.json", + "stability_report.json", "ocr_review.json", "template_alignment_report.json", "scientific_critic_report.json", + "paper_alignment_report.json", + "topology_critic_report.json", "aesthetic_critic_report.json", + "repair_plan.json", + "repair_history.json", "selected_image.png" if generation.get("selected_image") else "engineering_preview.png", ], } write_json(root / "run_summary.json", run_summary) + if editable_overlay: + run_summary["editable_overlay"] = editable_overlay return run_summary diff --git a/rfs/planning/__init__.py b/rfs/planning/__init__.py new file mode 100644 index 0000000..643fe0f --- /dev/null +++ b/rfs/planning/__init__.py @@ -0,0 +1,6 @@ +"""Paper, design, and image planning entry points.""" + +from ..paper_to_image.planner import plan_paper_image, validate_plan_grounding +from ..rebuild_design_planner import plan_rebuild_design + +__all__ = ["plan_paper_image", "plan_rebuild_design", "validate_plan_grounding"] diff --git a/rfs/ppt_compiler.py b/rfs/ppt_compiler.py index 5e4dbee..a029971 100644 --- a/rfs/ppt_compiler.py +++ b/rfs/ppt_compiler.py @@ -1,560 +1,6 @@ -from __future__ import annotations +"""Compatibility imports for PowerPoint composition. -from pathlib import Path +New code should import from :mod:`rfs.composition`. +""" -from PIL import Image -from pptx import Presentation -from pptx.dml.color import RGBColor -from pptx.enum.dml import MSO_LINE_DASH_STYLE -from pptx.enum.shapes import MSO_CONNECTOR, MSO_SHAPE -from pptx.enum.text import PP_ALIGN, MSO_ANCHOR -from pptx.oxml import parse_xml -from pptx.util import Inches, Pt - -from .utils import pct_to_inches, write_json - - -def _rgb(hex_color: str) -> RGBColor: - value = hex_color.strip().lstrip("#") or "000000" - return RGBColor(int(value[:2], 16), int(value[2:4], 16), int(value[4:6], 16)) - - -def _apply_arrow(connector, size: str = "sm") -> None: - ln = connector.line._get_or_add_ln() - ln.append(parse_xml(f'')) - - -def _apply_line_cap(connector, cap: str = "round") -> None: - value = {"round": "rnd", "square": "sq"}.get(str(cap).lower()) - if not value: - return - ln = connector.line._get_or_add_ln() - ln.set("cap", value) - - -def _connector_type_for_route(route_style: str): - if str(route_style).lower() in {"soft_curve", "dashed_loop", "dashed_spline_like"}: - return MSO_CONNECTOR.CURVE - return MSO_CONNECTOR.STRAIGHT - - -def _set_text(shape, text: str, font_size: float = 10, bold: bool = False, color: str = "#163B4D", align=PP_ALIGN.CENTER, font_family: str | None = None) -> None: - tf = shape.text_frame - tf.clear() - tf.word_wrap = True - tf.vertical_anchor = MSO_ANCHOR.MIDDLE - p = tf.paragraphs[0] - p.alignment = align - run = p.add_run() - run.text = text - run.font.size = Pt(float(font_size)) - run.font.bold = bold - if font_family: - run.font.name = str(font_family) - run.font.color.rgb = _rgb(color) - - -def _add_round_rect(slide, x: float, y: float, w: float, h: float, fill: str, stroke: str, width_pt: float = 1.2): - shape = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x), Inches(y), Inches(w), Inches(h)) - shape.fill.solid() - shape.fill.fore_color.rgb = _rgb(fill) - shape.line.color.rgb = _rgb(stroke) - shape.line.width = Pt(width_pt) - shape.shadow.inherit = False - return shape - - -def _add_label(slide, text: str, x: float, y: float, w: float, h: float, font_size: float = 9, bold: bool = False, align=PP_ALIGN.CENTER, color: str = "#163B4D", font_family: str | None = None): - box = slide.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h)) - _set_text(box, text, font_size=font_size, bold=bold, color=color, align=align, font_family=font_family) - return box - - -def _add_title_block(slide, program: dict, width_in: float, height_in: float) -> None: - title_block = program.get("title_block") - if not isinstance(title_block, dict): - return - title = str(title_block.get("title", "")).strip() - subtitle = str(title_block.get("subtitle", "")).strip() - bbox = title_block.get("bbox_percent") - if not title or not isinstance(bbox, dict): - return - x, y, w, h = pct_to_inches(bbox, width_in, height_in) - title_h = min(h * 0.58, 0.42) - subtitle_h = max(0.18, min(h - title_h, 0.26)) - _add_label( - slide, - title, - x, - y, - w, - title_h, - font_size=int(title_block.get("title_font_size", 24)), - bold=True, - align=PP_ALIGN.LEFT, - ) - if subtitle: - subtitle_box = _add_label( - slide, - subtitle, - x, - y + title_h * 0.94, - w, - subtitle_h, - font_size=int(title_block.get("subtitle_font_size", 11)), - bold=False, - align=PP_ALIGN.LEFT, - ) - for paragraph in subtitle_box.text_frame.paragraphs: - for run in paragraph.runs: - run.font.color.rgb = _rgb(str(title_block.get("subtitle_color", "#333333"))) - - -def _short_caption(text: str) -> str: - replacements = { - "participant seated before 49-inch screen": "Participant setup", - "3D virtual interviewer on screen": "Virtual interviewer", - "wide-angle camera capturing full body": "Full-body camera", - "virtual interview room setup": "Interview room", - "interview question prompt on screen": "Question prompt", - "raw interview video recording": "Raw video", - "287 participant video collection": "287 participants", - "36 interview questions": "36 questions", - "full-body video thumbnails": "Full-body clips", - "participant clip grid": "Clip grid", - "FFmpeg extracts audio from video": "FFmpeg audio", - "audio waveform stream": "Audio stream", - "FunASR speech recognition": "FunASR ASR", - "spoken text with timestamps": "Text + timestamps", - "MTCNN face detection": "MTCNN face", - "face clips": "Face clips", - "AlphaPose full-body skeleton extraction": "AlphaPose pose", - "full-body pose skeleton stream": "Pose stream", - "frame sampling from video": "Frame sampling", - "sampled video frames": "Video frames", - "timestamp alignment clock": "Alignment clock", - "aligned audio and text streams": "Audio + text", - "aligned face and pose streams": "Face + pose", - "aligned frame stream": "Frame stream", - "face modality": "Face", - "frame modality": "Frame", - "pose modality": "Pose", - "audio modality": "Audio", - "text modality": "Text", - "NEO-FFI-3 questionnaire": "NEO-FFI-3", - "OCEAN score vector": "OCEAN scores", - "Openness score": "Openness", - "Conscientiousness score": "Conscientiousness", - "Extraversion score": "Extraversion", - "Agreeableness score": "Agreeableness", - "Neuroticism score": "Neuroticism", - } - if text in replacements: - return replacements[text] - if len(text) <= 22: - return text - words = text.replace("/", " ").split() - return " ".join(words[:3]) if len(words) > 3 else text[:22] - - -def _picture_contain_box(image_path: Path, x: float, y: float, w: float, h: float) -> tuple[float, float, float, float, float]: - with Image.open(image_path) as img: - iw, ih = img.size - image_ratio = iw / max(ih, 1) - slot_ratio = w / max(h, 0.001) - if image_ratio > slot_ratio: - fit_w = w - fit_h = w / image_ratio - else: - fit_h = h - fit_w = h * image_ratio - left = x + (w - fit_w) / 2 - top = y + (h - fit_h) / 2 - fill_percent = fit_w * fit_h / max(w * h, 0.001) * 100 - return left, top, fit_w, fit_h, fill_percent - - -def _add_picture_contain(slide, image_path: Path, x: float, y: float, w: float, h: float): - left, top, fit_w, fit_h, _fill_percent = _picture_contain_box(image_path, x, y, w, h) - return slide.shapes.add_picture(str(image_path), Inches(left), Inches(top), width=Inches(fit_w), height=Inches(fit_h)) - - -def _panel_map(program: dict) -> dict[str, dict]: - return {panel["id"]: panel for panel in program.get("panels", [])} - - -def _text_program_items(program: dict) -> list[dict]: - text_program = program.get("text_program") - if not isinstance(text_program, dict): - return [] - items = text_program.get("items", []) - return [item for item in items if isinstance(item, dict) and item.get("visible", True)] - - -def _text_item_for_target(program: dict, target_id: str, role: str) -> dict | None: - for item in _text_program_items(program): - if str(item.get("target_id")) == target_id and str(item.get("role")) == role: - return item - return None - - -def _align_from_text_item(item: dict): - value = str(item.get("align") or "").lower() - if value == "left": - return PP_ALIGN.LEFT - if value == "right": - return PP_ALIGN.RIGHT - return PP_ALIGN.CENTER - - -def _object_map(program: dict) -> dict[str, dict]: - objects = {panel["id"]: panel for panel in program.get("panels", [])} - objects.update({slot["id"]: slot for slot in program.get("slots", [])}) - return objects - - -def _bbox_center(bbox: dict, canvas_w: float, canvas_h: float) -> tuple[float, float]: - x, y, w, h = pct_to_inches(bbox, canvas_w, canvas_h) - return x + w / 2, y + h / 2 - - -def _arrow_endpoints(source: dict, target: dict, canvas_w: float, canvas_h: float) -> tuple[float, float, float, float]: - sbox = source["bbox_percent"] - tbox = target["bbox_percent"] - sx, sy = _bbox_center(sbox, canvas_w, canvas_h) - tx, ty = _bbox_center(tbox, canvas_w, canvas_h) - s_left, s_top, s_w, s_h = pct_to_inches(sbox, canvas_w, canvas_h) - t_left, t_top, t_w, t_h = pct_to_inches(tbox, canvas_w, canvas_h) - - dx = tx - sx - dy = ty - sy - if abs(dx) >= abs(dy): - if dx >= 0: - return s_left + s_w + 0.03, sy, t_left - 0.03, ty - return s_left - 0.03, sy, t_left + t_w + 0.03, ty - if dy >= 0: - return sx, s_top + s_h + 0.03, tx, t_top - 0.03 - return sx, s_top - 0.03, tx, t_top + t_h + 0.03 - - -def _arrow_points_from_path(path: list, width_in: float, height_in: float) -> list[tuple[float, float]]: - points = [] - for point in path: - if isinstance(point, list) and len(point) >= 2: - points.append((float(point[0]) * width_in, float(point[1]) * height_in)) - return points - - -def _draw_program_arrows(slide, program: dict, width_in: float, height_in: float) -> list[dict]: - objects_by_id = _object_map(program) - style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} - token_map = {str(item.get("token_id")): item for item in style.get("color_tokens", []) if isinstance(item, dict)} - rendered: list[dict] = [] - for arrow in program.get("arrows", []): - if arrow.get("type") == "custom_bus": - continue - source = objects_by_id.get(arrow.get("source") or arrow.get("source_id")) - target = objects_by_id.get(arrow.get("target") or arrow.get("target_id")) - path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] - if len(path) >= 2 and all(isinstance(point, list) and len(point) >= 2 for point in path): - points = _arrow_points_from_path(path, width_in, height_in) - elif source and target: - x1, y1, x2, y2 = _arrow_endpoints(source, target, width_in, height_in) - points = [(x1, y1), (x2, y2)] - else: - continue - token = token_map.get(str(arrow.get("style_token_id"))) - arrow_color = str(token.get("hex")) if token else str(arrow.get("stroke_color") or "#1F6F8B") - control_kind = str(arrow.get("control_kind") or arrow.get("type", "")).lower() - dashed = control_kind in {"dashed_loop", "dashed", "loop"} - if str(arrow.get("line_pattern", "")).lower() in {"dash", "dashed"}: - dashed = True - line_width = float(arrow.get("stroke_width_pt") or style.get("arrow_weight_pt") or 1.7) - route_style = str(arrow.get("route_style") or "") - connector_type = _connector_type_for_route(route_style) - halo_width = float(arrow.get("halo_width_pt") or 0.0) - halo_color = str(arrow.get("halo_color") or "#FFFFFF") - arrowhead_size = str(arrow.get("arrowhead_size") or "sm").lower() - if arrowhead_size not in {"sm", "med", "lg"}: - arrowhead_size = "sm" - segment_count = 0 - for idx, ((x1, y1), (x2, y2)) in enumerate(zip(points[:-1], points[1:])): - if halo_width > 0: - halo = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) - halo.line.color.rgb = _rgb(halo_color) - halo.line.width = Pt(max(line_width + 1.2, halo_width)) - _apply_line_cap(halo, str(arrow.get("line_cap") or "round")) - if dashed: - halo.line.dash_style = MSO_LINE_DASH_STYLE.DASH - connector = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) - connector.line.color.rgb = _rgb(arrow_color) - connector.line.width = Pt(line_width) - _apply_line_cap(connector, str(arrow.get("line_cap") or "round")) - if dashed: - connector.line.dash_style = MSO_LINE_DASH_STYLE.DASH - if idx == len(points) - 2: - _apply_arrow(connector, arrowhead_size) - segment_count += 1 - rendered.append({ - "arrow_id": arrow.get("id"), - "control_kind": control_kind or "straight_arrow", - "semantic_role": arrow.get("semantic_role"), - "route_style": route_style, - "bundle_id": arrow.get("bundle_id"), - "lane_index": arrow.get("lane_index"), - "line_cap": arrow.get("line_cap", "round"), - "line_pattern": "dash" if dashed else "solid", - "connector_type": "curve" if connector_type == MSO_CONNECTOR.CURVE else "straight", - "halo_width_pt": halo_width, - "halo_color": halo_color if halo_width > 0 else None, - "stroke_width_pt": line_width, - "arrowhead_size": arrowhead_size, - "routing_algorithm": arrow.get("routing_algorithm"), - "route_generation_status": arrow.get("route_generation_status"), - "segment_count": segment_count, - "point_count": len(points), - "editable_in": "pptx", - "render_policy": "ppt_shape_not_image_asset", - "status": "ok" if segment_count else "not_rendered", - }) - return rendered - - -def _ocean_label(text: str) -> str: - low = text.lower() - if "openness" in low: - return "Openness" - if "conscientiousness" in low: - return "Conscientiousness" - if "extraversion" in low: - return "Extraversion" - if "agreeableness" in low: - return "Agreeableness" - if "neuroticism" in low: - return "Neuroticism" - return _short_caption(text) - - -def compile_ppt(program: dict, out_dir: str | Path) -> Path: - out = Path(out_dir) - canvas = program["canvas"] - width_in = float(canvas["width_in"]) - height_in = float(canvas["height_in"]) - style = program.get("style", {}) - palette = style.get("palette") or style.get("reference_palette") or ["#2D6FB7", "#E17721", "#6B57C8", "#1B9A94", "#4B9B52", "#D44E5D"] - panel_styles = style.get("panel_styles", {}) if isinstance(style.get("panel_styles"), dict) else {} - - prs = Presentation() - prs.slide_width = Inches(width_in) - prs.slide_height = Inches(height_in) - slide = prs.slides.add_slide(prs.slide_layouts[6]) - - _add_title_block(slide, program, width_in, height_in) - - panels_by_id = _panel_map(program) - panel_shapes = {} - has_text_program = bool(_text_program_items(program)) - ocr_panel_title_targets = { - str(item.get("target_id")) - for item in _text_program_items(program) - if str(item.get("role")) == "panel_title" and "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() - } - for idx, panel in enumerate(program["panels"]): - x, y, w, h = pct_to_inches(panel["bbox_percent"], width_in, height_in) - local_style = panel_styles.get(panel["id"], {}) if isinstance(panel_styles.get(panel["id"], {}), dict) else {} - fill = local_style.get("fill_color") or palette[idx % len(palette)] - stroke = local_style.get("stroke_color") or palette[(idx + 1) % len(palette)] - header_color = local_style.get("header_color") or stroke - shape = _add_round_rect(slide, x, y, w, h, fill, stroke, width_pt=1.4) - panel_shapes[panel["id"]] = shape - header_h = min(0.34, h * 0.17) - header = _add_round_rect(slide, x, y, w, header_h, header_color, header_color, width_pt=0.8) - title_text_item = _text_item_for_target(program, panel["id"], "panel_title") - _set_text( - header, - "" if str(panel["id"]) in ocr_panel_title_targets else panel["title"], - font_size=float(title_text_item.get("font_size_pt")) if title_text_item else (9 if w < 2.0 else 10), - bold=True, - color=str(title_text_item.get("color_hex")) if title_text_item else "#FFFFFF", - font_family=str(title_text_item.get("font_family_guess") or "") if title_text_item else None, - ) - - for idx, card in enumerate(program.get("cards", [])): - bbox = card.get("bbox_percent") - if not isinstance(bbox, dict): - continue - x, y, w, h = pct_to_inches(bbox, width_in, height_in) - fill = card.get("fill_color") or "#FFFFFF" - stroke = card.get("stroke_color") or (palette[(idx + 1) % len(palette)] if palette else "#B8C0CC") - _add_round_rect(slide, x, y, w, h, fill, stroke, width_pt=float(card.get("stroke_width_pt") or 1.0)) - if card.get("title") and not has_text_program: - _add_label(slide, str(card.get("title")), x + 0.03, y + 0.03, max(0.05, w - 0.06), min(0.22, h * 0.32), font_size=7, bold=True) - - rendered_arrows = _draw_program_arrows(slide, program, width_in, height_in) - - # Slot image layer and editable captions. - composition_items = [] - caption_queue = [] - ordered_slots = sorted(program["slots"], key=lambda item: int(item.get("z_index", 20))) - for slot in ordered_slots: - x, y, w, h = pct_to_inches(slot["bbox_percent"], width_in, height_in) - asset_path = out / "assets" / f"{slot['asset_id']}.png" - tile_added = False - if slot.get("composition_type") == "symbol_cutout": - if asset_path.exists(): - _add_picture_contain(slide, asset_path, x, y, w, h) - _left, _top, fit_w, fit_h, fill_percent = _picture_contain_box(asset_path, x, y, w, h) - else: - fill_percent = 0.0 - if bool(slot.get("show_slot_caption", False)) and not has_text_program: - parent = panels_by_id.get(slot.get("panel_id")) - if parent: - px, py, pw, ph = pct_to_inches(parent["bbox_percent"], width_in, height_in) - label_x = x + w + 0.035 - label_w = max(0.28, min(0.86, px + pw - label_x - 0.03)) - caption_queue.append(( - _ocean_label(slot.get("display_label") or slot["paper_concept"]), - label_x, - y + h * 0.10, - label_w, - h * 0.80, - 5 if label_w < 0.55 else 6, - PP_ALIGN.LEFT, - )) - composition_items.append({ - "slot_id": slot["id"], - "asset_id": slot["asset_id"], - "slot_frame_policy": slot.get("slot_frame_policy", "frameless_slot"), - "picture_fill_policy": slot.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), - "tile_frame_added": tile_added, - "caption_inside_image_slot": False, - "slot_bbox_percent": slot["bbox_percent"], - "image_slot_area_fill_percent": round(fill_percent, 2), - "status": "ok" if fill_percent >= 95 else "image_area_below_95", - }) - continue - if asset_path.exists(): - _add_picture_contain(slide, asset_path, x, y, w, h) - _left, _top, fit_w, fit_h, fill_percent = _picture_contain_box(asset_path, x, y, w, h) - else: - fill_percent = 0.0 - if bool(slot.get("show_slot_caption", False)) and not has_text_program: - caption_h = min(0.20, h * 0.18) - caption = slot.get("display_label") or _short_caption(slot["paper_concept"]) - caption_queue.append((caption, x + w * 0.03, y + h + 0.01, w * 0.94, caption_h, 5 if w < 0.8 else 6, PP_ALIGN.CENTER)) - composition_items.append({ - "slot_id": slot["id"], - "asset_id": slot["asset_id"], - "slot_frame_policy": slot.get("slot_frame_policy", "frameless_slot"), - "picture_fill_policy": slot.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), - "tile_frame_added": tile_added, - "caption_inside_image_slot": False, - "slot_bbox_percent": slot["bbox_percent"], - "image_slot_area_fill_percent": round(fill_percent, 2), - "status": "ok" if fill_percent >= 95 else "image_area_below_95", - }) - - for caption, x, y, w, h, font_size, align in caption_queue: - _add_label(slide, caption, x, y, w, h, font_size=font_size, align=align) - - rendered_text_items = [] - for item in _text_program_items(program): - is_ocr_text = "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() - if str(item.get("role")) == "panel_title" and not is_ocr_text: - rendered_text_items.append({ - "text_id": item.get("id"), - "role": item.get("role"), - "target_id": item.get("target_id"), - "rendered_as": "panel_header_text", - "editable_in": "pptx", - }) - continue - bbox = item.get("bbox_percent") - if not isinstance(bbox, dict): - continue - x, y, w, h = pct_to_inches(bbox, width_in, height_in) - _add_label( - slide, - str(item.get("text") or ""), - x, - y, - w, - h, - font_size=float(item.get("font_size_pt") or 6), - bold=bool(item.get("bold")), - align=_align_from_text_item(item), - color=str(item.get("color_hex") or "#263747"), - font_family=str(item.get("font_family_guess") or "") or None, - ) - rendered_text_items.append({ - "text_id": item.get("id"), - "role": item.get("role"), - "target_id": item.get("target_id"), - "source_reference_text_id": item.get("source_reference_text_id"), - "bbox_percent": item.get("bbox_percent"), - "font_size_pt": item.get("font_size_pt"), - "color_hex": item.get("color_hex"), - "font_family_guess": item.get("font_family_guess"), - "fit_strategy": item.get("fit_strategy"), - "ocr_confidence": item.get("ocr_confidence"), - "editable_in": "pptx", - "rendered_as": "ppt_textbox", - }) - - for label in program.get("labels", []): - bbox = label.get("bbox_percent") - if not isinstance(bbox, dict): - continue - x, y, w, h = pct_to_inches(bbox, width_in, height_in) - _add_label( - slide, - str(label.get("text") or ""), - x, - y, - w, - h, - font_size=float(label.get("font_size_pt") or 9), - bold=bool(label.get("bold", True)), - align=PP_ALIGN.LEFT if str(label.get("align")).lower() == "left" else PP_ALIGN.CENTER, - color=str(label.get("color_hex") or "#263747"), - ) - - # Shared resource bus as editable connectors. - shared = panels_by_id.get("shared_resource_library") - if shared: - sx, sy, sw, sh = pct_to_inches(shared["bbox_percent"], width_in, height_in) - bus_y = sy - 0.30 - line = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(sx + 0.1), Inches(bus_y), Inches(sx + sw - 0.1), Inches(bus_y)) - line.line.color.rgb = _rgb("#1F6F8B") - line.line.width = Pt(1.2) - for panel in program["panels"]: - if panel["id"] == "shared_resource_library": - continue - cx, _cy = _bbox_center(panel["bbox_percent"], width_in, height_in) - if not (sx + 0.1 <= cx <= sx + sw - 0.1): - continue - down = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(cx), Inches(bus_y), Inches(cx), Inches(sy)) - down.line.color.rgb = _rgb("#1F6F8B") - down.line.width = Pt(1.0) - if not has_text_program: - _add_label(slide, "shared resources", sx + 0.2, bus_y - 0.22, 1.5, 0.18, font_size=7, align=PP_ALIGN.LEFT) - - if program.get("show_visible_title"): - title = program.get("paper_brief", {}).get("title_guess") or "Research System Figure" - _add_label(slide, title[:90], width_in - 5.2, height_in - 0.50, 4.9, 0.38, font_size=7, bold=True, align=PP_ALIGN.RIGHT) - - pptx_path = out / "editable_composition.pptx" - prs.save(pptx_path) - write_json(out / "composition_quality_report.json", { - "summary": "PPT composition quality report checking frameless slot insertion, no extra white tiles, and image area fill inside each reference slot.", - "policy": { - "slot_frame_policy": "frameless_slot", - "picture_fill_policy": "direct_full_slot_contain_no_tile", - "min_image_slot_area_fill_percent": 95, - "caption_inside_image_slot_allowed": False, - }, - "slots": composition_items, - "arrows": rendered_arrows, - "text": rendered_text_items, - }) - return pptx_path +from .composition.pptx import * # noqa: F401,F403 diff --git a/rfs/professional_rebuild.py b/rfs/professional_rebuild.py index d9d433b..961f952 100644 --- a/rfs/professional_rebuild.py +++ b/rfs/professional_rebuild.py @@ -15,7 +15,7 @@ _ppt_package_counts, rebuild_editable, ) -from .ppt_compiler import compile_ppt +from .composition import compile_ppt from .professional_compiler import dsl_to_program from .professional_dsl import fallback_professional_dsl, validate_and_normalize_dsl from .professional_gap import build_professional_gap_report @@ -108,6 +108,7 @@ def _compile_professional( generate_assets: bool = True, baseline_program: dict | None = None, benchmark_out: str | Path | None = None, + asset_policy: str = "smart-api", ) -> dict: normalized, validation = validate_and_normalize_dsl(dsl, archived_reference, out) write_json(out / "professional_rebuild_script.dsl.json", normalized) @@ -134,7 +135,10 @@ def _compile_professional( _draw_geometry_overlay(archived_reference, out, program) _draw_controls_overlay(archived_reference, out, program.get("arrows", [])) - specs = _make_asset_specs(program, archived_reference, out) + specs = _make_asset_specs(program, archived_reference, out, asset_policy=asset_policy) + if asset_policy == "smart-api": + _write_contracts_from_program(out, program, planner_report) + _draw_geometry_overlay(archived_reference, out, program) write_json(out / "asset_generation_specs.json", {"summary": "Professional DSL slot-level asset generation specs.", "asset_mode": asset_mode, "specs": specs}) regen: set[str] = set() if isinstance(regenerate_slots, str): @@ -142,7 +146,7 @@ def _compile_professional( elif isinstance(regenerate_slots, list): regen = {str(item).strip() for item in regenerate_slots if str(item).strip()} if generate_assets: - asset_reports, asset_summary = _generate_assets(specs, program, out, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration) + asset_reports, asset_summary = _generate_assets(specs, program, out, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration, asset_policy=asset_policy) else: program["assets"] = [{"id": spec["asset_id"], "path": f"assets/{spec['asset_id']}.png", "source": "slot_asset"} for spec in specs] existing_report = _load_json_or_empty(out / "asset_generation_report.json") @@ -168,6 +172,7 @@ def _compile_professional( "rebuild_editable_summary": counts, "dsl_validation": validation, "asset_count": len(asset_reports), + "asset_policy": asset_policy, "professional_gap_report": gap_report, "no_full_image_policy": { "status": "pass", @@ -183,6 +188,7 @@ def _compile_professional( "pptx": str(pptx_path), "preview": str(preview) if preview else None, "asset_mode": asset_mode, + "asset_policy": asset_policy, "asset_workers": asset_workers, "asset_retries": asset_retries, "economy_mode": economy_mode, @@ -202,6 +208,9 @@ def _compile_professional( "figure_program": str(out / "figure_program.json"), "composition_quality_report": str(out / "composition_quality_report.json"), "professional_gap_report": str(out / "professional_gap_report.json"), + "asset_decision_report": str(out / "asset_decision_report.json"), + "text_asset_filter_report": str(out / "text_asset_filter_report.json"), + "api_asset_plan": str(out / "api_asset_plan.json"), }, } @@ -229,6 +238,7 @@ def rebuild_editable_pro( planner_adapter: Callable | None = None, repair_adapter: Callable | None = None, benchmark_out: str | Path | None = None, + asset_policy: str = "smart-api", ) -> dict: reference_path = Path(reference) if not reference_path.exists(): @@ -243,6 +253,7 @@ def rebuild_editable_pro( "reference": str(reference_path), "archived_reference": str(archived_reference), "asset_mode": asset_mode, + "asset_policy": asset_policy, "text_mode": text_mode, "control_mode": control_mode, "layout_mode": layout_mode, @@ -273,6 +284,7 @@ def rebuild_editable_pro( generate_assets=False, baseline_program=None, benchmark_out=benchmark_out, + asset_policy=asset_policy, ) result["compile_only"] = True write_json(out_path / "rebuild_result.json", result) @@ -298,6 +310,7 @@ def rebuild_editable_pro( vlm_layout_adapter=vlm_layout_adapter, control_adapter=control_adapter, semantic_adapter=semantic_adapter, + asset_policy="legacy", ) baseline_program = _load_json_or_empty(out_path / "figure_program.json") if not baseline_program: @@ -324,6 +337,7 @@ def rebuild_editable_pro( export_preview, baseline_program=baseline_program, benchmark_out=benchmark_out, + asset_policy=asset_policy, ) repair_reports = run_professional_repair_rounds(archived_reference, out_path, dsl, repair_rounds=repair_rounds, repair_adapter=repair_adapter) if any(int(item.get("applied_count") or 0) > 0 for item in repair_reports): @@ -343,6 +357,7 @@ def rebuild_editable_pro( export_preview, baseline_program=baseline_program, benchmark_out=benchmark_out, + asset_policy=asset_policy, ) result["repair_rounds"] = len(repair_reports) result["planner_status"] = planner_report.get("status") diff --git a/rfs/program_builder.py b/rfs/program_builder.py index d85e156..9267dc9 100644 --- a/rfs/program_builder.py +++ b/rfs/program_builder.py @@ -124,7 +124,10 @@ def build_figure_program(paper_brief: dict, inventory: dict, style: dict, out_di canvas_ratio = float(inventory.get("canvas_aspect_ratio") or 16 / 9) height_in = 7.5 - width_in = max(13.333, min(16.0, height_in * canvas_ratio)) + width_in = height_in * canvas_ratio + if width_in > 16.0: + width_in = 16.0 + height_in = width_in / canvas_ratio program = { "summary": "Structured figure program used as the only layout source for PPT composition.", "canvas": {"width_in": round(width_in, 3), "height_in": height_in, "ratio": round(canvas_ratio, 4), "background": "#FFFFFF"}, diff --git a/rfs/providers/__init__.py b/rfs/providers/__init__.py new file mode 100644 index 0000000..a9098dc --- /dev/null +++ b/rfs/providers/__init__.py @@ -0,0 +1,14 @@ +"""External model/provider integration entry points. + +Provider code must not contain repository paths or persist credentials. +""" + +from ..rebuild_vlm_adapters import build_rebuild_vlm_adapters +from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available + +__all__ = [ + "build_rebuild_vlm_adapters", + "call_vlm_json", + "resolve_vlm_model", + "vlm_credentials_available", +] diff --git a/rfs/rebuild_design_planner.py b/rfs/rebuild_design_planner.py new file mode 100644 index 0000000..d756c17 --- /dev/null +++ b/rfs/rebuild_design_planner.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +from PIL import Image + +from .layout_planner import canvas_inches, dominant_palette, estimate_background +from .utils import write_json, write_text +from .vlm_client import resolve_vlm_model + + +LAYER_KINDS = {"background", "panel", "card", "visual_slot", "text", "connector", "legend", "ignore"} +ASSET_POLICIES = {"reference_crop", "api_generate", "placeholder", "ppt_shape", "editable_text", "ppt_connector", "ignore"} + + +def _clamp_bbox(bbox: dict | None) -> dict[str, float]: + bbox = bbox if isinstance(bbox, dict) else {} + x = max(0.0, min(0.995, float(bbox.get("x", 0.0)))) + y = max(0.0, min(0.995, float(bbox.get("y", 0.0)))) + w = max(0.001, min(float(bbox.get("w", 0.001)), 1.0 - x)) + h = max(0.001, min(float(bbox.get("h", 0.001)), 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _clean_kind(value: object, default: str = "visual_slot") -> str: + text = str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + aliases = { + "slot": "visual_slot", + "image": "visual_slot", + "visual": "visual_slot", + "arrow": "connector", + "control": "connector", + "box": "card", + "background_panel": "panel", + } + text = aliases.get(text, text) + return text if text in LAYER_KINDS else default + + +def _clean_policy(value: object, kind: str) -> str: + text = str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + aliases = { + "crop": "reference_crop", + "screenshot": "reference_crop", + "api": "api_generate", + "generate": "api_generate", + "generated": "api_generate", + "shape": "ppt_shape", + "text": "editable_text", + "arrow": "ppt_connector", + "connector": "ppt_connector", + } + text = aliases.get(text, text) + if text in ASSET_POLICIES: + return text + if kind in {"background", "panel", "card"}: + return "ppt_shape" + if kind == "text": + return "editable_text" + if kind == "connector": + return "ppt_connector" + if kind in {"legend", "ignore"}: + return "ignore" + return "api_generate" + + +def _normalize_layers(raw_layers: list | None) -> list[dict]: + layers = [] + for idx, raw in enumerate(raw_layers or [], start=1): + if not isinstance(raw, dict): + continue + kind = _clean_kind(raw.get("kind") or raw.get("layer_kind") or raw.get("type")) + object_id = str(raw.get("id") or raw.get("object_id") or f"{kind}_{idx:02d}").strip() + if not object_id: + object_id = f"{kind}_{idx:02d}" + policy = _clean_policy(raw.get("asset_source_policy") or raw.get("policy") or raw.get("render_as"), kind) + record = { + "id": object_id, + "kind": kind, + "label": str(raw.get("label") or raw.get("title") or raw.get("prompt_subject") or object_id), + "bbox_percent": _clamp_bbox(raw.get("bbox_percent")), + "asset_source_policy": policy, + "asset_source_reason": str(raw.get("asset_source_reason") or raw.get("reason") or f"{kind}_default_policy"), + "z_order_hint": str(raw.get("z_order_hint") or raw.get("z_layer") or ""), + "confidence": _confidence(raw.get("confidence"), default=0.65), + } + if raw.get("panel_id"): + record["panel_id"] = str(raw.get("panel_id")) + if raw.get("prompt_subject"): + record["prompt_subject"] = str(raw.get("prompt_subject")) + if raw.get("semantic_role"): + record["semantic_role"] = str(raw.get("semantic_role")) + if raw.get("asset_type"): + record["asset_type"] = str(raw.get("asset_type")) + layers.append({key: value for key, value in record.items() if value not in ("", None)}) + return layers + + +def _confidence(value: object, default: float = 0.5) -> float: + try: + return round(max(0.0, min(1.0, float(value))), 4) + except Exception: + return default + + +def _normalize_asset_policies(raw_policies: list | None, layers: list[dict]) -> list[dict]: + by_id = {str(layer["id"]): layer for layer in layers} + policies = [] + for layer in layers: + if layer["kind"] == "visual_slot": + policies.append({ + "slot_id": layer["id"], + "object_id": layer["id"], + "policy": layer["asset_source_policy"], + "reason": layer.get("asset_source_reason") or "layer_policy", + "source": "layer_plan", + "confidence": layer.get("confidence", 0.65), + }) + for raw in raw_policies or []: + if not isinstance(raw, dict): + continue + object_id = str(raw.get("slot_id") or raw.get("object_id") or raw.get("id") or "") + if not object_id: + continue + kind = by_id.get(object_id, {}).get("kind", "visual_slot") + policies.append({ + "slot_id": object_id, + "object_id": object_id, + "policy": _clean_policy(raw.get("policy") or raw.get("asset_source_policy"), kind), + "reason": str(raw.get("reason") or raw.get("asset_source_reason") or "global_generation_policy"), + "source": "design_plan", + "confidence": _confidence(raw.get("confidence"), default=0.7), + }) + deduped = {} + for policy in policies: + deduped[str(policy["slot_id"])] = policy + return list(deduped.values()) + + +def _normalize_flow_graph(raw_flow: dict | None, layers: list[dict]) -> dict: + raw_flow = raw_flow if isinstance(raw_flow, dict) else {} + layer_ids = {str(layer["id"]) for layer in layers} + nodes = [] + seen = set() + for raw in raw_flow.get("nodes", []) if isinstance(raw_flow.get("nodes"), list) else []: + if not isinstance(raw, dict): + continue + node_id = str(raw.get("id") or raw.get("object_id") or "") + if not node_id: + continue + nodes.append({ + "id": node_id, + "label": str(raw.get("label") or node_id), + "kind": _clean_kind(raw.get("kind"), default="visual_slot"), + "confidence": _confidence(raw.get("confidence"), default=0.65), + }) + seen.add(node_id) + for layer in layers: + if layer["kind"] == "visual_slot" and layer["id"] not in seen: + nodes.append({"id": layer["id"], "label": layer.get("label", layer["id"]), "kind": "visual_slot", "confidence": layer.get("confidence", 0.65)}) + seen.add(layer["id"]) + edges = [] + invalid_count = 0 + for idx, raw in enumerate(raw_flow.get("edges", []) if isinstance(raw_flow.get("edges"), list) else [], start=1): + if not isinstance(raw, dict): + continue + source = str(raw.get("source_id") or raw.get("source") or "") + target = str(raw.get("target_id") or raw.get("target") or "") + if not source or not target: + invalid_count += 1 + continue + edges.append({ + "id": str(raw.get("id") or f"flow_{idx:02d}"), + "source_id": source, + "target_id": target, + "relation": str(raw.get("relation") or "flows_to"), + "expected_connector": str(raw.get("expected_connector") or raw.get("connector") or "solid_arrow"), + "confidence": _confidence(raw.get("confidence"), default=0.7), + "source_known_in_layer_plan": source in layer_ids, + "target_known_in_layer_plan": target in layer_ids, + }) + return { + "summary": "Reference flow graph inferred by global rebuild design planner.", + "nodes": nodes, + "edges": edges, + "invalid_edge_count": invalid_count, + } + + +def _heuristic_layers(reference_path: Path) -> tuple[dict, list[dict], list[dict], dict]: + with Image.open(reference_path).convert("RGB") as image: + width_px, height_px = image.size + width_in, height_in = canvas_inches(width_px, height_px) + background = estimate_background(image) + palette = dominant_palette(image) + layers = [ + { + "id": "reference_background", + "kind": "background", + "label": "Reference background", + "bbox_percent": {"x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0}, + "asset_source_policy": "ppt_shape", + "asset_source_reason": "full canvas background remains editable PPT shape", + "z_order_hint": "background", + "confidence": 0.35, + }, + { + "id": "reference_canvas", + "kind": "panel", + "label": "Reference canvas", + "bbox_percent": {"x": 0.025, "y": 0.065, "w": 0.95, "h": 0.84}, + "asset_source_policy": "ppt_shape", + "asset_source_reason": "default panel remains editable PPT shape", + "z_order_hint": "panel", + "confidence": 0.35, + }, + ] + logic = { + "summary": "Reference logic plan inferred by heuristic fallback.", + "mode": "heuristic", + "effective_mode": "heuristic", + "status": "fallback" if False else "pass", + "model": None, + "narrative": { + "figure_type": "workflow", + "main_story": "Reference-only editable rebuild of a scientific diagram.", + "key_message": "Preserve layout, editable text, connectors, and slot-level visual assets.", + }, + "reading_order": ["reference_canvas"], + "canvas": { + "width_px": width_px, + "height_px": height_px, + "width_in": width_in, + "height_in": height_in, + "background": background, + "palette": palette, + }, + "layers": layers, + "warnings": [], + } + generation = { + "summary": "Reference generation plan inferred by heuristic fallback.", + "mode": "heuristic", + "effective_mode": "heuristic", + "asset_policies": _normalize_asset_policies([], layers), + } + flow = _normalize_flow_graph({}, layers) + return logic, layers, generation["asset_policies"], flow + + +def _normalize_design_result(raw: dict, reference_path: Path, mode: str, model: str | None, fallback_reason: str | None = None) -> tuple[dict, dict, dict, dict]: + heuristic_logic, heuristic_layers, _heuristic_policies, heuristic_flow = _heuristic_layers(reference_path) + layers = _normalize_layers(raw.get("layers")) or heuristic_layers + policies = _normalize_asset_policies(raw.get("asset_policies"), layers) + flow = _normalize_flow_graph(raw.get("flow_graph"), layers) + effective_mode = "heuristic" if fallback_reason else ("off" if mode == "off" else mode) + status = "fallback_to_heuristic" if fallback_reason else ("skipped" if mode == "off" else "pass") + logic = { + "summary": "Global reference logic plan for editable rebuild.", + "mode": mode, + "effective_mode": effective_mode, + "status": status, + "model": raw.get("_vlm_model") or model, + "fallback_reason": fallback_reason, + "narrative": raw.get("narrative") if isinstance(raw.get("narrative"), dict) else heuristic_logic["narrative"], + "reading_order": raw.get("reading_order") if isinstance(raw.get("reading_order"), list) else heuristic_logic["reading_order"], + "canvas": heuristic_logic["canvas"], + "layers": layers, + "warnings": [fallback_reason] if fallback_reason else [], + } + layer_plan = { + "summary": "Reference layer plan for editable rebuild.", + "mode": mode, + "effective_mode": effective_mode, + "status": status, + "layer_count": len(layers), + "layers": layers, + "warnings": logic["warnings"], + } + generation = { + "summary": "Reference generation policy plan for editable rebuild.", + "mode": mode, + "effective_mode": effective_mode, + "status": status, + "asset_policies": policies, + "warnings": logic["warnings"], + } + flow["mode"] = mode + flow["effective_mode"] = effective_mode + flow["status"] = status + flow["warnings"] = logic["warnings"] + return logic, layer_plan, generation, flow + + +def _write_markdown(out: Path, logic: dict, layer_plan: dict, generation: dict, flow: dict) -> None: + lines = [ + "# Summary", + str(logic.get("summary") or "Global reference logic plan."), + "", + f"- Mode: {logic.get('mode')}", + f"- Effective mode: {logic.get('effective_mode')}", + f"- Status: {logic.get('status')}", + f"- Model: {logic.get('model')}", + f"- Layer count: {layer_plan.get('layer_count')}", + f"- Asset policy count: {len(generation.get('asset_policies', []))}", + f"- Flow edge count: {len(flow.get('edges', []))}", + ] + narrative = logic.get("narrative") if isinstance(logic.get("narrative"), dict) else {} + if narrative: + lines.extend(["", "## Narrative"]) + for key in ("figure_type", "main_story", "key_message"): + if narrative.get(key): + lines.append(f"- {key}: {narrative[key]}") + write_text(out / "reference_logic_plan.md", "\n".join(lines) + "\n") + + +def plan_rebuild_design( + reference_path: str | Path, + out_dir: str | Path, + *, + mode: str = "vlm", + model: str | None = None, + adapter: Callable[[str | Path, str | None], dict] | None = None, + fallback_on_error: bool = True, +) -> dict: + reference = Path(reference_path) + out = Path(out_dir) + requested = str(mode or "off").lower() + if requested == "off": + raw = {} + logic, layer_plan, generation, flow = _normalize_design_result(raw, reference, "off", None) + elif requested == "heuristic": + raw = {} + logic, layer_plan, generation, flow = _normalize_design_result(raw, reference, "heuristic", None) + elif requested == "vlm": + resolved_model = resolve_vlm_model("RFS_REBUILD_DESIGN_MODEL", "RFS_REBUILD_LAYOUT_MODEL", explicit_model=model) + try: + raw = adapter(reference, resolved_model) if adapter else {} + if not adapter: + raise RuntimeError("design VLM adapter unavailable") + logic, layer_plan, generation, flow = _normalize_design_result(raw, reference, "vlm", resolved_model) + except Exception as exc: + if not fallback_on_error: + raise + logic, layer_plan, generation, flow = _normalize_design_result({}, reference, "vlm", resolved_model, fallback_reason=f"vlm_design_planning_failed:{exc}") + else: + raise ValueError(f"Unsupported design plan mode: {mode}") + + write_json(out / "reference_logic_plan.json", logic) + write_json(out / "reference_layer_plan.json", layer_plan) + write_json(out / "reference_generation_plan.json", generation) + write_json(out / "reference_flow_graph.json", flow) + _write_markdown(out, logic, layer_plan, generation, flow) + return { + "logic": logic, + "layer_plan": layer_plan, + "generation_plan": generation, + "flow_graph": flow, + } diff --git a/rfs/rebuild_preview_renderer.py b/rfs/rebuild_preview_renderer.py new file mode 100644 index 0000000..8f707bc --- /dev/null +++ b/rfs/rebuild_preview_renderer.py @@ -0,0 +1,5 @@ +"""Compatibility import for rebuild preview rendering.""" + +from .composition.preview import render_rebuild_preview + +__all__ = ["render_rebuild_preview"] diff --git a/rfs/rebuild_visual_critic.py b/rfs/rebuild_visual_critic.py new file mode 100644 index 0000000..95f531f --- /dev/null +++ b/rfs/rebuild_visual_critic.py @@ -0,0 +1,6 @@ +"""Compatibility imports for rebuild visual evaluation. + +New code should import from :mod:`rfs.evaluation`. +""" + +from .evaluation.rebuild_visual import * # noqa: F401,F403 diff --git a/rfs/rebuild_vlm_adapters.py b/rfs/rebuild_vlm_adapters.py index f392c39..296ef90 100644 --- a/rfs/rebuild_vlm_adapters.py +++ b/rfs/rebuild_vlm_adapters.py @@ -7,6 +7,33 @@ from .vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available +def vlm_design_adapter(reference_path: str | Path, explicit_model: str | None = None) -> dict: + model = resolve_vlm_model("RFS_REBUILD_DESIGN_MODEL", "RFS_REBUILD_LAYOUT_MODEL", explicit_model=explicit_model) + prompt = """ +Analyze the complete reference diagram before local object detection. Return JSON only. + +Describe the narrative, reading order, editable layers, asset-source policy, and semantic flow graph. +Do not write PowerPoint code. Do not bake text or connectors into raster assets. +Use normalized bbox_percent coordinates in [0,1]. + +Allowed layer kinds: background, panel, card, visual_slot, text, connector, legend, ignore. +Allowed asset policies: reference_crop, api_generate, placeholder, ppt_shape, editable_text, ppt_connector, ignore. + +Return: +{ + "summary": "...", + "narrative": {"figure_type": "...", "main_story": "...", "key_message": "..."}, + "reading_order": ["stable_object_id"], + "layers": [{"id":"slot_input","kind":"visual_slot","label":"Input","bbox_percent":{"x":0,"y":0,"w":0.1,"h":0.1},"asset_source_policy":"api_generate","asset_source_reason":"...","prompt_subject":"...","asset_type":"generic","semantic_role":"input","confidence":0.0}], + "asset_policies": [{"slot_id":"slot_input","policy":"api_generate","reason":"...","confidence":0.0}], + "flow_graph": {"nodes":[{"id":"slot_input","label":"Input","kind":"visual_slot","confidence":0.0}],"edges":[{"id":"flow_1","source_id":"slot_input","target_id":"slot_output","relation":"flows_to","expected_connector":"solid_arrow","confidence":0.0}]} +} +""".strip() + result = call_vlm_json(prompt, [reference_path], model=model) + result["_vlm_model"] = model + return result + + def _brief_slots(slots: list[dict]) -> list[dict]: return [{ "id": item.get("id"), @@ -164,8 +191,9 @@ def vlm_semantic_adapter(reference_path: str | Path, slots: list[dict], panels: def build_rebuild_vlm_adapters(out_dir: str | Path) -> dict[str, Callable | None]: if not vlm_credentials_available(): - return {"layout": None, "control": None, "semantic": None} + return {"design": None, "layout": None, "control": None, "semantic": None} return { + "design": vlm_design_adapter, "layout": vlm_layout_adapter, "control": vlm_control_adapter_factory(out_dir), "semantic": vlm_semantic_adapter, diff --git a/rfs/reference_analyzer.py b/rfs/reference_analyzer.py index 6bf72af..2605d29 100644 --- a/rfs/reference_analyzer.py +++ b/rfs/reference_analyzer.py @@ -10,6 +10,7 @@ import requests from PIL import Image, ImageDraw +from .cv_compat import hough_line_coordinates from .utils import ratio_string, write_json @@ -374,8 +375,11 @@ def _detect_cv_control_candidates(reference_image: Image.Image, image_width: int raw: list[tuple[float, float, list[list[float]]]] = [] diag = (image_width * image_width + image_height * image_height) ** 0.5 - for line in lines[:, 0, :]: - x1, y1, x2, y2 = [float(v) for v in line] + for line in lines[:80]: + coordinates = hough_line_coordinates(line) + if coordinates is None: + continue + x1, y1, x2, y2 = [float(v) for v in coordinates] length = ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5 if length / max(diag, 1.0) < 0.025: continue @@ -603,17 +607,21 @@ def _supplement_reference_slots(slot_specs: list[Any], target_count: int) -> lis panel = str(_spec_value(base, "macro_panel") or "Paper Method") concept = str(_spec_value(base, "paper_concept") or base_id) composition = str(_spec_value(base, "composition_type") or "full_bleed_card") - supplemented.append({ - "id": f"{base_id}_detail_{detail_index:02d}", + detail = dict(base) if isinstance(base, dict) else {} + detail.update({ + "id": f"{base_id}_reference_detail_{detail_index:02d}", "macro_panel": panel, - "paper_concept": f"{concept} local detail {detail_index}", + "paper_concept": concept, "composition_type": "scene_thumbnail" if composition != "symbol_cutout" else "full_frame_icon", "visual_metaphor": str(_spec_value(base, "visual_metaphor") or ""), "must_show": _spec_value(base, "must_show") if isinstance(_spec_value(base, "must_show"), list) else [], "avoid_showing": _spec_value(base, "avoid_showing") if isinstance(_spec_value(base, "avoid_showing"), list) else [], "display_label": "", "show_slot_caption": False, + "reference_detail_index": detail_index, + "slot_origin": "reference_visual_split", }) + supplemented.append(detail) detail_index += 1 return supplemented supplemented = list(slot_specs) @@ -625,8 +633,8 @@ def _supplement_reference_slots(slot_specs: list[Any], target_count: int) -> lis col = quad % 2 row = quad // 2 new_spec = dict(base) - new_spec["id"] = f"{base['id']}_detail_{detail_index:02d}" - new_spec["paper_concept"] = f"{base.get('paper_concept', base['id'])} local detail {detail_index}" + new_spec["id"] = f"{base['id']}_reference_detail_{detail_index:02d}" + new_spec["paper_concept"] = str(base.get("paper_concept") or base["id"]) new_spec["composition_type"] = "scene_thumbnail" if base.get("composition_type") != "symbol_cutout" else "full_frame_icon" new_spec["bbox_percent"] = { "x": _round4(float(bbox["x"]) + float(bbox["w"]) * (0.04 + col * 0.50)), @@ -636,6 +644,8 @@ def _supplement_reference_slots(slot_specs: list[Any], target_count: int) -> lis } new_spec["display_label"] = "" new_spec["show_slot_caption"] = False + new_spec["reference_detail_index"] = detail_index + new_spec["slot_origin"] = "reference_visual_split" supplemented.append(new_spec) detail_index += 1 return supplemented @@ -1291,7 +1301,16 @@ def _generic_slot_specs(paper_brief: dict, slot_count: int) -> tuple[list[dict], "visual_metaphor": str(item.get("visual_metaphor") or "").strip(), "must_show": item.get("must_show") if isinstance(item.get("must_show"), list) else [], "avoid_showing": item.get("avoid_showing") if isinstance(item.get("avoid_showing"), list) else [], + "semantic_instance_id": item.get("semantic_instance_id"), + "semantic_id": item.get("semantic_id"), + "semantic_panel_id": item.get("semantic_panel_id"), + "semantic_role": item.get("semantic_role"), + "evidence_ids": item.get("evidence_ids") if isinstance(item.get("evidence_ids"), list) else [], + "contract_source": item.get("contract_source"), + "slot_origin": item.get("slot_origin") or "semantic_contract_node", }) + if specs and str(paper_brief.get("semantic_authority") or "") in {"panel_graphs", "global_graph"}: + return specs[:slot_count], panel_layout if len(specs) >= min(25, slot_count): return specs[:slot_count], panel_layout @@ -1385,14 +1404,12 @@ def analyze_reference( control_warnings.append("hybrid_downgraded_to_heuristic:no_api_credentials") allow_legacy_templates = False try: - import os allow_legacy_templates = os.getenv("RFS_ALLOW_LEGACY_TEMPLATES", "").lower() in {"1", "true", "yes"} except Exception: allow_legacy_templates = False reference_slot_source = str(slot_source or "").lower() or "paper" try: - import os reference_slot_source = str(slot_source or os.getenv("RFS_SLOT_SOURCE", "paper")).lower() except Exception: reference_slot_source = str(slot_source or "paper").lower() @@ -1502,6 +1519,13 @@ def analyze_reference( "avoid_showing": _spec_value(spec, "avoid_showing") if isinstance(_spec_value(spec, "avoid_showing"), list) else [], "display_label": str(_spec_value(spec, "display_label") or "").strip(), "show_slot_caption": bool(_spec_value(spec, "show_slot_caption")), + "semantic_instance_id": str(_spec_value(spec, "semantic_instance_id") or "").strip() or None, + "semantic_id": str(_spec_value(spec, "semantic_id") or "").strip() or None, + "semantic_panel_id": str(_spec_value(spec, "semantic_panel_id") or "").strip() or None, + "semantic_role": str(_spec_value(spec, "semantic_role") or "").strip() or None, + "semantic_evidence_ids": [str(value) for value in (_spec_value(spec, "evidence_ids") or []) if value], + "slot_origin": str(_spec_value(spec, "slot_origin") or "semantic_contract_node").strip(), + "reference_detail_index": _spec_value(spec, "reference_detail_index"), }) panel_geometry = [] @@ -1553,6 +1577,11 @@ def analyze_reference( reference_palette.append(color) reference_palette = _diverse_palette(reference_palette) for slot in slots: + if not slot.get("reference_dominant_colors"): + panel_id = str(slot.get("panel_id") or "") + panel_colors = panel_styles.get(panel_id, {}).get("dominant_colors", []) + fallback_colors = list(panel_colors[:3]) or list(reference_palette[:3]) or ["#4B86C5"] + slot["reference_dominant_colors"] = fallback_colors token_ids = [] for color_index, color in enumerate(slot.get("reference_dominant_colors", [])[:3], start=1): token_ids.append( diff --git a/rfs/reference_text_extractor.py b/rfs/reference_text_extractor.py index 25b5dbc..d76a634 100644 --- a/rfs/reference_text_extractor.py +++ b/rfs/reference_text_extractor.py @@ -1,6 +1,8 @@ from __future__ import annotations import math +import os +import threading from pathlib import Path from typing import Any, Callable @@ -8,6 +10,21 @@ OCR_LOW_CONFIDENCE = 0.65 +_EASYOCR_READERS: dict[tuple[str, ...], Any] = {} +_RAPIDOCR_LOCAL = threading.local() + + +def _positive_ocr_setting(explicit: int | None, env_name: str, default: int) -> int: + try: + return max(1, int(explicit if explicit is not None else os.getenv(env_name) or default)) + except (TypeError, ValueError): + return max(1, int(default)) + + +def _rapidocr_detector_limit(explicit: int | None = None) -> int: + """Return a bounded detector resize limit suitable for RapidOCR.""" + value = _positive_ocr_setting(explicit, "RFS_RAPIDOCR_DET_LIMIT", 512) + return max(256, min(2048, value)) def _round4(value: float) -> float: @@ -160,7 +177,16 @@ def run_easyocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: import easyocr # type: ignore lang_map = {"en_ch": ["ch_sim", "en"], "ch": ["ch_sim"], "en": ["en"]} - reader = easyocr.Reader(lang_map.get(str(lang), ["ch_sim", "en"]), gpu=False) + languages = tuple(lang_map.get(str(lang), ["ch_sim", "en"])) + reader = _EASYOCR_READERS.get(languages) + if reader is None: + allow_download = str(os.getenv("RFS_OCR_ALLOW_DOWNLOAD") or "").strip().casefold() in {"1", "true", "yes", "on"} + try: + reader = easyocr.Reader(list(languages), gpu=False, download_enabled=allow_download, verbose=False) + except Exception as exc: + hint = " Set RFS_OCR_ALLOW_DOWNLOAD=1 for an explicit one-time model download." if not allow_download else "" + raise RuntimeError(f"EasyOCR model is not ready for languages {list(languages)}.{hint} Original error: {exc}") from exc + _EASYOCR_READERS[languages] = reader records = [] for quad, text, confidence in reader.readtext(str(image_path)): points = _normalize_quad(quad) @@ -169,6 +195,68 @@ def run_easyocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: return records +def run_rapidocr_detailed( + image_path: str | Path, + lang: str, + *, + threads: int | None = None, + batch_size: int | None = None, + detector_limit: int | None = None, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + del lang + thread_count = _positive_ocr_setting(threads, "RFS_RAPIDOCR_THREADS", 1) + recognition_batch = _positive_ocr_setting(batch_size, "RFS_RAPIDOCR_BATCH", 6) + detector_side_limit = _rapidocr_detector_limit(detector_limit) + engines = getattr(_RAPIDOCR_LOCAL, "engines", None) + if engines is None: + engines = {} + _RAPIDOCR_LOCAL.engines = engines + engine_key = (thread_count, recognition_batch, detector_side_limit) + if engine_key not in engines: + from rapidocr_onnxruntime import RapidOCR # type: ignore + + engines[engine_key] = RapidOCR( + intra_op_num_threads=thread_count, + inter_op_num_threads=1, + det_limit_side_len=detector_side_limit, + rec_batch_num=recognition_batch, + ) + result, timings = engines[engine_key](str(image_path)) + records = [] + for item in result or []: + if not isinstance(item, (list, tuple)) or len(item) < 3: + continue + quad = _normalize_quad(item[0]) + text = str(item[1] or "").strip() + if quad and text: + records.append({"text": text, "confidence": float(item[2] or 0.0), "quad": quad}) + timing_values = [float(value or 0.0) for value in (timings or [])] + while len(timing_values) < 3: + timing_values.append(0.0) + diagnostics = { + "detector_limit": detector_side_limit, + "thread_count": thread_count, + "recognition_batch": recognition_batch, + "detection_seconds": round(timing_values[0], 4), + "classification_seconds": round(timing_values[1], 4), + "recognition_seconds": round(timing_values[2], 4), + "inference_seconds": round(sum(timing_values[:3]), 4), + "region_count": len(records), + } + return records, diagnostics + + +def run_rapidocr(image_path: str | Path, lang: str, *, threads: int | None = None, batch_size: int | None = None, detector_limit: int | None = None) -> list[dict[str, Any]]: + records, _diagnostics = run_rapidocr_detailed( + image_path, + lang, + threads=threads, + batch_size=batch_size, + detector_limit=detector_limit, + ) + return records + + def _record_to_region(record: dict[str, Any], image: Image.Image, program: dict, index: int, canvas_height_in: float, engine: str) -> dict: width, height = image.size xs = [point[0] for point in record["quad"]] @@ -237,6 +325,8 @@ def extract_reference_text( try: if ocr_adapter: raw_records = ocr_adapter(reference_path, lang) + elif requested_engine == "rapidocr": + raw_records = run_rapidocr(reference_path, lang) elif requested_engine == "easyocr": raw_records = run_easyocr(reference_path, lang) else: diff --git a/rfs/semantic_contract.py b/rfs/semantic_contract.py new file mode 100644 index 0000000..756d331 --- /dev/null +++ b/rfs/semantic_contract.py @@ -0,0 +1,6 @@ +"""Compatibility imports for the semantic contract API. + +New code should import from :mod:`rfs.contracts`. +""" + +from .contracts.semantic import * # noqa: F401,F403 diff --git a/rfs/text_grouping.py b/rfs/text_grouping.py new file mode 100644 index 0000000..c823d3e --- /dev/null +++ b/rfs/text_grouping.py @@ -0,0 +1,434 @@ +from __future__ import annotations + +import json +from pathlib import Path +from statistics import median +from typing import Callable + +from .vlm_client import call_vlm_json, resolve_vlm_model + +from .utils import write_json + + +def _clamp_bbox(bbox: dict) -> dict[str, float]: + x = max(0.0, min(0.995, float(bbox["x"]))) + y = max(0.0, min(0.995, float(bbox["y"]))) + w = max(0.001, min(float(bbox["w"]), 1.0 - x)) + h = max(0.001, min(float(bbox["h"]), 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _center(bbox: dict) -> dict[str, float]: + return { + "x": round(float(bbox["x"]) + float(bbox["w"]) / 2, 4), + "y": round(float(bbox["y"]) + float(bbox["h"]) / 2, 4), + } + + +def _union_bbox(regions: list[dict], expand: float = 0.0) -> dict[str, float]: + xs0 = [float(item["bbox_percent"]["x"]) for item in regions] + ys0 = [float(item["bbox_percent"]["y"]) for item in regions] + xs1 = [float(item["bbox_percent"]["x"]) + float(item["bbox_percent"]["w"]) for item in regions] + ys1 = [float(item["bbox_percent"]["y"]) + float(item["bbox_percent"]["h"]) for item in regions] + x0 = min(xs0) - expand + y0 = min(ys0) - expand + x1 = max(xs1) + expand + y1 = max(ys1) + expand + return _clamp_bbox({"x": x0, "y": y0, "w": x1 - x0, "h": y1 - y0}) + + +def _raw_ratio(region: dict) -> float: + try: + return float(region.get("raw_estimated_font_ratio") or region.get("estimated_font_ratio") or 0.0) + except Exception: + return 0.0 + + +def _font_pt(region: dict) -> float: + try: + return float(region.get("raw_font_size_pt") or region.get("font_size_pt") or 0.0) + except Exception: + return 0.0 + + +def _line_sort_key(region: dict) -> tuple[float, float]: + bbox = region["bbox_percent"] + return float(bbox["y"]), float(bbox["x"]) + + +def _eligible_for_heuristic_grouping(region: dict) -> bool: + if not str(region.get("source") or "").startswith("reference_ocr"): + return False + if str(region.get("role") or "") == "panel_title": + return False + text = str(region.get("text") or "").strip() + return bool(text) + + +def _same_paragraph(previous: dict, current: dict) -> bool: + if str(previous.get("target_id") or "") != str(current.get("target_id") or ""): + return False + if str(previous.get("role") or "") != str(current.get("role") or ""): + return False + if str(previous.get("font_family_guess") or "") != str(current.get("font_family_guess") or ""): + return False + prev = previous["bbox_percent"] + cur = current["bbox_percent"] + prev_h = float(prev["h"]) + cur_h = float(cur["h"]) + median_h = max(0.001, median([prev_h, cur_h])) + vertical_gap = float(cur["y"]) - (float(prev["y"]) + prev_h) + if vertical_gap < -median_h * 0.25 or vertical_gap > max(0.012, median_h * 0.95): + return False + height_delta = abs(prev_h - cur_h) / median_h + if height_delta > 0.35: + return False + left_delta = abs(float(prev["x"]) - float(cur["x"])) + if left_delta > max(0.018, median_h * 1.25): + return False + width_ratio = min(float(prev["w"]), float(cur["w"])) / max(float(prev["w"]), float(cur["w"]), 0.001) + if width_ratio < 0.28: + return False + ratio_values = [value for value in [_raw_ratio(previous), _raw_ratio(current)] if value > 0] + if len(ratio_values) == 2 and abs(ratio_values[0] - ratio_values[1]) / max(median(ratio_values), 0.0001) > 0.35: + return False + return True + + +def _make_group(group_id: str, members: list[dict]) -> dict: + ordered = sorted(members, key=_line_sort_key) + bbox = _union_bbox(ordered, expand=0.0015) + raw_ratios = [_raw_ratio(item) for item in ordered if _raw_ratio(item) > 0] + raw_pts = [_font_pt(item) for item in ordered if _font_pt(item) > 0] + confidences = [float(item.get("confidence") or 0.0) for item in ordered if item.get("confidence") is not None] + estimated_ratio = round(median(raw_ratios), 5) if raw_ratios else round(float(bbox["h"]) * 0.62, 5) + font_pt = round(median(raw_pts), 2) if raw_pts else None + region = dict(ordered[0]) + region.update({ + "id": group_id, + "text": "\n".join(str(item.get("text") or "").strip() for item in ordered if str(item.get("text") or "").strip()), + "raw_text": "\n".join(str(item.get("raw_text") or item.get("text") or "").strip() for item in ordered if str(item.get("raw_text") or item.get("text") or "").strip()), + "bbox_percent": bbox, + "line_bbox_percent": [item["bbox_percent"] for item in ordered], + "word_bbox_percent": [box for item in ordered for box in (item.get("word_bbox_percent") or [item["bbox_percent"]])], + "center_percent": _center(bbox), + "width_percent": round(float(bbox["w"]), 4), + "height_percent": round(float(bbox["h"]), 4), + "estimated_font_ratio": estimated_ratio, + "raw_estimated_font_ratio": estimated_ratio, + "font_size_pt": font_pt or region.get("font_size_pt"), + "raw_font_size_pt": font_pt or region.get("raw_font_size_pt"), + "confidence": round(sum(confidences) / len(confidences), 4) if confidences else region.get("confidence"), + "source": "reference_ocr_paragraph_group_heuristic", + "ocr_member_ids": [str(item.get("id") or "") for item in ordered], + "ocr_member_count": len(ordered), + "grouping_source": "heuristic", + "editable_in": "pptx", + }) + return region + + +def _object_brief(program: dict) -> dict: + def brief(items: list, keys: tuple[str, ...]) -> list[dict]: + result = [] + for item in items or []: + if not isinstance(item, dict): + continue + result.append({key: item.get(key) for key in keys if key in item}) + return result + + return { + "panels": brief(program.get("panels", []), ("id", "title", "bbox_percent")), + "cards": brief(program.get("cards", []), ("id", "title", "bbox_percent")), + "slots": brief(program.get("slots", []), ("id", "paper_concept", "bbox_percent")), + "arrows": brief(program.get("arrows", []), ("id", "source_id", "target_id", "source", "target", "path_percent")), + } + + +def _raw_ocr_brief(regions: list[dict]) -> list[dict]: + return [{ + "id": item.get("id"), + "text": item.get("text"), + "role_guess": item.get("role"), + "target_id": item.get("target_id"), + "bbox_percent": item.get("bbox_percent"), + "font_size_pt": item.get("font_size_pt"), + "confidence": item.get("confidence"), + } for item in regions if isinstance(item, dict)] + + +def _heuristic_group_brief(regions: list[dict]) -> list[dict]: + return [{ + "id": item.get("id"), + "text": item.get("text"), + "ocr_member_ids": item.get("ocr_member_ids"), + "role": item.get("role"), + "target_id": item.get("target_id"), + "bbox_percent": item.get("bbox_percent"), + } for item in regions if isinstance(item, dict)] + + +def _call_vlm_grouping( + reference_path: str | Path, + raw_regions: list[dict], + heuristic_regions: list[dict], + program: dict, + model: str | None = None, +) -> dict: + model_name = resolve_vlm_model("RFS_TEXT_GROUPING_MODEL", "RFS_TEXT_ROLE_MODEL", explicit_model=model) + prompt = f""" +You are grouping OCR text lines for an editable PowerPoint reconstruction. +Only output JSON. Do not output markdown, prose, Python, SVG, or PPT code. + +Task: +- Decide which raw OCR line ids belong to the same visible text paragraph/label. +- Decide whether a raw OCR line should be ignored because it is decorative, unreadable noise, or belongs only inside a raster asset. +- You may assign role and align, but you may NOT invent coordinates. +- Bboxes will be computed by code as the union of raw OCR boxes. +- Do not change text content. + +Return schema: +{{ + "summary": "...", + "groups": [ + {{"group_id":"group_1","ocr_member_ids":["ref_text_ocr_001"],"role":"panel_title|section_title|body_label|slot_caption|free_text|annotation|arrow_label|legend_label|method_label|modality_label|trait_label","align":"left|center|right","ignore":false,"confidence":0.0,"reason":"..."}} + ], + "ignored_ocr_ids": ["ref_text_ocr_999"], + "warnings": [] +}} + +Raw OCR regions: +{json.dumps(_raw_ocr_brief(raw_regions), ensure_ascii=False)} + +Heuristic groups: +{json.dumps(_heuristic_group_brief(heuristic_regions), ensure_ascii=False)} + +Objects: +{json.dumps(_object_brief(program), ensure_ascii=False)} +""".strip() + result = call_vlm_json(prompt, [reference_path], model=model_name) + result.setdefault("_vlm_model", model_name) + return result + + +def _coerce_align(value: object) -> str | None: + text = str(value or "").strip().lower() + if text in {"left", "center", "right"}: + return text + return None + + +def _apply_grouping_plan(raw_regions: list[dict], raw_plan: dict) -> tuple[list[dict], dict, list[str]]: + raw_by_id = {str(item.get("id") or ""): item for item in raw_regions} + used: set[str] = set() + ignored = {str(item) for item in raw_plan.get("ignored_ocr_ids", []) if str(item).strip()} + warnings: list[str] = [] + planned_regions: list[dict] = [] + normalized_groups = [] + for index, item in enumerate(raw_plan.get("groups", []) if isinstance(raw_plan.get("groups"), list) else [], start=1): + if not isinstance(item, dict): + continue + member_ids = [str(value) for value in item.get("ocr_member_ids", []) if str(value) in raw_by_id] + member_ids = [value for value in member_ids if value not in ignored and value not in used] + if not member_ids: + continue + if bool(item.get("ignore")): + ignored.update(member_ids) + used.update(member_ids) + continue + members = [raw_by_id[value] for value in member_ids] + if len(members) == 1: + region = dict(members[0]) + region.setdefault("ocr_member_ids", member_ids) + region.setdefault("ocr_member_count", 1) + region["grouping_source"] = "vlm_singleton" + else: + region = _make_group(str(item.get("group_id") or f"ref_text_vlm_group_{index:03d}"), members) + region["source"] = "reference_ocr_paragraph_group_vlm" + region["grouping_source"] = "vlm" + if str(item.get("role") or "").strip(): + region["role"] = str(item.get("role")) + region["ocr_role_guess"] = region.get("ocr_role_guess") or members[0].get("role") + align = _coerce_align(item.get("align")) + if align: + region["align"] = align + region["grouping_confidence"] = item.get("confidence") + region["grouping_reason"] = item.get("reason") + planned_regions.append(region) + used.update(member_ids) + normalized_groups.append({ + "group_id": region["id"], + "ocr_member_ids": member_ids, + "role": region.get("role"), + "align": region.get("align"), + "bbox_percent": region.get("bbox_percent"), + "confidence": item.get("confidence"), + "reason": item.get("reason"), + }) + + for raw in raw_regions: + rid = str(raw.get("id") or "") + if rid in used or rid in ignored: + continue + singleton = dict(raw) + singleton.setdefault("ocr_member_ids", [rid]) + singleton.setdefault("ocr_member_count", 1) + singleton.setdefault("grouping_source", "vlm_unmentioned_singleton") + planned_regions.append(singleton) + normalized_groups.append({ + "group_id": singleton["id"], + "ocr_member_ids": [rid], + "role": singleton.get("role"), + "align": singleton.get("align"), + "bbox_percent": singleton.get("bbox_percent"), + "confidence": None, + "reason": "raw OCR id was not mentioned by VLM plan", + }) + if not planned_regions and raw_regions: + warnings.append("vlm_grouping_plan_produced_no_regions") + planned_regions.sort(key=lambda item: _line_sort_key(item) if isinstance(item.get("bbox_percent"), dict) else (1.0, 1.0)) + plan = { + "summary": str(raw_plan.get("summary") or "VLM text grouping plan."), + "mode": "vlm", + "vlm_model": raw_plan.get("_vlm_model"), + "groups": normalized_groups, + "ignored_ocr_ids": sorted(ignored), + "warnings": list(raw_plan.get("warnings", [])) + warnings if isinstance(raw_plan.get("warnings", []), list) else warnings, + } + return planned_regions, plan, warnings + + +def group_text_regions_heuristic(regions: list[dict]) -> tuple[list[dict], dict, dict]: + eligible = [item for item in regions if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) and _eligible_for_heuristic_grouping(item)] + ineligible = [item for item in regions if item not in eligible] + grouped: list[list[dict]] = [] + for region in sorted(eligible, key=lambda item: (str(item.get("target_id") or ""), str(item.get("role") or ""), *_line_sort_key(item))): + if grouped and _same_paragraph(grouped[-1][-1], region): + grouped[-1].append(region) + else: + grouped.append([region]) + + result: list[dict] = [] + group_records: list[dict] = [] + for index, members in enumerate(grouped, start=1): + if len(members) == 1: + item = dict(members[0]) + item.setdefault("ocr_member_ids", [str(item.get("id") or "")]) + item.setdefault("ocr_member_count", 1) + item.setdefault("grouping_source", "heuristic_singleton") + result.append(item) + continue + group_id = f"ref_text_group_{index:03d}" + group = _make_group(group_id, members) + result.append(group) + group_records.append({ + "group_id": group_id, + "ocr_member_ids": group["ocr_member_ids"], + "target_id": group.get("target_id"), + "role": group.get("role"), + "bbox_percent": group.get("bbox_percent"), + "text": group.get("text"), + }) + for item in ineligible: + singleton = dict(item) + singleton.setdefault("ocr_member_ids", [str(singleton.get("id") or "")]) + singleton.setdefault("ocr_member_count", 1) + singleton.setdefault("grouping_source", "heuristic_ineligible_singleton") + result.append(singleton) + result.sort(key=lambda item: _line_sort_key(item) if isinstance(item.get("bbox_percent"), dict) else (1.0, 1.0)) + + plan = { + "summary": "Heuristic OCR grouping plan.", + "mode": "heuristic", + "groups": group_records, + "ignored_ocr_ids": [], + } + report = { + "summary": "Text grouping report.", + "mode": "heuristic", + "status": "pass", + "raw_region_count": len(regions), + "grouped_region_count": len(result), + "paragraph_group_count": len(group_records), + "single_region_count": len([item for item in result if int(item.get("ocr_member_count") or 1) == 1]), + "warnings": [], + } + return result, plan, report + + +def group_text_regions( + regions: list[dict], + mode: str = "heuristic", + adapter: Callable | None = None, + reference_path: str | Path | None = None, + program: dict | None = None, + model: str | None = None, +) -> tuple[list[dict], dict, dict]: + effective_mode = str(mode or "heuristic").lower() + if effective_mode == "off" or not regions: + plan = {"summary": "Text grouping skipped.", "mode": effective_mode, "groups": [], "ignored_ocr_ids": []} + report = { + "summary": "Text grouping report.", + "mode": effective_mode, + "status": "skipped" if effective_mode == "off" else "pass", + "raw_region_count": len(regions), + "grouped_region_count": len(regions), + "paragraph_group_count": 0, + "single_region_count": len(regions), + "warnings": [], + } + return regions, plan, report + heuristic_regions, heuristic_plan, heuristic_report = group_text_regions_heuristic(regions) + if effective_mode == "heuristic": + return heuristic_regions, heuristic_plan, heuristic_report + if effective_mode not in {"vlm", "hybrid"}: + raise ValueError(f"Unsupported text grouping mode: {mode}") + + try: + if adapter: + raw_plan = adapter(reference_path, regions, heuristic_regions, program or {}, model) + else: + if reference_path is None: + raise RuntimeError("reference_path is required for VLM text grouping") + raw_plan = _call_vlm_grouping(reference_path, regions, heuristic_regions, program or {}, model=model) + vlm_regions, plan, warnings = _apply_grouping_plan(regions, raw_plan if isinstance(raw_plan, dict) else {}) + if not vlm_regions and regions: + raise RuntimeError("VLM text grouping produced no usable regions") + report = { + "summary": "Text grouping report.", + "mode": effective_mode, + "effective_mode": "vlm", + "status": "pass", + "raw_region_count": len(regions), + "grouped_region_count": len(vlm_regions), + "paragraph_group_count": len([item for item in vlm_regions if int(item.get("ocr_member_count") or 1) > 1]), + "single_region_count": len([item for item in vlm_regions if int(item.get("ocr_member_count") or 1) == 1]), + "warnings": warnings, + "heuristic_group_count": heuristic_report.get("paragraph_group_count", 0), + "vlm_model": plan.get("vlm_model"), + } + plan["mode"] = effective_mode + return vlm_regions, plan, report + except Exception as exc: + if effective_mode == "vlm": + fallback_status = "fallback_to_heuristic" + else: + fallback_status = "fallback_to_heuristic" + heuristic_plan = dict(heuristic_plan) + heuristic_plan["mode"] = effective_mode + heuristic_plan["effective_mode"] = "heuristic" + heuristic_plan.setdefault("warnings", []) + heuristic_plan["warnings"].append(f"vlm_grouping_failed:{exc}") + heuristic_report = dict(heuristic_report) + heuristic_report["mode"] = effective_mode + heuristic_report["effective_mode"] = "heuristic" + heuristic_report["status"] = fallback_status + heuristic_report.setdefault("warnings", []) + heuristic_report["warnings"].append(f"vlm_grouping_failed:{exc}") + return heuristic_regions, heuristic_plan, heuristic_report + + +def write_text_grouping_artifacts(out_dir, raw_geometry: dict, plan: dict, report: dict) -> None: + write_json(out_dir / "reference_text_geometry_raw.json", raw_geometry) + write_json(out_dir / "text_grouping_plan.json", plan) + write_json(out_dir / "text_grouping_report.json", report) diff --git a/rfs/text_layer.py b/rfs/text_layer.py index d642238..8c99d70 100644 --- a/rfs/text_layer.py +++ b/rfs/text_layer.py @@ -7,6 +7,8 @@ from PIL import Image from .reference_text_extractor import extract_reference_text +from .text_grouping import group_text_regions, write_text_grouping_artifacts +from .text_layer_ownership import apply_text_layer_ownership, write_text_layer_ownership_artifacts from .utils import write_json @@ -336,6 +338,9 @@ def build_text_layer( ocr_engine: str = "paddle", ocr_lang: str = "en_ch", ocr_adapter: Callable[[str | Path, str], list[dict]] | None = None, + text_grouping_mode: str = "heuristic", + text_grouping_model: str | None = None, + text_grouping_adapter: Callable | None = None, ) -> dict: """Build reference-first editable text artifacts and attach them to the program.""" @@ -352,7 +357,27 @@ def build_text_layer( lang=ocr_lang, ocr_adapter=ocr_adapter, ) - regions = _merge_ocr_with_fallback(ocr_regions, fallback_regions) + raw_geometry = { + "summary": "Raw line-level OCR geometry before paragraph grouping.", + "reference_path": str(reference_path), + "detection_mode": "ocr" if ocr_regions else "ocr_empty_or_unavailable", + "ocr_engine": ocr_engine, + "ocr_lang": ocr_lang, + "text_region_count": len(ocr_regions), + "text_regions": ocr_regions, + } + grouped_ocr_regions, grouping_plan, grouping_report = group_text_regions( + ocr_regions, + mode=text_grouping_mode if ocr_regions else "off", + adapter=text_grouping_adapter, + reference_path=reference_path, + program=program, + model=text_grouping_model, + ) + write_text_grouping_artifacts(out, raw_geometry, grouping_plan, grouping_report) + regions = _merge_ocr_with_fallback(grouped_ocr_regions, fallback_regions) + regions, ownership_plan, ownership_report = apply_text_layer_ownership(regions, program, mode="heuristic") + write_text_layer_ownership_artifacts(out, ownership_plan, ownership_report) if not ocr_regions: ocr_report["text_region_count"] = len(regions) ocr_report.setdefault("warnings", []) @@ -360,6 +385,8 @@ def build_text_layer( items: list[dict] = [] for region in regions: + if region.get("layer_ownership") != "editable_text_layer": + continue font_size = float(region.get("font_size_pt") or _font_pt_from_reference_region(region, canvas_height_in)) token_id = _nearest_token_id(style, region["color_hex"]) item = { @@ -385,6 +412,7 @@ def build_text_layer( "ocr_confidence": region.get("confidence"), "editable_in": "pptx", "visible": True, + "layer_ownership": region.get("layer_ownership"), } items.append(item) @@ -395,6 +423,13 @@ def build_text_layer( "detection_mode": "ocr" if ocr_regions else "reference_geometry_and_local_color_sampling", "ocr_engine": ocr_engine, "ocr_lang": ocr_lang, + "reference_text_geometry_raw_path": "reference_text_geometry_raw.json", + "text_grouping_plan_path": "text_grouping_plan.json", + "text_grouping_report_path": "text_grouping_report.json", + "text_grouping_mode": text_grouping_mode, + "text_grouping_status": grouping_report.get("status"), + "text_layer_ownership_plan_path": "text_layer_ownership_plan.json", + "text_layer_ownership_report_path": "text_layer_ownership_report.json", "text_regions": regions, } text_program = { diff --git a/rfs/text_layer_ownership.py b/rfs/text_layer_ownership.py new file mode 100644 index 0000000..007a566 --- /dev/null +++ b/rfs/text_layer_ownership.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from typing import Callable + +from .utils import write_json + + +OWNERSHIP_VALUES = {"editable_text_layer", "raster_asset_layer", "decorative_asset_text", "ignore"} +CRITICAL_EDITABLE_ROLES = { + "panel_title", + "section_title", + "arrow_label", + "legend_label", + "method_label", + "modality_label", + "trait_label", + "slot_caption", + "title", + "subtitle", +} + + +def _contains_center(container: dict, item: dict) -> bool: + box = item.get("bbox_percent") + cbox = container.get("bbox_percent") + if not isinstance(box, dict) or not isinstance(cbox, dict): + return False + cx = float(box["x"]) + float(box["w"]) / 2 + cy = float(box["y"]) + float(box["h"]) / 2 + return ( + float(cbox["x"]) <= cx <= float(cbox["x"]) + float(cbox["w"]) + and float(cbox["y"]) <= cy <= float(cbox["y"]) + float(cbox["h"]) + ) + + +def _slot_ids_containing_text(region: dict, program: dict) -> list[str]: + return [ + str(slot.get("id") or "") + for slot in program.get("slots", []) or [] + if isinstance(slot, dict) and str(slot.get("id") or "").strip() and _contains_center(slot, region) + ] + + +def _font_size(region: dict) -> float: + try: + return float(region.get("font_size_pt") or region.get("raw_font_size_pt") or 0.0) + except Exception: + return 0.0 + + +def _decide_heuristic(region: dict, program: dict) -> tuple[str, str]: + text = str(region.get("text") or "").strip() + role = str(region.get("role") or "") + if not text: + return "ignore", "empty text" + if role in CRITICAL_EDITABLE_ROLES: + return "editable_text_layer", f"{role} is a critical editable text role" + if _font_size(region) < 4.0 or len(text) <= 1: + return "decorative_asset_text", "tiny or one-character OCR text is treated as decorative asset text" + containing_slots = _slot_ids_containing_text(region, program) + target_id = str(region.get("target_id") or "") + if target_id in containing_slots or (containing_slots and role in {"free_text", "body_label", "annotation"}): + return "raster_asset_layer", "OCR text is visually inside a slot asset and is not a critical label" + if role in {"body_label", "annotation", "free_text"}: + return "editable_text_layer", f"{role} remains editable because it is not owned by a slot asset" + return "editable_text_layer", "default editable text ownership" + + +def apply_text_layer_ownership( + regions: list[dict], + program: dict, + mode: str = "heuristic", + adapter: Callable | None = None, +) -> tuple[list[dict], dict, dict]: + effective_mode = str(mode or "heuristic").lower() + raw_overrides = {} + warnings = [] + if effective_mode in {"vlm", "hybrid"} and adapter: + try: + raw = adapter(regions, program) + records = raw.get("items", []) if isinstance(raw, dict) else raw + for item in records or []: + if not isinstance(item, dict): + continue + text_id = str(item.get("text_id") or item.get("id") or "") + ownership = str(item.get("layer_ownership") or item.get("ownership") or "") + if text_id and ownership in OWNERSHIP_VALUES: + raw_overrides[text_id] = item + except Exception as exc: + warnings.append(f"ownership_adapter_failed:{exc}") + effective_mode = "heuristic" + elif effective_mode in {"vlm", "hybrid"}: + warnings.append("ownership_vlm_unavailable_fallback_to_heuristic") + effective_mode = "heuristic" + + planned_regions = [] + items = [] + for region in regions: + region = dict(region) + ownership, reason = _decide_heuristic(region, program) + override = raw_overrides.get(str(region.get("id") or "")) + if override: + ownership = str(override.get("layer_ownership") or override.get("ownership") or ownership) + reason = str(override.get("reason") or reason) + if ownership not in OWNERSHIP_VALUES: + warnings.append(f"invalid_ownership:{region.get('id')}:{ownership}") + ownership = "editable_text_layer" + reason = "invalid ownership value fell back to editable text" + region["layer_ownership"] = ownership + region["layer_ownership_source"] = "vlm" if override else "heuristic" + region["layer_ownership_reason"] = reason + planned_regions.append(region) + items.append({ + "text_id": region.get("id"), + "text": region.get("text"), + "role": region.get("role"), + "target_id": region.get("target_id"), + "layer_ownership": ownership, + "source": region["layer_ownership_source"], + "reason": reason, + "included_in_text_program": ownership == "editable_text_layer", + }) + + plan = { + "summary": "Text layer ownership plan.", + "mode": mode, + "effective_mode": effective_mode, + "items": items, + "warnings": warnings, + } + report = { + "summary": "Text layer ownership report.", + "mode": mode, + "effective_mode": effective_mode, + "status": "pass" if not warnings else "warning", + "text_region_count": len(planned_regions), + "editable_text_count": sum(1 for item in items if item["layer_ownership"] == "editable_text_layer"), + "raster_asset_text_count": sum(1 for item in items if item["layer_ownership"] == "raster_asset_layer"), + "decorative_asset_text_count": sum(1 for item in items if item["layer_ownership"] == "decorative_asset_text"), + "ignored_text_count": sum(1 for item in items if item["layer_ownership"] == "ignore"), + "warnings": warnings, + "items": items, + } + return planned_regions, plan, report + + +def write_text_layer_ownership_artifacts(out_dir, plan: dict, report: dict) -> None: + write_json(out_dir / "text_layer_ownership_plan.json", plan) + write_json(out_dir / "text_layer_ownership_report.json", report) diff --git a/rfs/vlm_client.py b/rfs/vlm_client.py index b29f1f0..8cc1cf1 100644 --- a/rfs/vlm_client.py +++ b/rfs/vlm_client.py @@ -42,7 +42,31 @@ def resolve_vlm_model(*env_names: str, explicit_model: str | None = None) -> str return os.getenv("MODEL_VLM", "").strip() or DEFAULT_VLM_MODEL -def call_vlm_json(prompt: str, image_paths: list[str | Path], model: str | None = None, timeout: int = 180, retries: int = 1) -> dict: +def _error_category(exc: Exception) -> str: + name = type(exc).__name__.casefold() + message = str(exc).casefold() + if "timeout" in name or "timeout" in message: + return "timeout" + if "ssl" in name or "ssl" in message or "eof" in message: + return "tls" + if "json" in name or "json" in message: + return "invalid_json" + if "connection" in name or "connection" in message: + return "connection" + if "http" in name or "status" in message: + return "http" + return "provider" + + +def call_vlm_json( + prompt: str, + image_paths: list[str | Path], + model: str | None = None, + timeout: int = 180, + retries: int = 1, + call_metadata: dict[str, Any] | None = None, + deadline_at: float | None = None, +) -> dict: api_base = os.getenv("API_BASE", "").rstrip("/") api_key = os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY") if not api_base or not api_key: @@ -61,20 +85,64 @@ def call_vlm_json(prompt: str, image_paths: list[str | Path], model: str | None "messages": [{"role": "user", "content": content}], "temperature": 0.1, } + started = time.monotonic() last_error: Exception | None = None - for attempt in range(max(1, int(retries) + 1)): + failures: list[dict[str, Any]] = [] + attempts = max(1, int(retries) + 1) + if call_metadata is not None: + call_metadata.update({ + "attempts": 0, + "retries_allowed": max(0, int(retries)), + "retries_used": 0, + "success": False, + "elapsed_seconds": 0.0, + "failure_categories": [], + }) + attempts_made = 0 + for attempt in range(attempts): + request_timeout = max(0.1, float(timeout)) + if deadline_at is not None: + remaining = float(deadline_at) - time.monotonic() + if remaining <= 0.1: + last_error = TimeoutError("VLM deadline budget exhausted") + failures.append({"attempt": attempts_made, "category": "timeout"}) + break + request_timeout = min(request_timeout, remaining) + attempts_made += 1 + if call_metadata is not None: + call_metadata["attempts"] = attempts_made try: response = requests.post( f"{api_base}/chat/completions", headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, data=json.dumps(payload), - timeout=timeout, + timeout=request_timeout, ) response.raise_for_status() content_text = response.json()["choices"][0]["message"]["content"] - return extract_json(content_text) + result = extract_json(content_text) + if call_metadata is not None: + call_metadata.update({ + "success": True, + "retries_used": attempt, + "elapsed_seconds": round(time.monotonic() - started, 3), + "failure_categories": list(dict.fromkeys(item["category"] for item in failures)), + }) + return result except Exception as exc: last_error = exc - if attempt < max(1, int(retries) + 1) - 1: - time.sleep(1.5) + failures.append({"attempt": attempts_made, "category": _error_category(exc)}) + if attempt < attempts - 1: + delay = min(2.0, 0.75 * (2 ** attempt)) + if deadline_at is not None and float(deadline_at) - time.monotonic() <= delay + 0.1: + break + time.sleep(delay) + if call_metadata is not None: + call_metadata.update({ + "success": False, + "retries_used": max(0, attempts_made - 1), + "elapsed_seconds": round(time.monotonic() - started, 3), + "failure_categories": list(dict.fromkeys(item["category"] for item in failures)), + "deadline_reached": bool(deadline_at is not None and time.monotonic() >= float(deadline_at)), + }) raise last_error or RuntimeError("VLM call failed") diff --git a/rfs/workflow.py b/rfs/workflow.py index 7982b6a..4169a6f 100644 --- a/rfs/workflow.py +++ b/rfs/workflow.py @@ -1,6 +1,8 @@ from __future__ import annotations +import time from pathlib import Path +from typing import Any from .asset_generator import generate_assets from .asset_reviewer import review_assets @@ -10,7 +12,8 @@ from .input_loader import load_text from .paper_analyzer import analyze_paper from .presentations_qa import run_presentations_qa -from .ppt_compiler import compile_ppt +from .composition import compile_ppt +from .contracts import apply_paper_semantic_contract, compile_paper_brief_from_semantic_contract, load_paper_semantic_contract from .prompt_planner import plan_slot_prompts from .layout_locator import locate_layout from .program_builder import build_figure_program @@ -22,6 +25,21 @@ from .visual_critic import apply_layout_corrections, run_visual_critic +def _resolve_semantic_contract( + semantic_contract: dict[str, Any] | str | Path | None, + paper_contract_dir: str | Path | None, +) -> tuple[dict[str, Any] | None, str | None]: + if semantic_contract is not None and paper_contract_dir is not None: + raise ValueError("use either semantic_contract or paper_contract_dir, not both") + source = semantic_contract if semantic_contract is not None else paper_contract_dir + if source is None: + return None, None + if isinstance(source, dict): + return source, "inline" + path = Path(source) + return load_paper_semantic_contract(path), str(path.resolve()) + + def _write_alignment_review(out_dir: Path, paper_brief: dict, inventory: dict, layout_plan: dict, export_result: dict) -> None: text = "\n".join([ "# Summary", @@ -114,14 +132,35 @@ def make_framework( presentations_workspace: str | Path | None = None, presentations_scale: int = 2, export: bool = True, + semantic_contract: dict[str, Any] | str | Path | None = None, + paper_contract_dir: str | Path | None = None, ) -> dict: + started = time.perf_counter() + stage_started = started + stage_timings: dict[str, float] = {} + + def mark(stage: str) -> None: + nonlocal stage_started + now = time.perf_counter() + stage_timings[stage] = round(now - stage_started, 3) + stage_started = now + out_dir = ensure_dir(out) prompt_plan_workers = max(1, min(12, int(prompt_plan_workers))) + contract, contract_source = _resolve_semantic_contract(semantic_contract, paper_contract_dir) input_manifest = archive_inputs(paper, reference, out_dir) + input_manifest["semantic_contract_source"] = contract_source + input_manifest["paper_planner_bypassed"] = bool(contract) + write_json(out_dir / "input_manifest.json", input_manifest) archived_paper = input_manifest.get("paper_archived") or str(paper) archived_reference = input_manifest.get("reference_archived") or str(reference) - loaded = load_text(archived_paper) - paper_brief = analyze_paper(loaded, out_dir) + mark("archive_inputs") + if contract: + paper_brief = compile_paper_brief_from_semantic_contract(contract, out_dir, source_path=archived_paper) + else: + loaded = load_text(archived_paper) + paper_brief = analyze_paper(loaded, out_dir) + mark("paper_brief") inventory = analyze_reference( archived_reference, paper_brief, @@ -130,9 +169,14 @@ def make_framework( slot_source=slot_source, control_localizer_mode=control_localizer_mode, ) + mark("reference_analysis") style = build_style_sheet(paper_brief, inventory, out_dir) layout_plan = locate_layout(archived_reference, inventory, style, out_dir, mode=locator_mode, model=locator_model) + mark("style_and_layout") program = build_figure_program(paper_brief, inventory, style, out_dir, layout_plan=layout_plan) + semantic_binding_report = None + if contract: + program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_dir) program = style_and_route_arrows(program, out_dir, mode=arrow_style_mode) _slot_prompt_plan, program = plan_slot_prompts( archived_reference, @@ -147,6 +191,7 @@ def make_framework( complexity_profile=complexity_profile, ) program = style_and_route_arrows(program, out_dir, mode=arrow_style_mode) + mark("program_and_prompt_plan") program = build_text_layer( archived_reference, program, @@ -156,6 +201,10 @@ def make_framework( ocr_engine=ocr_engine, ocr_lang=ocr_lang, ) + if contract: + program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_dir) + program = style_and_route_arrows(program, out_dir, mode=arrow_style_mode) + mark("editable_text_and_connectors") generate_assets( program, style, @@ -166,10 +215,12 @@ def make_framework( asset_retries=asset_retries, ) asset_review = review_assets(out_dir, mode=asset_review_mode, model=critic_model) + mark("assets_and_review") pptx = compile_ppt(program, out_dir) export_result = {"status": "skipped", "pdf": None, "png": None} if export: export_result = export_outputs(pptx, out_dir) + mark("composition_and_export") visual_critic = run_visual_critic( out_dir=out_dir, @@ -190,6 +241,8 @@ def make_framework( break write_json(out_dir / "layout_plan.json", layout_plan) program = build_figure_program(paper_brief, inventory, style, out_dir, layout_plan=layout_plan) + if contract: + program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_dir) program = style_and_route_arrows(program, out_dir, mode=arrow_style_mode) program = build_text_layer( archived_reference, @@ -200,6 +253,9 @@ def make_framework( ocr_engine=ocr_engine, ocr_lang=ocr_lang, ) + if contract: + program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_dir) + program = style_and_route_arrows(program, out_dir, mode=arrow_style_mode) pptx = compile_ppt(program, out_dir) if export: export_result = export_outputs(pptx, out_dir) @@ -213,6 +269,7 @@ def make_framework( model=critic_model, iteration=iteration, ) + mark("visual_critic") _write_alignment_review(out_dir, paper_brief, inventory, layout_plan, export_result) write_text(out_dir / "critic_report.md", "# Summary\nCritic review placeholder created before final validation.\n") validation_for_critic = validate_output(out_dir) @@ -227,6 +284,18 @@ def make_framework( scale=presentations_scale, run_inspect=True, ) + mark("validation_and_qa") + run_report = { + "summary": "Timed make-framework execution report.", + "status": "completed" if validation.get("ok") else "completed_with_validation_errors", + "paper_planner": paper_brief.get("planner"), + "semantic_contract_source": contract_source, + "semantic_contract_active": bool(contract), + "semantic_binding_status": semantic_binding_report.get("status") if semantic_binding_report else None, + "stage_timings_seconds": stage_timings, + "total_seconds": round(time.perf_counter() - started, 3), + } + write_json(out_dir / "run_report.json", run_report) return { "summary": "ResearchFigureStudio make-framework run result.", "ok": validation.get("ok", False), @@ -253,5 +322,9 @@ def make_framework( "presentations_qa": presentations_report, "slot_count": inventory.get("slot_count"), "slot_source": slot_source, + "paper_planner": paper_brief.get("planner"), + "semantic_contract_source": contract_source, + "semantic_binding": semantic_binding_report, + "run_report": run_report, "validation": validation, } diff --git a/rfs/workflows/__init__.py b/rfs/workflows/__init__.py new file mode 100644 index 0000000..7318dfd --- /dev/null +++ b/rfs/workflows/__init__.py @@ -0,0 +1,5 @@ +"""Product-level workflow entry points.""" + +from .paper_to_editable import run_paper_to_editable + +__all__ = ["run_paper_to_editable"] diff --git a/rfs/workflows/paper_to_editable.py b/rfs/workflows/paper_to_editable.py new file mode 100644 index 0000000..3831341 --- /dev/null +++ b/rfs/workflows/paper_to_editable.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ..contracts import load_paper_semantic_contract +from ..editable_rebuild import rebuild_editable +from ..paper_to_image import run_paper_to_image +from ..rebuild_vlm_adapters import build_rebuild_vlm_adapters +from ..utils import ensure_dir, write_json +from ..workflow import make_framework + + +def run_paper_to_editable( + paper: str | Path, + out: str | Path, + preferences_path: str | Path | None = None, + positive_references: list[str] | None = None, + negative_references: list[str] | None = None, + planner_mode: str = "vlm", + planner_model: str | None = None, + image_asset_mode: str = "image2", + image_candidates: int = 3, + aspect_ratio: str | None = None, + language: str | None = None, + image_model: str | None = None, + image_retries: int = 2, + review_mode: str = "vlm", + review_model: str | None = None, + domain_profile: str = "auto", + template: str = "auto", + repair_rounds: int = 3, + rebuild_asset_mode: str = "api", + rebuild_asset_policy: str = "smart-api", + layout_mode: str = "hybrid", + control_mode: str = "hybrid", + text_mode: str = "ocr", + design_plan_mode: str = "vlm", + export_preview: bool = True, + allow_engineering_preview: bool = False, + ocr_engine: str = "auto", + ocr_lang: str = "en_ch", + editable_route: str = "rebuild", + framework_slot_count: int = 25, + framework_asset_mode: str = "image2", + framework_candidates_per_slot: int = 1, + framework_asset_workers: int = 3, + critic_adapter=None, +) -> dict[str, Any]: + root = ensure_dir(out).resolve() + image_dir = root / "paper_to_image" + editable_dir = root / "editable" + image_result = run_paper_to_image( + paper=paper, + out=image_dir, + preferences_path=preferences_path, + positive_references=positive_references, + negative_references=negative_references, + planner_mode=planner_mode, + planner_model=planner_model, + asset_mode=image_asset_mode, + candidates=image_candidates, + aspect_ratio=aspect_ratio, + language=language, + image_model=image_model, + image_retries=image_retries, + review_mode=review_mode, + review_model=review_model, + domain_profile=domain_profile, + template=template, + repair_rounds=repair_rounds, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + critic_adapter=critic_adapter, + ) + reference = image_result.get("selected_image") + reference_status = "production_selected_image" + if not reference and allow_engineering_preview: + reference = image_result.get("engineering_preview") + reference_status = "engineering_preview_explicitly_allowed" + if not reference: + blocked = { + "summary": "Paper-to-editable delivery blocked before rebuild because no production-approved image exists.", + "ok": False, + "delivery_blocked": True, + "out_dir": str(root), + "paper_to_image": image_result, + "policy": "failed image candidates are never promoted to production editable output", + } + write_json(root / "paper_to_editable_result.json", blocked) + return blocked + + contract = load_paper_semantic_contract(image_dir) + if editable_route == "framework": + framework_ocr_engine = ocr_engine if ocr_engine in {"rapidocr", "paddle", "easyocr", "off"} else "off" + framework_result = make_framework( + paper=paper, + reference=reference, + out=editable_dir, + slot_count=framework_slot_count, + slot_source="reference-primary", + asset_mode=framework_asset_mode, + candidates_per_slot=framework_candidates_per_slot, + asset_workers=framework_asset_workers, + asset_retries=image_retries, + asset_review_mode="vlm" if review_mode == "vlm" else "heuristic", + locator_mode="vlm" if layout_mode in {"vlm", "hybrid"} else "heuristic", + control_localizer_mode="hybrid" if control_mode in {"vlm", "hybrid"} else "heuristic", + arrow_style_mode="reference", + prompt_plan_mode="vlm" if planner_mode == "vlm" else "heuristic", + prompt_plan_model=planner_model, + prompt_plan_workers=max(1, min(12, framework_asset_workers)), + complexity_profile="reference-dense", + critic_mode="vlm" if review_mode == "vlm" else "heuristic", + critic_model=review_model, + text_extractor_mode="ocr" if text_mode == "ocr" else "heuristic", + ocr_engine=framework_ocr_engine, + ocr_lang=ocr_lang, + export=export_preview, + semantic_contract=contract, + ) + result = { + "summary": "Paper-to-editable workflow completed through the authoritative multi-asset framework compiler.", + "ok": bool(framework_result.get("ok")), + "delivery_blocked": False, + "editable_route": "framework", + "out_dir": str(root), + "reference_status": reference_status, + "reference_image": str(reference), + "paper_to_image_dir": str(image_dir), + "editable_dir": str(editable_dir), + "pptx": framework_result.get("pptx"), + "preview": framework_result.get("png"), + "semantic_binding_status": (framework_result.get("semantic_binding") or {}).get("status"), + "paper_to_image": image_result, + "framework_compiler": framework_result, + } + write_json(root / "paper_to_editable_result.json", result) + return result + if editable_route != "rebuild": + raise ValueError(f"Unsupported editable_route: {editable_route}") + + adapters = build_rebuild_vlm_adapters(editable_dir) + rebuild_ocr_engine = ocr_engine if ocr_engine in {"paddle", "easyocr", "off"} else "paddle" + editable_result = rebuild_editable( + reference=reference, + out=editable_dir, + asset_mode=rebuild_asset_mode, + asset_policy=rebuild_asset_policy, + text_mode=text_mode, + layout_mode=layout_mode, + control_mode=control_mode, + export_preview=export_preview, + ocr_engine=rebuild_ocr_engine, + ocr_lang=ocr_lang, + vlm_layout_adapter=adapters["layout"] if layout_mode in {"vlm", "hybrid"} else None, + control_adapter=adapters["control"] if control_mode in {"vlm", "hybrid"} else None, + semantic_adapter=adapters["semantic"] if layout_mode in {"vlm", "hybrid"} or control_mode in {"vlm", "hybrid"} else None, + design_plan_mode=design_plan_mode, + design_adapter=adapters["design"] if design_plan_mode == "vlm" else None, + semantic_contract=contract, + ) + result = { + "summary": "Paper-to-editable workflow completed with paper-grounded semantic bindings.", + "ok": bool(editable_result.get("ok")), + "delivery_blocked": False, + "editable_route": "rebuild", + "out_dir": str(root), + "reference_status": reference_status, + "reference_image": str(reference), + "paper_to_image_dir": str(image_dir), + "editable_dir": str(editable_dir), + "pptx": editable_result.get("pptx"), + "preview": editable_result.get("preview"), + "semantic_binding_status": editable_result.get("semantic_binding_status"), + "paper_to_image": image_result, + "editable_rebuild": editable_result, + } + write_json(root / "paper_to_editable_result.json", result) + return result diff --git a/skills/research-figure-studio/SKILL.md b/skills/research-figure-studio/SKILL.md new file mode 100644 index 0000000..8ad9cba --- /dev/null +++ b/skills/research-figure-studio/SKILL.md @@ -0,0 +1,107 @@ +--- +name: research-figure-studio +description: Create paper-grounded, editable PowerPoint scientific framework figures from papers and optional visual references. Use for paper-to-PPT figures, editable architecture diagrams, method overviews, system pipelines, or image-to-editable-PPT reconstruction. +--- + +# Research Figure Studio + +## Summary + +Create scientifically faithful figures whose labels, entities, and relations come from the paper, while generated images or user references provide layout and visual style. The deliverable is an editable PPTX, not a flattened diagram image. + +## Preferred route + +Use the installed `rfs` command from any working directory. Do not assume a repository location or embed machine-specific paths. + +For a complete paper-to-editable workflow: + +```powershell +rfs paper-to-editable --paper --out --positive-reference --json +``` + +For image-only reconstruction: + +```powershell +rfs rebuild-editable --reference --out --asset-policy smart-api --json +``` + +Use `--image-asset-mode placeholder --rebuild-asset-mode placeholder --allow-engineering-preview` only for explicit offline engineering validation. Never present that result as production-approved. + +For paper understanding or prompt-only work, start with: + +```powershell +rfs doctor --json +rfs inspect-pdf --paper --out --json +rfs fast-framework-prompt --paper --out --deadline 180 --json +``` + +`inspect-pdf` and the fast path share a global document cache. For scanned PDFs, `auto` prefers RapidOCR when installed. Do not enable automatic EasyOCR model downloads implicitly; use `RFS_OCR_ALLOW_DOWNLOAD=1` only when the user explicitly wants the one-time download. + +## Required semantic policy + +- Read and structure the paper before generating the visual reference. +- Treat `paper_review.json` and `figure_specification.json` as scientific ground truth. +- Treat the generated/reference image as layout and style evidence only. +- Keep exact paper labels, relations, formulas, numbers, and citations out of raster assets whenever possible. +- Bind paper entities and relations into editable PPT text and connector objects through `paper_semantic_contract.json` and `semantic_binding_report.json`. +- Never reconstruct scientific truth by OCR alone when paper contracts are available. +- Never promote a failed candidate or engineering preview to production output automatically. + +## Workflow + +1. Archive the paper and references. +2. Build the evidence map and structured paper review. +3. Validate evidence grounding, including relations whose endpoints may be inputs, outputs, concepts, research objects, modules, or innovations. +4. Plan the figure and generate candidate reference images. +5. Stop if no production candidate passes all gates. +6. Analyze the selected image globally before local extraction. +7. Rebuild panels, cards, assets, editable labels, and editable connectors. +8. Apply the paper semantic contract so exact labels and scientific relations override image/OCR guesses. +9. Generate deterministic visual QA and a fallback preview. +10. Validate the PPTX and report warnings or blockers. + +## References + +Read only what the task needs: + +- Paper analysis: `references/paper_to_figure.md` +- Framework figures: `references/visual_framework_figures.md` +- User-provided image alignment: `references/reference_image_alignment.md` +- Structured figure program: `references/figure_program.md` +- Image block prompting: `references/image_block_prompting.md` +- Critic and delivery gates: `references/critic_stage.md` and `references/journal_quality_checklist.md` +- Data plots: `references/data_figures.md` + +## Delivery checklist + +The normal paper-to-editable output should contain: + +- `paper_to_image/paper_review.json` +- `paper_to_image/figure_specification.json` +- `paper_to_image/planning_validation_report.json` +- `editable/paper_semantic_contract.json` +- `editable/semantic_binding_report.json` +- `editable/figure_program.json` +- `editable/text_program.json` +- `editable/rebuild_visual_quality_report.json` +- `editable/editable_composition.pptx` +- `editable/rebuild_preview.png` when preview export is requested + +If production gates fail, return the engineering artifacts and the blocker; do not claim that the figure is ready for paper submission. + +## Benchmark commands + +Use the two independent suites to locate failures before changing the workflow: + +```powershell +rfs benchmark list --root benchmarks --json +rfs benchmark fetch --case --json +rfs benchmark fast --case --out --deadline 180 --json +rfs benchmark fast-suite --root benchmarks --out --planner-mode heuristic --json +rfs benchmark run --case --out --json +rfs benchmark score --case --run --json +``` + +`paper-to-image` scientific and terminology errors are hard failures. `image-to-ppt` must reject full-slide reference images even when raster similarity is high. + +For prompt-only work, inspect `extraction_report.json`, `contract_completion_report.json`, and `planning_validation_report.json` in that order. Treat `engineering_only: true` as non-production even when deterministic topology coverage is high. Use `fast-suite` after parser or contract changes and reject regressions in forbidden content, evidence grounding, provider reliability, or stage timings. diff --git a/skills/research-figure-studio/agents/openai.yaml b/skills/research-figure-studio/agents/openai.yaml new file mode 100644 index 0000000..891e199 --- /dev/null +++ b/skills/research-figure-studio/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "Research Figure Studio" + short_description: "Create paper-grounded scientific figures as editable PowerPoint diagrams." + default_prompt: "Use $research-figure-studio to turn my paper into an editable PowerPoint framework figure." + +policy: + allow_implicit_invocation: true diff --git a/codex-skills/research-figure-making/references/critic_stage.md b/skills/research-figure-studio/references/critic_stage.md similarity index 100% rename from codex-skills/research-figure-making/references/critic_stage.md rename to skills/research-figure-studio/references/critic_stage.md diff --git a/codex-skills/research-figure-making/references/data_figures.md b/skills/research-figure-studio/references/data_figures.md similarity index 100% rename from codex-skills/research-figure-making/references/data_figures.md rename to skills/research-figure-studio/references/data_figures.md diff --git a/codex-skills/research-figure-making/references/enhancement_pipeline.md b/skills/research-figure-studio/references/enhancement_pipeline.md similarity index 100% rename from codex-skills/research-figure-making/references/enhancement_pipeline.md rename to skills/research-figure-studio/references/enhancement_pipeline.md diff --git a/codex-skills/research-figure-making/references/figure_program.md b/skills/research-figure-studio/references/figure_program.md similarity index 100% rename from codex-skills/research-figure-making/references/figure_program.md rename to skills/research-figure-studio/references/figure_program.md diff --git a/codex-skills/research-figure-making/references/image_block_prompting.md b/skills/research-figure-studio/references/image_block_prompting.md similarity index 100% rename from codex-skills/research-figure-making/references/image_block_prompting.md rename to skills/research-figure-studio/references/image_block_prompting.md diff --git a/codex-skills/research-figure-making/references/journal_quality_checklist.md b/skills/research-figure-studio/references/journal_quality_checklist.md similarity index 100% rename from codex-skills/research-figure-making/references/journal_quality_checklist.md rename to skills/research-figure-studio/references/journal_quality_checklist.md diff --git a/codex-skills/research-figure-making/references/paper_to_figure.md b/skills/research-figure-studio/references/paper_to_figure.md similarity index 100% rename from codex-skills/research-figure-making/references/paper_to_figure.md rename to skills/research-figure-studio/references/paper_to_figure.md diff --git a/codex-skills/research-figure-making/references/reference_image_alignment.md b/skills/research-figure-studio/references/reference_image_alignment.md similarity index 100% rename from codex-skills/research-figure-making/references/reference_image_alignment.md rename to skills/research-figure-studio/references/reference_image_alignment.md diff --git a/codex-skills/research-figure-making/references/stylist_stage.md b/skills/research-figure-studio/references/stylist_stage.md similarity index 100% rename from codex-skills/research-figure-making/references/stylist_stage.md rename to skills/research-figure-studio/references/stylist_stage.md diff --git a/codex-skills/research-figure-making/references/visual_framework_figures.md b/skills/research-figure-studio/references/visual_framework_figures.md similarity index 100% rename from codex-skills/research-figure-making/references/visual_framework_figures.md rename to skills/research-figure-studio/references/visual_framework_figures.md diff --git a/codex-skills/research-figure-making/scripts/estimate_asset_fill.py b/skills/research-figure-studio/scripts/estimate_asset_fill.py similarity index 100% rename from codex-skills/research-figure-making/scripts/estimate_asset_fill.py rename to skills/research-figure-studio/scripts/estimate_asset_fill.py diff --git a/codex-skills/research-figure-making/scripts/validate_framework_outputs.py b/skills/research-figure-studio/scripts/validate_framework_outputs.py similarity index 100% rename from codex-skills/research-figure-making/scripts/validate_framework_outputs.py rename to skills/research-figure-studio/scripts/validate_framework_outputs.py diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..fcc8285 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,11 @@ +# Test layers + +- `unit/`: deterministic functions and local file/PPTX behavior. +- `integration/`: multi-component workflows using mocks, placeholders, or temporary files. +- `e2e/`: real paper-to-editable acceptance cases. This layer is intentionally empty until benchmark fixtures and production credentials are defined. + +Run all local tests with: + +```powershell +python -m unittest discover -s tests -q +``` diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..1d1083a --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""ResearchFigureStudio test suite.""" diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000..1142560 --- /dev/null +++ b/tests/e2e/__init__.py @@ -0,0 +1 @@ +"""Real paper-to-editable acceptance tests and benchmark-backed smoke tests.""" diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..93b8465 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Tests spanning multiple ResearchFigureStudio engine components.""" diff --git a/tests/test_coevolution.py b/tests/integration/test_coevolution.py similarity index 100% rename from tests/test_coevolution.py rename to tests/integration/test_coevolution.py diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py new file mode 100644 index 0000000..09191fd --- /dev/null +++ b/tests/integration/test_paper_to_image.py @@ -0,0 +1,1669 @@ +from __future__ import annotations + +import base64 +import io +import json +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch + +import fitz +import requests +from PIL import Image, ImageChops, ImageDraw + +from rfs.cli import _doctor, _probe_executable, build_parser +from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image +from rfs.paper_to_image.critics import required_labels, review_candidate +from rfs.paper_to_image.generator import generate_and_select, native_image2_aspect_ratio +from rfs.paper_to_image.overlay_renderer import render_visual_substrate_blueprint +from rfs.paper_to_image.planner import collect_visible_labels, collect_visual_relations, collect_visual_roles, compile_image_prompt +from rfs.paper_to_image.review import detect_domain_profile, validate_review_coverage +from rfs.paper_to_image.semantic_blueprint import compile_semantic_blueprint +from rfs.paper_to_image.templates import build_template_profiles, render_layout_blueprint, select_template + + +PAPER_TEXT = """ +Evidence-Grounded Modular Reasoning for Scientific Documents + +Abstract +We introduce ModularTrace, a method that converts scientific documents into an evidence graph and then performs constrained reasoning over the graph. The method reduces unsupported conclusions while preserving exact paper terminology. + +1 Introduction +Scientific document assistants often generate claims without traceable evidence. ModularTrace addresses this problem by linking every extracted fact to a source passage. + +2 Method +The Document Parser splits the input paper into page-aware passages. The Evidence Graph Builder extracts entities and directed relations. The Constrained Reasoner consumes the graph and produces an answer with evidence identifiers. During training, a consistency loss penalizes unsupported relations. During inference, only retrieved evidence nodes may support an answer. + +3 Experiments +We evaluate on three scientific question-answering datasets using answer accuracy and evidence precision. + +4 Conclusion +ModularTrace improves evidence precision while maintaining answer accuracy. +""".strip() + + +def _png_b64(size=(1536, 1024), color=(235, 242, 247)) -> str: + buffer = io.BytesIO() + Image.new("RGB", size, color).save(buffer, format="PNG") + return base64.b64encode(buffer.getvalue()).decode("ascii") + + +class FakeResponse: + status_code = 200 + + def raise_for_status(self): + return None + + def json(self): + return {"data": [{"b64_json": _png_b64()}]} + + +def _passing_critic(_path, _blueprint, _prompt): + return { + "summary": "Mock passing critic.", + "ocr": {"detected_labels": ["Document Parser", "Evidence Graph Builder"], "missing_labels": [], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "score": 1.0, "passed": True}, + "scientific": {"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "innovation_visible": True, "score": 1.0, "passed": True}, + "template": {"macro_panel_match": 0.9, "reading_order_match": 0.9, "connector_rhythm_match": 0.9, "visual_density_match": 0.9, "copied_reference_content": [], "score": 0.9, "passed": True}, + "aesthetic": {"hierarchy": 0.9, "balance": 0.9, "whitespace": 0.9, "color": 0.9, "icon_consistency": 0.9, "readability": 0.9, "score": 0.9, "passed": True}, + "preserve": ["macro layout"], + "repair": [], + "remove": [], + "repair_regions": [], + "hard_errors": [], + } + + +def _plan() -> dict: + modules = [ + {"id": "parser", "name": "Document Parser", "role": "module", "evidence_ids": ["E0001"]}, + {"id": "graph", "name": "Evidence Graph Builder", "role": "module", "evidence_ids": ["E0001"]}, + ] + return { + "paper_summary": {"summary": "Paper summary.", "title": "ModularTrace"}, + "figure_specification": {"summary": "Scientific contract.", "figure_goal": "Show the method.", "modules": modules, "relations": [{"source": "parser", "target": "graph", "type": "data_flow", "evidence_ids": ["E0001"]}], "terminology": {"Document Parser": "Document Parser", "Evidence Graph Builder": "Evidence Graph Builder"}, "forbidden_inventions": []}, + "design_plan": {"summary": "Design.", "reading_order": ["parser", "graph"]}, + "layout_intent": {"summary": "Layout.", "pattern": "left_to_right"}, + "visual_metaphors": {"summary": "Metaphors.", "items": []}, + "style_plan": {"summary": "Style.", "medium": "academic"}, + } + + +class PaperToImageTests(unittest.TestCase): + def test_offline_workflow_is_engineering_only(self): + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_KEY": "known-test-secret"}, clear=False): + root = Path(temp) + paper = root / "paper.md" + paper.write_text(PAPER_TEXT, encoding="utf-8") + out = root / "run" + result = run_paper_to_image(paper=paper, out=out, planner_mode="heuristic", asset_mode="placeholder", candidates=2, aspect_ratio="auto", review_mode="heuristic", ocr_engine="off") + + self.assertTrue(result["ok"]) + self.assertFalse(result["pptx_generated"]) + self.assertEqual(result["candidate_count"], 2) + self.assertIsNone(result["selected_image"]) + self.assertTrue((out / "engineering_preview.png").exists()) + self.assertFalse((out / "selected_image.png").exists()) + for name in ["document_index.json", "paper_review.json", "review_coverage_report.json", "domain_profile.json", "selected_template.json", "layout_blueprint.png", "image2_request_manifest.json", "ocr_review.json", "template_alignment_report.json", "scientific_critic_report.json", "aesthetic_critic_report.json"]: + self.assertTrue((out / name).exists(), name) + document_index = json.loads((out / "document_index.json").read_text(encoding="utf-8")) + self.assertTrue(document_index["sections"]) + manifest = json.loads((out / "image2_request_manifest.json").read_text(encoding="utf-8")) + self.assertEqual(manifest["requests"][0]["model"], "offline-placeholder") + self.assertIn("api_key_present", manifest) + self.assertNotIn("known-test-secret", json.dumps(manifest)) + + def test_cli_production_defaults(self): + parser = build_parser() + args = parser.parse_args(["paper-to-image", "--paper", "paper.pdf", "--out", "output/run"]) + self.assertEqual(args.command, "paper-to-image") + self.assertEqual(args.asset_mode, "image2") + self.assertEqual(args.candidates, 3) + self.assertEqual(args.review_mode, "vlm") + self.assertEqual(args.aspect_ratio, "auto") + self.assertEqual(args.domain_profile, "auto") + self.assertEqual(args.template, "auto") + self.assertEqual(args.repair_rounds, 3) + self.assertFalse(args.editable_overlay_ppt) + + def test_offline_workflow_can_compile_raw_substrate_to_editable_ppt_overlay(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + paper = root / "paper.md" + paper.write_text(PAPER_TEXT, encoding="utf-8") + out = root / "run" + + result = run_paper_to_image( + paper=paper, + out=out, + planner_mode="heuristic", + asset_mode="placeholder", + candidates=1, + aspect_ratio="3:2", + review_mode="heuristic", + repair_rounds=0, + ocr_engine="off", + editable_overlay_ppt=True, + ) + + self.assertTrue(result["pptx_generated"]) + self.assertTrue(result["editable_overlay_ppt"]) + self.assertTrue((out / "editable_composition.pptx").exists()) + self.assertTrue((out / "editable_overlay_report.json").exists()) + with zipfile.ZipFile(out / "editable_composition.pptx") as archive: + slide_xml = archive.read("ppt/slides/slide1.xml").decode("utf-8") + self.assertIn("RFS Image2 Visual Substrate", slide_xml) + self.assertIn("RFS Semantic Label", slide_xml) + self.assertIn("RFS Connector", slide_xml) + + def test_image2_native_canvas_ratio_normalization(self): + self.assertEqual(native_image2_aspect_ratio("16:9"), "3:2") + self.assertEqual(native_image2_aspect_ratio("9:16"), "2:3") + self.assertEqual(native_image2_aspect_ratio("1:1"), "1:1") + + def test_prompt_and_critic_share_complete_visible_label_contract(self): + plan = _plan() + spec = plan["figure_specification"] + spec.update({ + "required_labels": ["Input Matrix", "Document Parser", "Final Answer"], + "inputs": [{"id": "input", "name": "Input Matrix"}], + "outputs": [{"id": "answer", "name": "Final Answer"}], + "innovations": [{"id": "grounding", "name": "Evidence Grounding"}], + }) + + labels = required_labels(plan) + prompt = compile_image_prompt(plan, {"aspect_ratio": "16:9", "language": "English"}) + + self.assertEqual(labels[:3], ["Input Matrix", "Document Parser", "Final Answer"]) + for label in ("Input Matrix", "Document Parser", "Final Answer"): + self.assertIn(label, labels) + self.assertIn(label, prompt) + self.assertNotIn("Evidence Grounding", labels) + self.assertIn("Evidence Grounding", prompt) + self.assertIn("never replace an output label with an icon-only node", prompt) + + def test_evidence_required_innovation_label_is_added_to_visible_contract(self): + plan = _plan() + plan["figure_specification"]["innovations"] = [{ + "id": "innovation_masked_autoencoder", + "name": "masked autoencoder", + "visible_label_required": True, + "must_appear_in_figure": True, + }] + + labels = required_labels(plan) + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + + self.assertIn("masked autoencoder", labels) + self.assertIn("masked autoencoder", prompt) + + def test_semantic_generation_prompt_stays_compact_for_nine_node_figure(self): + plan = _plan() + spec = { + "topology": "linear", + "figure_goal": "Show source, modalities, training stages, and evaluation outputs.", + "central_claim": {"text": "A named foundation model is pretrained then fine-tuned."}, + "inputs": [ + {"id": "source", "name": "Dataset + public data", "role": "data_source"}, + {"id": "image_a", "name": "Image A", "role": "input modality"}, + {"id": "image_b", "name": "Image B", "role": "input modality"}, + ], + "modules": [ + {"id": "ssl", "name": "SSL", "role": "training_objective"}, + {"id": "model", "name": "Named Model", "role": "module"}, + {"id": "tune", "name": "Supervised fine-tuning", "role": "training_objective"}, + ], + "outputs": [ + {"id": "diagnosis", "name": "Diagnosis", "role": "output"}, + {"id": "internal", "name": "internal evaluation", "role": "evaluation output"}, + {"id": "external", "name": "external evaluation", "role": "evaluation output"}, + ], + "innovations": [{"id": "masked", "name": "masked autoencoder", "role": "technique used by SSL", "visible_label_required": True}], + "required_labels": ["Dataset + public data", "Image A", "Image B", "SSL", "Named Model", "Supervised fine-tuning", "Diagnosis", "internal evaluation", "external evaluation"], + "relations": [ + {"source": "source", "target": "image_a", "type": "data_source"}, + {"source": "source", "target": "image_b", "type": "data_source"}, + {"source": "image_a", "target": "ssl", "type": "data_flow"}, + {"source": "image_b", "target": "ssl", "type": "data_flow"}, + {"source": "ssl", "target": "model", "type": "encoding"}, + {"source": "model", "target": "tune", "type": "conditioning"}, + {"source": "tune", "target": "diagnosis", "type": "prediction"}, + {"source": "tune", "target": "internal", "type": "evaluation"}, + {"source": "tune", "target": "external", "type": "evaluation"}, + ], + "forbidden_inventions": ["Do not add unsupported layers."], + } + plan["figure_specification"] = spec + semantic_plan = compile_semantic_blueprint(spec) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}, selected_template={"template_id": "linear", "semantic_plan": semantic_plan}) + + self.assertLess(len(prompt), 14000) + self.assertIn("masked autoencoder", prompt) + self.assertIn('"route_style":"straight"', prompt) + + def test_visual_relation_checklist_deduplicates_edges_and_skips_shared_model_wiring(self): + spec = { + "inputs": [{"id": "input", "name": "Input"}], + "modules": [ + {"id": "model", "name": "Model M"}, + {"id": "generate", "name": "Generate"}, + {"id": "initial", "name": "Initial Output"}, + {"id": "feedback", "name": "Feedback"}, + {"id": "refine", "name": "Refine"}, + ], + "outputs": [{"id": "refined", "name": "Refined Output"}], + "innovations": [], + "repeatable_labels": ["Model M"], + "relations": [ + {"source": "input", "target": "model", "type": "data_flow"}, + {"source": "model", "target": "initial", "type": "prediction"}, + {"source": "input", "target": "generate", "type": "generation_input"}, + {"source": "input", "target": "generate", "type": "data_flow"}, + {"source": "generate", "target": "initial", "type": "data_flow"}, + {"source": "initial", "target": "feedback", "type": "data_flow"}, + {"source": "initial", "target": "feedback", "type": "evaluation"}, + {"source": "initial", "target": "refine", "type": "revision_input"}, + {"source": "feedback", "target": "refine", "type": "feedback"}, + {"source": "refine", "target": "refined", "type": "prediction"}, + {"source": "refine", "target": "refined", "type": "data_flow"}, + {"source": "refined", "target": "feedback", "type": "feedback_loop"}, + ], + } + + relations = collect_visual_relations(spec) + pairs = {(item["source"], item["target"]): item["type"] for item in relations} + + self.assertNotIn(("input", "model"), pairs) + self.assertNotIn(("model", "initial"), pairs) + self.assertEqual(pairs[("input", "generate")], "data_flow") + self.assertEqual(pairs[("initial", "feedback")], "evaluation") + self.assertEqual(pairs[("refine", "refined")], "data_flow") + self.assertEqual(pairs[("refined", "feedback")], "feedback_loop") + + def test_candidate_review_accepts_requested_ratio_when_image2_ignores_blueprint_ratio(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1600, 900), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + + review = review_candidate( + candidate, + blueprint, + _plan(), + {"template_id": "linear", "forbidden_copy_terms": []}, + mode="vlm", + ocr_engine="off", + critic_adapter=_passing_critic, + acceptable_aspect_ratios=["16:9", "3:2"], + ) + + self.assertTrue(review["basic"]["passed"]) + self.assertEqual(review["basic"]["matched_aspect_ratio"], "16:9") + + def test_image2_candidate_must_visually_enrich_the_bare_blueprint(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + + review = review_candidate( + candidate, + blueprint, + _plan(), + {"template_id": "linear", "forbidden_copy_terms": []}, + mode="vlm", + ocr_engine="off", + critic_adapter=_passing_critic, + require_visual_enrichment=True, + ) + + self.assertFalse(review["basic"]["visual_enrichment_passed"]) + self.assertFalse(review["production_pass"]) + self.assertTrue(any("too close to the bare layout blueprint" in value for value in review["hard_errors"])) + self.assertTrue(any("Enrich node interiors" in value for value in review["repair"])) + + def test_visual_enrichment_is_measured_inside_the_blueprint_active_region(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + base = Image.new("RGB", (1000, 1000), "white") + draw = ImageDraw.Draw(base) + draw.rectangle((100, 100, 900, 900), outline="#1F4E79", width=4) + base.save(blueprint) + enriched = base.copy() + ImageDraw.Draw(enriched).rectangle((350, 350, 649, 549), fill="#3A9D9A") + enriched.save(candidate) + + review = review_candidate( + candidate, + blueprint, + _plan(), + {"template_id": "linear", "forbidden_copy_terms": []}, + mode="vlm", + ocr_engine="off", + critic_adapter=_passing_critic, + require_visual_enrichment=True, + ) + + self.assertLess(review["basic"]["blueprint_global_enrichment_ratio"], 0.08) + self.assertGreater(review["basic"]["blueprint_active_enrichment_ratio"], 0.08) + self.assertTrue(review["basic"]["visual_enrichment_passed"]) + + def test_candidate_generation_reviews_deterministically_composed_overlay(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + plan = _plan() + specification = plan["figure_specification"] + specification["topology"] = "linear" + specification["required_labels"] = ["Document Parser", "Evidence Graph Builder"] + semantic = compile_semantic_blueprint(specification) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": [], "semantic_plan": semantic} + review_blueprint = root / "layout_blueprint.png" + visual_blueprint = root / "visual_substrate_blueprint.png" + render_layout_blueprint(template, review_blueprint, "3:2", figure_specification=specification) + render_visual_substrate_blueprint(semantic, visual_blueprint, width=1536, height=1024) + overlay = { + "summary": "Exact overlay.", + "labels": [ + {"target_id": "parser", "text": "Document Parser", "role": "module"}, + {"target_id": "graph", "text": "Evidence Graph Builder", "role": "module"}, + ], + "connectors": [{"source": "parser", "target": "graph", "type": "data_flow", "label": ""}], + "groups": [], "reading_order": ["parser", "graph"], + } + + result = generate_and_select( + plan, + {"aspect_ratio": "3:2", "language": "English"}, + template, + review_blueprint, + root / "run", + asset_mode="placeholder", + candidates=1, + review_mode="vlm", + repair_rounds=0, + ocr_engine="off", + critic_adapter=_passing_critic, + require_visual_enrichment=False, + generation_blueprint_path=visual_blueprint, + overlay_spec=overlay, + deterministic_overlay=True, + ) + + candidate = result["candidates"][0] + self.assertTrue(result["deterministic_overlay"]) + self.assertTrue(Path(candidate["raw_path"]).exists()) + self.assertTrue(Path(candidate["visual_substrate_path"]).exists()) + self.assertTrue(Path(candidate["path"]).exists()) + self.assertTrue(candidate["geometry_alignment_report"]["passed"]) + self.assertTrue(candidate["review"]["substrate_geometry"]["passed"]) + self.assertTrue(candidate["overlay_report"]["passed"]) + self.assertEqual(candidate["overlay_report"]["label_count"], 2) + self.assertEqual(candidate["overlay_report"]["connector_count"], 1) + prompt = Path(candidate["prompt_path"]).read_text(encoding="utf-8") + self.assertIn("do not render any arrow, line, connector, arrowhead", prompt) + with Image.open(candidate["raw_path"]) as raw, Image.open(candidate["path"]) as composed: + self.assertIsNotNone(ImageChops.difference(raw.convert("RGB"), composed.convert("RGB")).getbbox()) + + def test_candidate_review_allows_evidence_supported_shared_label_reuse(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"]["repeatable_labels"] = ["Document Parser"] + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"].update({"duplicate_labels": ["Document Parser"], "passed": False}) + result["hard_errors"] = ["Duplicate label: Document Parser", "Template mismatch: too many content nodes"] + return result + + review = review_candidate(candidate, blueprint, plan, {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertTrue(review["production_pass"]) + self.assertEqual(review["ocr"]["allowed_duplicate_labels"], ["Document Parser"]) + self.assertEqual(review["ocr"]["duplicate_labels"], []) + + def test_candidate_review_records_but_allows_paper_supported_labels_outside_required_list(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"]["detected_labels"].append("Contrastive Loss") + return result + + review = review_candidate(candidate, blueprint, _plan(), {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertTrue(review["production_pass"]) + self.assertEqual(review["ocr"]["unexpected_labels"], ["Contrastive Loss"]) + self.assertEqual(review["ocr"]["unsupported_unexpected_labels"], []) + self.assertTrue(review["scientific"]["passed"]) + self.assertNotIn("Contrastive Loss", review["remove"]) + + def test_candidate_review_rejects_extra_label_when_evidence_review_marks_it_unsupported(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"]["detected_labels"].append("External Memory Bank") + result["scientific"]["invented_items"] = ["External Memory Bank"] + result["paper_alignment"] = { + "priorities": [], + "missing_priorities": [], + "evidence_conflicts": [], + "unsupported_visual_claims": ["External Memory Bank is not supported by the paper"], + "repair": ["Remove External Memory Bank"], + "score": 0.8, + "passed": False, + } + result["hard_errors"] = ["Unexpected visible label: External Memory Bank"] + return result + + review = review_candidate(candidate, blueprint, _plan(), {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertFalse(review["production_pass"]) + self.assertEqual(review["ocr"]["unsupported_unexpected_labels"], ["External Memory Bank"]) + self.assertIn("External Memory Bank", review["remove"]) + + def test_candidate_review_allows_duplicate_nonrequired_detail_label(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"]["detected_labels"].append("Add & Norm") + result["ocr"]["duplicate_labels"] = ["Add & Norm"] + result["hard_errors"] = ["Duplicate label: Add & Norm"] + return result + + review = review_candidate(candidate, blueprint, _plan(), {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertTrue(review["production_pass"]) + self.assertEqual(review["ocr"]["noncritical_duplicate_labels"], ["Add & Norm"]) + self.assertEqual(review["ocr"]["duplicate_labels"], []) + + def test_explicit_required_labels_are_the_only_forced_visible_labels(self): + specification = { + "required_labels": ["Image Patches", "Transformer Encoder"], + "modules": [{"name": "MLP Head"}, {"name": "Classifier"}], + "terminology": {"patches": "Patches"}, + "panel_graphs": [{"id": "a", "nodes": [{"name": "Add & Norm"}]}], + } + + self.assertEqual(collect_visible_labels(specification), ["Image Patches", "Transformer Encoder"]) + + def test_candidate_review_allows_evidence_backed_optional_panel_and_symbol_labels(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"]["optional_visible_labels"] = ["a", "b", "c", "[CLS]"] + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"]["detected_labels"].extend(["a", "b", "c", "[CLS]"]) + return result + + review = review_candidate(candidate, blueprint, plan, {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertTrue(review["production_pass"]) + self.assertEqual(review["ocr"]["unexpected_labels"], []) + + def test_candidate_review_rejects_dataset_drawn_as_peer_modality(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"].update({ + "inputs": [ + {"id": "dataset", "name": "Source Dataset", "role": "data_source"}, + {"id": "image", "name": "Image", "role": "input modality"}, + ], + "relations": [ + {"source": "dataset", "target": "image", "type": "data_source"}, + {"source": "image", "target": "parser", "type": "data_flow"}, + {"source": "parser", "target": "graph", "type": "data_flow"}, + ], + "required_labels": ["Source Dataset", "Image", "Document Parser", "Evidence Graph Builder"], + }) + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"]["detected_labels"] = ["Source Dataset", "Image", "Document Parser", "Evidence Graph Builder"] + result["scientific"]["role_mismatches"] = ["Source Dataset is shown as a peer input modality instead of the source of Image"] + return result + + def topology(_path, _prompt): + return {"summary": "Topology pass.", "relations": [], "missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": [], "repair": [], "repair_regions": [], "score": 1.0, "passed": True} + + review = review_candidate(candidate, blueprint, plan, {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic, topology_adapter=topology) + + self.assertFalse(review["production_pass"]) + self.assertFalse(review["scientific"]["passed"]) + self.assertEqual(len(review["scientific"]["role_mismatches"]), 1) + roles = collect_visual_roles(plan["figure_specification"]) + self.assertIn("never show it as a peer modality", next(item["visual_constraint"] for item in roles if item["id"] == "dataset")) + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + self.assertIn("source -> modality -> method chain", prompt) + + def test_focused_topology_arbitrates_false_general_connector_claim(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"]["topology"] = "linear" + plan["review_ground_truth"] = { + "strict_evidence_required": True, + "priorities": [{"id": "claim", "kind": "central_claim", "label": "Parser feeds graph", "required": True, "evidence_ids": ["E1"], "evidence": [{"evidence_id": "E1", "text": "Parser feeds graph."}]}], + } + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["scientific"].update({"invented_items": ["Invented shortcut arrow from Document Parser directly to output"], "score": 0.0, "passed": False}) + result["paper_alignment"] = { + "priorities": [{"id": "claim", "status": "conflicting", "visible_evidence": "A direct arrow bypasses the Evidence Graph Builder", "evidence_ids": ["E1"]}], + "missing_priorities": [], + "evidence_conflicts": ["The shortcut arrow bypasses Evidence Graph Builder"], + "unsupported_visual_claims": ["Direct connection from Document Parser to output"], + "repair": ["Remove the shortcut arrow"], + "score": 0.0, + "passed": False, + } + result["hard_errors"] = ["Invented relation: shortcut arrow bypasses Evidence Graph Builder"] + return result + + def topology(path, prompt): + return { + "relations": [{"source": "parser", "target": "graph", "status": "present", "visible_evidence": "Visible arrow enters graph"}], + "missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": [], + "repair": [], "repair_regions": [], "score": 1.0, "passed": True, + } + + review = review_candidate(candidate, blueprint, plan, {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic, topology_adapter=topology) + + self.assertTrue(review["production_pass"]) + self.assertTrue(review["cross_critic_reconciliation"]["applied"]) + self.assertEqual(review["scientific"]["invented_items"], []) + self.assertEqual(review["paper_alignment"]["evidence_conflicts"], []) + self.assertEqual(review["paper_alignment"]["priorities"][0]["status"], "supported") + + def test_visual_role_contract_merges_duplicate_innovation_with_existing_node(self): + spec = { + "inputs": [{"id": "image", "name": "Input Image", "role": "input"}], + "modules": [{"id": "roi", "name": "RoIAlign", "role": "module"}], + "outputs": [{"id": "mask", "name": "mask branch", "role": "output head"}], + "innovations": [ + {"id": "innovation_roi", "name": "RoIAlign"}, + {"id": "innovation_mask", "name": "mask branch"}, + ], + } + + roles = collect_visual_roles(spec) + + self.assertEqual([item["label"] for item in roles].count("RoIAlign"), 1) + self.assertEqual([item["label"] for item in roles].count("mask branch"), 1) + self.assertIn("without duplicating the node", next(item["visual_constraint"] for item in roles if item["label"] == "RoIAlign")) + self.assertIn("output/evaluation boundary", next(item["visual_constraint"] for item in roles if item["label"] == "mask branch")) + + def test_visible_label_whitelist_includes_declared_connector_labels(self): + plan = _plan() + plan["figure_specification"]["relations"][0]["label"] = "iterate" + + labels = required_labels(plan) + + self.assertIn("iterate", labels) + + def test_feedback_template_aligns_input_with_generation_and_stacks_refinement_below_initial_output(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + feedback = next(item for item in profiles if item["template_id"] == "feedback") + boxes = {item["id"]: item["bbox_percent"] for item in feedback["panels"]} + center = lambda box: (box["x"] + box["w"] / 2, box["y"] + box["h"] / 2) + + self.assertLess(abs(center(boxes["task_input"])[1] - center(boxes["generation"])[1]), 0.04) + self.assertLess(abs(center(boxes["initial_output"])[0] - center(boxes["refinement"])[0]), 0.04) + self.assertLess(abs(center(boxes["feedback"])[0] - center(boxes["refined_output"])[0]), 0.06) + + def test_feedback_prompt_forbids_known_wrong_edges_and_pins_iterate_target(self): + plan = _plan() + plan["figure_specification"].update({ + "topology": "feedback", + "required_labels": ["input x", "Generate", "Initial Output", "FEEDBACK", "Self-Feedback", "REFINE", "Refined output", "Model M"], + "repeatable_labels": ["Model M"], + "inputs": [{"id": "input", "name": "input x"}], + "modules": [ + {"id": "generate", "name": "Generate"}, + {"id": "initial", "name": "Initial Output"}, + {"id": "feedback", "name": "FEEDBACK"}, + {"id": "self_feedback", "name": "Self-Feedback"}, + {"id": "refine", "name": "REFINE"}, + {"id": "model", "name": "Model M"}, + ], + "outputs": [{"id": "refined", "name": "Refined output"}], + "relations": [ + {"source": "input", "target": "generate", "type": "data_flow"}, + {"source": "generate", "target": "initial", "type": "data_flow"}, + {"source": "initial", "target": "feedback", "type": "evaluation"}, + {"source": "feedback", "target": "self_feedback", "type": "feedback"}, + {"source": "initial", "target": "refine", "type": "revision_input"}, + {"source": "self_feedback", "target": "refine", "type": "feedback"}, + {"source": "refine", "target": "refined", "type": "data_flow"}, + {"source": "refined", "target": "feedback", "type": "feedback_loop", "label": "iterate"}, + ], + }) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + + self.assertIn("Initial Output must have exactly two distinct outgoing arrows", prompt) + self.assertIn("Refined output -> Generate", prompt) + self.assertIn("terminate only at FEEDBACK", prompt) + self.assertIn("Model M is a reusable badge", prompt) + + def test_feedback_blueprint_embeds_exact_semantic_labels(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + feedback = next(item for item in profiles if item["template_id"] == "feedback") + specification = { + "required_labels": ["input x", "Generate", "Initial Output", "FEEDBACK", "Self-Feedback", "REFINE", "Refined output", "Model M"], + "inputs": [{"id": "input", "name": "input x"}], + "modules": [ + {"id": "generate", "name": "Generate"}, + {"id": "initial", "name": "Initial Output"}, + {"id": "feedback", "name": "FEEDBACK"}, + {"id": "self_feedback", "name": "Self-Feedback"}, + {"id": "refine", "name": "REFINE"}, + {"id": "model", "name": "Model M"}, + ], + "outputs": [{"id": "refined", "name": "Refined output"}], + } + + report = render_layout_blueprint(feedback, Path(temp) / "feedback_semantic.png", "3:2", figure_specification=specification) + + self.assertTrue(report["contains_paper_semantic_labels"]) + self.assertEqual(report["semantic_labels"][-1], "iterate") + self.assertTrue(Path(report["path"]).exists()) + with Image.open(report["path"]).convert("RGB") as rendered: + feedback_box = next(item["bbox_percent"] for item in feedback["panels"] if item["id"] == "feedback") + connector_x = round((feedback_box["x"] + 0.50 * feedback_box["w"]) * rendered.width) + self_feedback_bottom = round((feedback_box["y"] + 0.88 * feedback_box["h"]) * rendered.height) + connector_pixel = rendered.getpixel((connector_x, self_feedback_bottom + 5)) + self.assertLess(sum(abs(value - expected) for value, expected in zip(connector_pixel, (31, 117, 126))), 30) + + def test_generic_feedback_contract_does_not_receive_self_refine_blueprint_or_prompt_terms(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + feedback = next(item for item in profiles if item["template_id"] == "feedback") + plan = _plan() + plan["figure_specification"].update({ + "topology": "feedback", + "inputs": [{"id": "sample", "name": "Sample"}], + "modules": [{"id": "measure", "name": "Measure"}, {"id": "update", "name": "Update"}], + "outputs": [{"id": "estimate", "name": "Estimate"}], + "relations": [ + {"source": "sample", "target": "measure", "type": "data_flow"}, + {"source": "measure", "target": "update", "type": "feedback"}, + {"source": "update", "target": "estimate", "type": "data_flow"}, + {"source": "estimate", "target": "measure", "type": "feedback_loop"}, + ], + }) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + report = render_layout_blueprint(feedback, Path(temp) / "generic_feedback.png", "3:2", figure_specification=plan["figure_specification"]) + + self.assertNotIn("Model M is a reusable badge", prompt) + self.assertIn("Do not introduce Self-Refine-specific labels", prompt) + self.assertTrue(report["contains_paper_semantic_labels"]) + self.assertEqual(report["semantic_labels"], ["Sample", "Measure", "Update", "Estimate"]) + self.assertTrue(report["semantic_plan"]["applied"]) + self.assertNotIn("Self-Feedback", report["semantic_labels"]) + + def test_dense_multiframe_prompt_forbids_crossed_sam_paths(self): + plan = _plan() + labels = [ + "Promptable Segmentation Task", "Segment Anything Model", "Data Engine", "Image", "Prompt", + "Image Encoder", "Prompt Encoder", "Mask Decoder", "Valid Segmentation Mask", + "Assisted-manual", "Semi-automatic", "Fully Automatic", "SA-1B", + ] + plan["figure_specification"].update({ + "topology": "dense_multiframe", + "inputs": [{"id": "image", "name": "Image"}, {"id": "prompt", "name": "Prompt"}], + "modules": [{"id": f"module_{index}", "name": label} for index, label in enumerate(labels) if label not in {"Image", "Prompt", "Valid Segmentation Mask", "SA-1B"}], + "outputs": [{"id": "mask", "name": "Valid Segmentation Mask"}, {"id": "sa1b", "name": "SA-1B"}], + }) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + + self.assertIn("Image must connect only to Image Encoder", prompt) + self.assertIn("Image Encoder -> Prompt Encoder", prompt) + self.assertIn("Fully Automatic -> Valid Segmentation Mask", prompt) + self.assertIn("container-to-container arrow", prompt) + + def test_generic_dense_multiframe_prompt_does_not_inject_sam_terms(self): + plan = _plan() + plan["figure_specification"].update({ + "topology": "dense_multiframe", + "inputs": [{"id": "sensor", "name": "Sensor Stream"}], + "modules": [{"id": "fusion", "name": "Fusion Core"}, {"id": "archive", "name": "Archive Builder"}], + "outputs": [{"id": "report", "name": "Report"}], + "relations": [{"source": "sensor", "target": "fusion", "type": "data_flow"}, {"source": "fusion", "target": "report", "type": "data_flow"}], + }) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + + self.assertNotIn("Image must connect only to Image Encoder", prompt) + self.assertIn("Do not introduce SAM-specific encoders", prompt) + + def test_dense_overview_prompt_preserves_panels_without_exposing_supporting_labels(self): + plan = _plan() + plan["figure_specification"].update({ + "topology": "dense_multiframe", + "overview_panels": [ + {"id": "panel_a", "panel_label": "a", "label": "Model architecture", "description": "Model architecture and inference flow", "entity_ids": ["parser"]}, + {"id": "panel_b", "panel_label": "b", "label": "Tile-level pretraining", "description": "Tile-level pretraining using a teacher-student objective", "entity_ids": []}, + {"id": "panel_c", "panel_label": "c", "label": "Slide-level pretraining", "description": "Slide-level masked pretraining", "entity_ids": []}, + ], + "supporting_details": [ + {"name": "Contrastive Loss", "visual_cue": "paired augmented views converging on an unlabeled comparison objective"}, + ], + }) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + + self.assertIn("Preserve exactly 3 macro panels", prompt) + self.assertIn("Tile-level pretraining", prompt) + self.assertIn('\"supporting_visual_cues\"', prompt) + self.assertNotIn('\"label\":\"Contrastive Loss\"', prompt) + self.assertIn("Never render the cue wording itself", prompt) + + def test_generic_dense_semantic_blueprint_preserves_three_panel_geometry(self): + specification = { + "topology": "dense_multiframe", + "required_labels": ["Input", "Encoder", "Output", "Architecture", "Pretraining A", "Pretraining B"], + "overview_panels": [ + {"id": "panel_a", "panel_label": "a", "label": "Architecture", "description": "Main architecture", "entity_ids": ["input", "encoder", "output"]}, + {"id": "panel_b", "panel_label": "b", "label": "Pretraining A", "description": "First pretraining flow", "entity_ids": []}, + {"id": "panel_c", "panel_label": "c", "label": "Pretraining B", "description": "Second pretraining flow", "entity_ids": []}, + ], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "encoder", "name": "Encoder"}], + "outputs": [{"id": "output", "name": "Output"}], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow"}, + {"source": "encoder", "target": "output", "type": "data_flow"}, + ], + } + + semantic = compile_semantic_blueprint(specification) + + self.assertTrue(semantic["applied"]) + self.assertEqual([item["id"] for item in semantic["panels"]], ["panel_a", "panel_b", "panel_c"]) + self.assertTrue(all(item.get("panel_id") == "panel_a" for item in semantic["nodes"])) + self.assertTrue(all(item["bbox_percent"]["y"] < 0.47 for item in semantic["nodes"])) + + def test_generic_linear_blueprint_renders_contract_nodes_and_connectors(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + linear = next(item for item in profiles if item["template_id"] == "linear") + specification = { + "topology": "linear", + "required_labels": ["Source", "Parser", "Reasoner", "Answer"], + "inputs": [{"id": "source", "name": "Source"}], + "modules": [{"id": "parser", "name": "Parser"}, {"id": "reasoner", "name": "Reasoner"}], + "outputs": [{"id": "answer", "name": "Answer"}], + "relations": [ + {"source": "source", "target": "parser", "type": "data_flow"}, + {"source": "parser", "target": "reasoner", "type": "data_flow"}, + {"source": "reasoner", "target": "answer", "type": "data_flow"}, + ], + } + + report = render_layout_blueprint(linear, Path(temp) / "generic_linear.png", "3:2", figure_specification=specification) + + self.assertTrue(report["contains_paper_semantic_labels"]) + self.assertEqual(report["semantic_labels"], ["Source", "Parser", "Reasoner", "Answer"]) + self.assertTrue(report["semantic_plan"]["applied"]) + self.assertEqual(report["semantic_plan"]["connector_count"], 3) + self.assertTrue(Path(report["path"]).exists()) + prompt = compile_image_prompt(_plan(), {"aspect_ratio": "3:2", "language": "English"}, selected_template={**linear, "semantic_plan": report["semantic_plan"]}) + self.assertIn("Paper-grounded semantic blueprint geometry", prompt) + self.assertIn('"route_style":"straight"', prompt) + self.assertIn("preserve every node bounding box", prompt) + + def test_generic_branch_multimodal_and_feedback_blueprints_render_without_paper_specific_rules(self): + cases = [ + ( + "branch", + { + "topology": "branch", + "required_labels": ["Input", "Shared Trunk", "Head A", "Head B"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "trunk", "name": "Shared Trunk"}], + "outputs": [{"id": "a", "name": "Head A"}, {"id": "b", "name": "Head B"}], + "relations": [{"source": "input", "target": "trunk", "type": "data_flow"}, {"source": "trunk", "target": "a", "type": "branch"}, {"source": "trunk", "target": "b", "type": "branch"}], + }, + ), + ( + "multimodal", + { + "topology": "multimodal", + "required_labels": ["Vision", "Language", "Fusion", "Decision"], + "inputs": [{"id": "vision", "name": "Vision"}, {"id": "language", "name": "Language"}], + "modules": [{"id": "fusion", "name": "Fusion", "role": "shared representation"}], + "outputs": [{"id": "decision", "name": "Decision"}], + "relations": [{"source": "vision", "target": "fusion", "type": "encoding"}, {"source": "language", "target": "fusion", "type": "encoding"}, {"source": "fusion", "target": "decision", "type": "data_flow"}], + }, + ), + ( + "feedback", + { + "topology": "feedback", + "required_labels": ["Observation", "Estimator", "Correction", "State", "repeat"], + "inputs": [{"id": "observation", "name": "Observation"}], + "modules": [{"id": "estimator", "name": "Estimator"}, {"id": "correction", "name": "Correction"}], + "outputs": [{"id": "state", "name": "State"}], + "relations": [{"source": "observation", "target": "estimator", "type": "data_flow"}, {"source": "estimator", "target": "correction", "type": "data_flow"}, {"source": "correction", "target": "state", "type": "data_flow"}, {"source": "state", "target": "estimator", "type": "feedback_loop", "label": "repeat"}], + }, + ), + ] + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + by_id = {item["template_id"]: item for item in profiles} + for template_id, specification in cases: + with self.subTest(template_id=template_id): + report = render_layout_blueprint(by_id[template_id], Path(temp) / f"{template_id}.png", "3:2", figure_specification=specification) + self.assertTrue(report["semantic_plan"]["applied"]) + self.assertEqual(set(report["semantic_labels"]), set(specification["required_labels"])) + self.assertTrue(Path(report["path"]).exists()) + + def test_dense_multiframe_blueprint_embeds_sam_semantic_topology(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + dense = next(item for item in profiles if item["template_id"] == "dense-multiframe") + labels = [ + "Promptable Segmentation Task", "Segment Anything Model", "Data Engine", "Image", "Prompt", + "Image Encoder", "Prompt Encoder", "Mask Decoder", "Valid Segmentation Mask", + "Assisted-manual", "Semi-automatic", "Fully Automatic", "SA-1B", + ] + specification = { + "required_labels": labels, + "inputs": [{"id": "image", "name": "Image"}, {"id": "prompt", "name": "Prompt"}], + "modules": [{"id": f"module_{index}", "name": label} for index, label in enumerate(labels) if label not in {"Image", "Prompt", "Valid Segmentation Mask", "SA-1B"}], + "outputs": [{"id": "mask", "name": "Valid Segmentation Mask"}, {"id": "sa1b", "name": "SA-1B"}], + } + + report = render_layout_blueprint(dense, Path(temp) / "dense_semantic.png", "3:2", figure_specification=specification) + + self.assertTrue(report["contains_paper_semantic_labels"]) + self.assertIn("Mask Decoder", report["semantic_labels"]) + self.assertEqual(report["semantic_labels"][-1], "SA-1B") + + def test_complex_feedback_candidate_requires_focused_topology_pass(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"].update({ + "topology": "feedback", + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "refine", "name": "Refine"}], + "outputs": [{"id": "output", "name": "Refined Output"}], + "relations": [{"source": "input", "target": "refine", "type": "revision_input"}, {"source": "refine", "target": "output", "type": "data_flow"}], + "terminology": {}, + }) + + topology = lambda _path, _prompt: {"summary": "Focused topology.", "missing_relations": ["Input -> Refine"], "reversed_relations": [], "bypassed_relations": [], "invented_relations": [], "repair": ["Connect Input to Refine"], "repair_regions": ["Refine input"], "score": 0.5, "passed": False} + review = review_candidate(candidate, blueprint, plan, {"template_id": "feedback", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=_passing_critic, topology_adapter=topology) + + self.assertFalse(review["production_pass"]) + self.assertFalse(review["topology"]["passed"]) + self.assertIn("Input -> Refine", review["hard_errors"]) + self.assertIn("Connect Input to Refine", review["repair"]) + + def test_topology_critic_prompt_accepts_shared_component_inside_source_container(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"].update({ + "topology": "feedback", + "repeatable_labels": ["Model M"], + "modules": [ + {"id": "feedback", "name": "FEEDBACK"}, + {"id": "model", "name": "Model M"}, + {"id": "self_feedback", "name": "Self-Feedback"}, + ], + "relations": [{"source": "feedback", "target": "self_feedback", "type": "feedback"}], + "terminology": {}, + }) + captured = {} + + def topology(_path, prompt): + captured["prompt"] = prompt + return {"summary": "Focused topology.", "missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": [], "repair": [], "repair_regions": [], "score": 1.0, "passed": True} + + review_candidate(candidate, blueprint, plan, {"template_id": "feedback", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=_passing_critic, topology_adapter=topology) + + self.assertIn('"Model M"', captured["prompt"]) + self.assertIn("FEEDBACK container holding an allowed shared Model M badge", captured["prompt"]) + self.assertIn("count the container-to-target relation as present", captured["prompt"]) + + def test_issue_free_ten_point_scale_review_is_normalized_by_findings(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"]["topology"] = "feedback" + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + for section in ("ocr", "scientific", "template", "aesthetic"): + result[section]["score"] = 8.0 + result[section]["passed"] = False + return result + + topology = lambda _path, _prompt: {"summary": "Focused topology.", "missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": [], "repair": [], "repair_regions": [], "score": 10.0, "passed": True} + review = review_candidate(candidate, blueprint, plan, {"template_id": "feedback", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic, topology_adapter=topology) + + self.assertTrue(review["production_pass"]) + self.assertEqual(review["ocr"]["score"], 1.0) + self.assertEqual(review["scientific"]["score"], 1.0) + self.assertEqual(review["template"]["score"], 0.8) + self.assertEqual(review["aesthetic"]["score"], 0.8) + + def test_inspect_pdf_cli_defaults(self): + parser = build_parser() + args = parser.parse_args(["inspect-pdf", "--paper", "paper.pdf", "--out", "output/inspect"]) + self.assertEqual(args.command, "inspect-pdf") + self.assertEqual(args.deadline, 180) + self.assertEqual(args.ocr_engine, "auto") + + def test_scan_benchmark_cli_accepts_rasterize_dpi(self): + parser = build_parser() + args = parser.parse_args(["benchmark", "fast-suite", "--out", "output/scan", "--rasterize-dpi", "144"]) + self.assertEqual(args.rasterize_dpi, 144) + + def test_doctor_reports_easyocr_model_readiness(self): + report = _doctor() + models = report["pdf_tools"]["easyocr"]["models"] + self.assertIn("en_ready", models) + self.assertIn("en_ch_ready", models) + self.assertIn("allow_download", models) + self.assertIn("effective_detector_limit", report["pdf_tools"]["rapidocr"]) + + def test_executable_probe_rejects_broken_path_wrapper(self): + with patch("rfs.cli.shutil.which", return_value=r"C:\broken\pdftoppm.cmd"), patch("rfs.cli.subprocess.run") as run: + run.return_value.returncode = 1 + run.return_value.stdout = "" + run.return_value.stderr = "The system cannot find the path specified." + + result = _probe_executable("pdftoppm") + + self.assertFalse(result["available"]) + self.assertEqual(result["returncode"], 1) + self.assertIn("cannot find", result["error"]) + + def test_fast_framework_prompt_writes_contract_without_generation(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + paper = root / "paper.md" + paper.write_text(PAPER_TEXT, encoding="utf-8") + out = root / "fast" + + result = run_fast_framework_prompt( + paper=paper, + out=out, + deadline_seconds=180, + planner_mode="heuristic", + ocr_engine="off", + ) + + self.assertTrue(result["ok"]) + self.assertTrue(result["engineering_only"]) + self.assertFalse(result["production_ready"]) + self.assertEqual(result["deadline_seconds"], 180) + for name in ["paper.md", "document_model.json", "extraction_report.json", "section_index.json", "section_summary.md", "key_evidence.json", "paper_review.json", "figure_specification.json", "planning_validation_report.json", "image_prompt.md", "overlay_spec.json", "run_report.json"]: + self.assertTrue((out / name).exists(), name) + self.assertFalse((out / "selected_image.png").exists()) + overlay = json.loads((out / "overlay_spec.json").read_text(encoding="utf-8")) + self.assertTrue(overlay["labels"]) + + def test_long_scanned_paper_produces_explicit_sampled_engineering_contract(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + paper = root / "scan.pdf" + document = fitz.open() + for _ in range(12): + document.new_page(width=600, height=800) + document.save(paper) + document.close() + + def adapter(image_path, _lang): + page_number = int(Path(image_path).stem.rsplit("_", 1)[-1]) + return [{ + "text": f"Page {page_number} Abstract Method Input Image enters a CNN Backbone and Transformer Encoder Decoder to produce Class Predictions and Bounding Box Predictions with Bipartite Matching.", + "confidence": 0.97, + "quad": [[20, 20], [920, 20], [920, 100], [20, 100]], + }] + + out = root / "fast" + result = run_fast_framework_prompt( + paper=paper, + out=out, + deadline_seconds=180, + planner_mode="heuristic", + ocr_engine="easyocr", + ocr_adapter=adapter, + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["status"], "completed_with_warnings") + self.assertFalse(result["production_ready"]) + self.assertEqual(result["extraction_quality"]["semantic_scope"], "sampled_pages_only") + self.assertTrue(result["extraction_quality"]["sampled_scan_ready"]) + self.assertTrue(any("sample pages" in item for item in result["uncertainties"])) + + def test_production_image_generation_rejects_sampled_scan_scope(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + prepared = { + "ok": True, + "root": root, + "paper": str(root / "paper.pdf"), + "archived_positive": [], + "archived_negative": [], + "preferences": {"aspect_ratio": "16:9"}, + "parsed": {"extraction_report": {"scientific_scope_complete": False}}, + "selected_domain": {"id": "general"}, + "paper_review": {}, + "review_metadata": {"mode": "vlm"}, + "plan": _plan(), + "planner_metadata": {"mode": "vlm"}, + "planning_validation": {"ok": True}, + } + with patch("rfs.paper_to_image.workflow.prepare_paper_figure_contract", return_value=prepared): + with self.assertRaisesRegex(RuntimeError, "full-document scientific scope"): + run_paper_to_image(paper=root / "paper.pdf", out=root / "run", asset_mode="image2") + + def test_fast_framework_cli_defaults(self): + parser = build_parser() + args = parser.parse_args(["fast-framework-prompt", "--paper", "paper.pdf", "--out", "output/fast"]) + self.assertEqual(args.command, "fast-framework-prompt") + self.assertEqual(args.deadline, 180) + self.assertEqual(args.planner_mode, "vlm") + self.assertFalse(args.verify_ppt_roundtrip) + + def test_fast_framework_cli_accepts_ppt_roundtrip_verification(self): + parser = build_parser() + args = parser.parse_args([ + "fast-framework-prompt", + "--paper", "paper.pdf", + "--out", "output/fast", + "--verify-ppt-roundtrip", + ]) + + self.assertTrue(args.verify_ppt_roundtrip) + + def test_benchmark_fast_cli_defaults(self): + parser = build_parser() + args = parser.parse_args(["benchmark", "fast", "--case", "benchmarks/case", "--out", "output/fast"]) + self.assertEqual(args.benchmark_action, "fast") + self.assertEqual(args.deadline, 180) + self.assertEqual(args.planner_mode, "vlm") + + def test_benchmark_fast_suite_cli_accepts_repeatable_case_ids(self): + parser = build_parser() + args = parser.parse_args(["benchmark", "fast-suite", "--root", "benchmarks", "--out", "output/suite", "--case-id", "101_vit_linear", "--case-id", "106_detr_set_prediction"]) + self.assertEqual(args.benchmark_action, "fast-suite") + self.assertEqual(args.case_id, ["101_vit_linear", "106_detr_set_prediction"]) + + def test_domain_profiles_detect_method_and_survey(self): + method = {"evidence": [{"text": "A neural model is optimized with a training loss and used during inference."}]} + survey = {"evidence": [{"text": "This systematic survey presents a taxonomy and review of the landscape."}]} + self.assertEqual(detect_domain_profile(method)["id"], "ai-ml-method") + self.assertEqual(detect_domain_profile(survey)["id"], "survey-review") + + def test_strict_review_rejects_ungrounded_relation(self): + parsed = {"evidence": [{"id": "E0001"}]} + profile = {"id": "general", "required_sections": ["research_questions", "central_claims", "contributions", "modules", "limitations"]} + fact = {"id": "f1", "statement": "fact", "status": "required", "evidence_ids": ["E0001"]} + review = {"research_questions": [fact], "central_claims": [fact], "contributions": [fact], "modules": [{**fact, "id": "m1"}], "limitations": [fact], "research_objects": [], "relations": [{"id": "r1", "source_id": "m1", "target_id": "missing", "evidence_ids": []}], "workflows": {"training": [], "inference": []}, "experiments": {}} + report = validate_review_coverage(review, parsed, profile, strict=True) + self.assertFalse(report["ok"]) + self.assertTrue(any("unknown endpoint" in item for item in report["errors"])) + + def test_four_reference_ratios_map_to_expected_templates(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + refs = [] + for name, size in [("arbor", (1831, 979)), ("linear", (1336, 306)), ("tripanel", (804, 277)), ("dense", (2498, 1086))]: + path = root / f"{name}.png" + Image.new("RGB", size, "white").save(path) + refs.append(str(path)) + profiles = build_template_profiles(refs, root / "profiles", mode="heuristic") + self.assertEqual([item["template_id"] for item in profiles], ["arbor", "linear", "tripanel", "dense-multimodal"]) + selected = select_template(profiles, {"modules": [{"statement": "multimodal retrieval"}] * 10, "inputs": [], "relations": [], "workflows": {"feedback": []}}, requested="auto", target_ratio="auto") + self.assertEqual(selected["template_id"], "dense-multimodal") + report = render_layout_blueprint(selected, root / "blueprint.png", "auto") + self.assertFalse(report["contains_reference_text"]) + self.assertTrue((root / "blueprint.png").exists()) + + def test_three_stage_non_loop_workflow_selects_linear_template(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + review = { + "modules": [{"statement": value} for value in ("SSL", "RETFound", "supervised learning")], + "inputs": [{"statement": "CFP"}, {"statement": "OCT"}], + "relations": [], + "workflows": {"feedback": []}, + } + + selected = select_template(profiles, review, requested="auto", target_ratio="16:9") + + self.assertEqual(selected["template_id"], "linear") + + def test_feedback_workflow_selects_feedback_template_instead_of_arbor(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + review = { + "modules": [{"statement": value} for value in ("Generate", "Initial Output", "Feedback", "Refine", "Refined Output")], + "inputs": [{"statement": "Input"}], + "relations": [{"relation_type": "feedback", "statement": "Refined Output returns to Feedback"}], + "workflows": {"feedback": [{"statement": "iterative self-feedback loop"}]}, + } + + selected = select_template(profiles, review, requested="auto", target_ratio="16:9") + + self.assertEqual(selected["template_id"], "feedback") + + def test_parallel_head_workflow_selects_branch_template(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + review = { + "modules": [{"statement": value} for value in ("Backbone", "RPN", "RoIAlign", "classification branch", "box branch", "mask branch")], + "inputs": [{"statement": "Input Image"}], + "relations": [{"relation_type": "branch", "statement": "RoIAlign feeds three parallel heads"}], + "workflows": {"feedback": []}, + } + + selected = select_template(profiles, review, requested="auto", target_ratio="16:9") + + self.assertEqual(selected["template_id"], "branch") + + def test_many_modality_workflow_selects_multimodal_template(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + review = { + "inputs": [{"statement": value, "visible_label": value} for value in ("Image", "Text", "Audio", "Depth", "Thermal", "IMU")], + "modules": [{"statement": "Modality Encoders"}, {"statement": "Joint Embedding Space"}], + "central_claims": [{"statement": "Cross-modal inputs share one embedding space"}], + "relations": [], + "workflows": {"feedback": []}, + } + + selected = select_template(profiles, review, requested="auto", target_ratio="16:9") + + self.assertEqual(selected["template_id"], "multimodal") + + def test_dense_contract_topology_selects_dense_multiframe_template(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + review = { + "inputs": [{"statement": "Image"}, {"statement": "Prompt"}], + "modules": [{"statement": value} for value in ("Image Encoder", "Prompt Encoder", "Mask Decoder")], + "relations": [], + "workflows": {"feedback": []}, + } + + selected = select_template(profiles, review, requested="auto", target_ratio="16:9", contract_topology="dense_multiframe") + + self.assertEqual(selected["template_id"], "dense-multiframe") + + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=_passing_critic, require_visual_enrichment=False) + self.assertTrue(result["selected_passed_all_checks"]) + self.assertEqual(result["successful_candidates"], 3) + self.assertTrue((root / "selected_image.png").exists()) + manifest_text = (root / "image2_request_manifest.json").read_text(encoding="utf-8") + manifest = json.loads(manifest_text) + self.assertEqual(manifest["model"], "gpt-image-2") + self.assertEqual(len(manifest["requests"]), 3) + self.assertNotIn("secret-value", manifest_text) + self.assertTrue(all(item["endpoint"].endswith("/images/edits") for item in manifest["requests"])) + self.assertEqual(result["stability"]["seed_count"], 3) + self.assertEqual(result["stability"]["production_pass_rate"], 1.0) + self.assertEqual(result["stability"]["status"], "stable") + self.assertTrue((root / "stability_report.json").exists()) + + @patch("rfs.paper_to_image.generator.requests.post", side_effect=requests.Timeout("provider stalled")) + def test_image2_timeout_is_not_blindly_retried(self, post): + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE2_TIMEOUT": "30"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + + with self.assertRaises(RuntimeError): + generate_and_select( + _plan(), + {"aspect_ratio": "3:2"}, + template, + root / "layout_blueprint.png", + root, + asset_mode="image2", + candidates=1, + image_retries=3, + review_mode="vlm", + repair_rounds=0, + ocr_engine="off", + critic_adapter=_passing_critic, + ) + + self.assertEqual(post.call_count, 1) + + @patch("rfs.paper_to_image.generator.requests.post") + def test_three_seed_stability_replaces_only_provider_failed_candidate(self, post): + calls = {"count": 0} + + def response(*_args, **_kwargs): + calls["count"] += 1 + if calls["count"] == 1: + raise requests.Timeout("provider stalled") + return FakeResponse() + + post.side_effect = response + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_STABILITY_REPLACEMENT_RETRIES": "1", "RFS_PAPER_IMAGE_WORKERS": "3"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + + result = generate_and_select(_plan(), {"aspect_ratio": "3:2"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, image_retries=1, review_mode="vlm", repair_rounds=0, ocr_engine="off", critic_adapter=_passing_critic, require_visual_enrichment=False) + + self.assertEqual(post.call_count, 4) + self.assertEqual(result["stability"]["production_pass_rate"], 1.0) + self.assertEqual(sum("provider_replacement_attempt" in item for item in result["candidates"]), 1) + + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + def test_resume_candidates_reuses_existing_images_and_generates_only_missing_seed(self, post): + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_PAPER_IMAGE_WORKERS": "3"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + candidate_dir = root / "candidates" + candidate_dir.mkdir() + Image.new("RGB", (1536, 1024), "white").save(candidate_dir / "candidate_01.png") + Image.new("RGB", (1536, 1024), "white").save(candidate_dir / "candidate_02.png") + + result = generate_and_select(_plan(), {"aspect_ratio": "3:2"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, image_retries=1, review_mode="vlm", repair_rounds=0, ocr_engine="off", critic_adapter=_passing_critic, resume_candidates=True, require_visual_enrichment=False) + + self.assertEqual(post.call_count, 1) + modes = {item["candidate_id"]: item["generation"]["mode"] for item in result["candidates"]} + self.assertEqual(modes["candidate_01"], "existing_candidate_resume") + self.assertEqual(modes["candidate_02"], "existing_candidate_resume") + self.assertEqual(modes["candidate_03"], "edit") + self.assertEqual(result["stability"]["production_pass_rate"], 1.0) + + @patch("rfs.paper_to_image.generator.requests.post") + def test_repair_source_skips_fresh_candidate_generation_when_source_passes(self, post): + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value"}, clear=False): + root = Path(temp) + blueprint = root / "blueprint.png" + source = root / "failed_candidate.png" + Image.new("RGB", (1536, 1024), "white").save(blueprint) + Image.new("RGB", (1536, 1024), "white").save(source) + + result = generate_and_select( + _plan(), + {"aspect_ratio": "3:2", "language": "English"}, + {"template_id": "linear", "forbidden_copy_terms": []}, + blueprint, + root / "run", + asset_mode="image2", + candidates=3, + review_mode="vlm", + repair_rounds=1, + repair_source=source, + critic_adapter=_passing_critic, + require_visual_enrichment=False, + ) + + post.assert_not_called() + self.assertEqual(result["requested_candidates"], 0) + self.assertEqual(result["selected_candidate_id"], "source_candidate") + self.assertTrue(Path(result["selected_image"]).exists()) + + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + def test_failed_candidates_get_one_repair(self, _post): + calls = {"count": 0} + def critic(path, blueprint, prompt): + calls["count"] += 1 + if calls["count"] <= 3: + result = _passing_critic(path, blueprint, prompt) + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.5, "passed": False}) + result["repair"] = ["Correct the Document Parser label"] + result["repair_regions"] = ["left panel"] + result["hard_errors"] = ["missing critical label"] + return result + return _passing_critic(path, blueprint, prompt) + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=critic, require_visual_enrichment=False) + self.assertEqual(result["repair_candidates"], 1) + self.assertEqual(result["selected_candidate_id"], "repair_01") + repair = next(item for item in result["candidates"] if item["candidate_id"] == "repair_01") + self.assertIn("generation_seconds", repair["timings"]) + self.assertIn("review_seconds", repair["timings"]) + + @patch("rfs.paper_to_image.generator.requests.post") + @patch("rfs.paper_to_image.workflow.validate_review_coverage", return_value={"ok": True, "errors": [], "warnings": []}) + @patch("rfs.paper_to_image.planner.call_vlm_json") + def test_mock_full_production_workflow(self, planner_call, _coverage, _post): + def echo_generation_blueprint(_url, **kwargs): + payload = base64.b64encode(kwargs["files"]["image"][1].read()).decode("ascii") + + class BlueprintResponse: + status_code = 200 + + @staticmethod + def raise_for_status(): + return None + + @staticmethod + def json(): + return {"data": [{"b64_json": payload}]} + + return BlueprintResponse() + + _post.side_effect = echo_generation_blueprint + fact = {"id": "fact_01", "statement": "Evidence-grounded reasoning", "status": "required", "importance": "critical", "confidence": 1.0, "evidence_ids": ["E0001"], "must_appear_in_figure": True, "visual_role": "module"} + planner_call.return_value = {"summary": "Planning result.", **_plan()} + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "known-test-secret", "RFS_IMAGE_MODEL": "image-2", "RFS_CACHE_DIR": temp}, clear=False): + root = Path(temp) + paper = root / "paper.md" + paper.write_text(PAPER_TEXT, encoding="utf-8") + out = root / "run" + result = run_paper_to_image(paper=paper, out=out, planner_mode="vlm", domain_profile="general", template="linear", asset_mode="image2", candidates=3, aspect_ratio="1.5:1", review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=_passing_critic, require_visual_enrichment=False) + self.assertTrue(result["ok"]) + self.assertEqual(result["paper_review_mode"], "derived_from_vlm_plan") + self.assertEqual(result["template_id"], "linear") + self.assertTrue((out / "selected_image.png").exists()) + self.assertFalse(any(out.glob("*.pptx"))) + self.assertEqual(json.loads((out / "image2_request_manifest.json").read_text(encoding="utf-8"))["model"], "gpt-image-2") + + def test_local_ocr_can_block_vlm_pass(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + image = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (800, 600), "white").save(image) + Image.new("RGB", (800, 600), "white").save(blueprint) + review = review_candidate(image, blueprint, _plan(), {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="paddle", ocr_adapter=lambda _p, _l: [{"text": "Evidence Graph Builder"}], critic_adapter=_passing_critic) + self.assertFalse(review["ocr"]["passed"]) + self.assertIn("Document Parser", review["ocr"]["missing_labels"]) + self.assertFalse(review["production_pass"]) + + def test_paper_evidence_is_sent_to_critic_and_alignment_is_a_hard_gate(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + image = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (800, 600), "white").save(image) + Image.new("RGB", (800, 600), "white").save(blueprint) + plan = _plan() + plan["review_ground_truth"] = { + "summary": "Evidence contract.", + "strict_evidence_required": True, + "priorities": [{ + "id": "GT_MODULE_parser", + "kind": "module", + "label": "Document Parser", + "required": True, + "evidence_ids": ["E0001"], + "evidence": [{"evidence_id": "E0001", "page": 2, "section": "Method", "text": "The Document Parser splits the input paper into page-aware passages."}], + }], + "forbidden": [], + "unknown": [], + } + captured = {} + + def critic(path, blueprint_path, prompt): + captured["prompt"] = prompt + result = _passing_critic(path, blueprint_path, prompt) + result["paper_alignment"] = { + "priorities": [{"id": "GT_MODULE_parser", "status": "supported", "visible_evidence": "Document Parser is shown", "evidence_ids": ["E0001"]}], + "missing_priorities": [], + "evidence_conflicts": [], + "unsupported_visual_claims": ["A memory bank is shown although the paper does not support it"], + "score": 0.8, + "passed": False, + } + return result + + review = review_candidate(image, blueprint, plan, {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertIn("page-aware passages", captured["prompt"]) + self.assertIn("E0001", captured["prompt"]) + self.assertFalse(review["paper_alignment"]["passed"]) + self.assertFalse(review["production_pass"]) + self.assertTrue(any("memory bank" in item.casefold() for item in review["hard_errors"])) + + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + def test_multi_round_repair_can_succeed_on_second_round(self, post): + calls = {"count": 0} + + def critic(path, blueprint, prompt): + calls["count"] += 1 + result = _passing_critic(path, blueprint, prompt) + if calls["count"] == 1: + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.3, "passed": False}) + result["repair"] = ["Restore the exact Document Parser label"] + result["hard_errors"] = ["Document Parser is missing"] + elif calls["count"] == 2: + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.7, "passed": False}) + result["repair"] = ["Correct the remaining Document Parser spelling"] + result["hard_errors"] = ["Document Parser is still misspelled"] + return result + + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=1, review_mode="vlm", repair_rounds=4, ocr_engine="off", critic_adapter=critic, require_visual_enrichment=False) + + self.assertEqual(result["selected_candidate_id"], "repair_02") + self.assertEqual(result["repair_candidates"], 2) + self.assertEqual(result["repair_stop_reason"], "all_gates_passed") + self.assertEqual(post.call_count, 3) + history = json.loads((root / "repair_history.json").read_text(encoding="utf-8")) + self.assertTrue(all(item["accepted"] for item in history["rounds"])) + + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + def test_scientific_regressions_are_rejected_and_two_failures_stop_repairs(self, post): + calls = {"count": 0} + + def critic(path, blueprint, prompt): + calls["count"] += 1 + result = _passing_critic(path, blueprint, prompt) + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.5, "passed": False}) + result["repair"] = ["Restore Document Parser"] + result["hard_errors"] = ["Document Parser is missing"] + if calls["count"] >= 2: + result["ocr"].update({"missing_labels": [], "score": 1.0, "passed": True}) + result["scientific"].update({"missing_modules": ["Evidence Graph Builder"], "score": 0.6, "passed": False}) + result["hard_errors"] = ["Evidence Graph Builder disappeared"] + return result + + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + with self.assertRaises(RuntimeError): + generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=1, review_mode="vlm", repair_rounds=4, ocr_engine="off", critic_adapter=critic, require_visual_enrichment=False) + + history = json.loads((root / "repair_history.json").read_text(encoding="utf-8")) + self.assertEqual(history["completed_rounds"], 2) + self.assertEqual(history["stop_reason"], "two_consecutive_rounds_without_improvement") + self.assertTrue(all(not item["accepted"] for item in history["rounds"])) + self.assertTrue(any("scientific score decreased" in reason for reason in history["rounds"][0]["rejection_reasons"])) + self.assertEqual(post.call_count, 3) + + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + def test_resume_reuses_completed_repair_rounds_without_new_image_calls(self, post): + def critic(path, blueprint, prompt): + result = _passing_critic(path, blueprint, prompt) + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.5, "passed": False}) + result["repair"] = ["Restore Document Parser"] + result["hard_errors"] = ["Document Parser is missing"] + return result + + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + with self.assertRaises(RuntimeError): + generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=1, review_mode="vlm", repair_rounds=4, ocr_engine="off", critic_adapter=critic, require_visual_enrichment=False) + self.assertEqual(post.call_count, 3) + + post.reset_mock() + with self.assertRaises(RuntimeError): + generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=1, review_mode="vlm", repair_rounds=4, ocr_engine="off", critic_adapter=critic, resume_candidates=True, require_visual_enrichment=False) + + post.assert_not_called() + manifest = json.loads((root / "image2_request_manifest.json").read_text(encoding="utf-8")) + resumed = [item for item in manifest["requests"] if item.get("candidate_id", "").startswith("repair_")] + self.assertTrue(resumed) + self.assertTrue(all(item["mode"] == "existing_repair_resume" for item in resumed)) + + @patch("rfs.paper_to_image.generator.review_candidate") + def test_transient_inconclusive_vlm_review_is_retried_without_regenerating_image(self, review_mock): + base = { + "basic": {"passed": True}, + "ocr": {"score": 1.0, "passed": True}, + "scientific": {"score": 1.0, "passed": True}, + "paper_alignment": {"score": 1.0, "passed": True}, + "topology": {"score": 1.0, "passed": True}, + "template": {"score": 1.0, "passed": True}, + "aesthetic": {"score": 1.0, "passed": True}, + "hard_errors": [], "preserve": [], "repair": [], "remove": [], "repair_regions": [], + "production_pass": True, "overall_score": 1.0, + } + review_mock.side_effect = [ + {**base, "vlm_warning": "temporary provider timeout", "production_pass": False, "overall_score": 0.0}, + dict(base), + ] + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"RFS_PAPER_IMAGE_REVIEW_ATTEMPTS": "2"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + + result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1"}, template, root / "layout_blueprint.png", root, asset_mode="placeholder", candidates=1, review_mode="vlm", repair_rounds=0, ocr_engine="off", require_visual_enrichment=False) + + self.assertEqual(review_mock.call_count, 2) + self.assertEqual(result["candidates"][0]["review"]["review_attempts"], 2) + self.assertEqual(result["candidates"][0]["review"]["review_attempt_history"][0]["inconclusive"], True) + + @patch("rfs.paper_to_image.generator.requests.post") + def test_provider_failures_do_not_count_as_semantic_no_improvement(self, post): + class RateLimitedResponse: + status_code = 429 + + post.return_value = RateLimitedResponse() + + def critic(path, blueprint, prompt): + result = _passing_critic(path, blueprint, prompt) + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.5, "passed": False}) + result["repair"] = ["Restore Document Parser"] + result["hard_errors"] = ["Document Parser is missing"] + return result + + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value"}, clear=False): + root = Path(temp) + blueprint = root / "blueprint.png" + source = root / "source.png" + Image.new("RGB", (1536, 1024), "white").save(blueprint) + Image.new("RGB", (1536, 1024), "white").save(source) + with self.assertRaises(RuntimeError): + generate_and_select(_plan(), {"aspect_ratio": "3:2"}, {"template_id": "linear", "forbidden_copy_terms": []}, blueprint, root / "run", asset_mode="image2", candidates=1, image_retries=0, review_mode="vlm", repair_rounds=3, repair_source=source, ocr_engine="off", critic_adapter=critic, require_visual_enrichment=False) + + history = json.loads((root / "run" / "repair_history.json").read_text(encoding="utf-8")) + self.assertEqual(history["completed_rounds"], 3) + self.assertEqual(history["stop_reason"], "maximum_repair_rounds_reached") + self.assertTrue(all(item["provider_failure"] for item in history["rounds"])) + self.assertEqual(post.call_count, 3) + + @patch("rfs.paper_to_image.generator._call_image2_edit") + def test_provider_read_timeout_stops_repair_loop_without_blind_second_wait(self, edit_mock): + edit_mock.side_effect = RuntimeError("HTTPSConnectionPool: Read timed out. (read timeout=120)") + + def critic(path, blueprint, prompt): + result = _passing_critic(path, blueprint, prompt) + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.5, "passed": False}) + result["repair"] = ["Restore Document Parser"] + result["hard_errors"] = ["Document Parser is missing"] + return result + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + blueprint = root / "blueprint.png" + source = root / "source.png" + Image.new("RGB", (1536, 1024), "white").save(blueprint) + Image.new("RGB", (1536, 1024), "white").save(source) + with self.assertRaises(RuntimeError): + generate_and_select(_plan(), {"aspect_ratio": "3:2"}, {"template_id": "linear", "forbidden_copy_terms": []}, blueprint, root / "run", asset_mode="image2", candidates=1, image_retries=0, review_mode="vlm", repair_rounds=4, repair_source=source, ocr_engine="off", critic_adapter=critic, require_visual_enrichment=False) + + history = json.loads((root / "run" / "repair_history.json").read_text(encoding="utf-8")) + self.assertEqual(history["completed_rounds"], 1) + self.assertEqual(history["stop_reason"], "provider_timeout") + self.assertTrue(history["rounds"][0]["provider_timeout"]) + self.assertEqual(edit_mock.call_count, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_presentations_qa.py b/tests/integration/test_presentations_qa.py similarity index 100% rename from tests/test_presentations_qa.py rename to tests/integration/test_presentations_qa.py diff --git a/tests/test_rebuild_editable.py b/tests/integration/test_rebuild_editable.py similarity index 82% rename from tests/test_rebuild_editable.py rename to tests/integration/test_rebuild_editable.py index 9349be9..a2c65e4 100644 --- a/tests/test_rebuild_editable.py +++ b/tests/integration/test_rebuild_editable.py @@ -37,6 +37,63 @@ def _fixture(path: Path) -> Path: class RebuildEditableTests(unittest.TestCase): + def test_skip_analysis_reuses_existing_text_layer_without_rerunning_ocr(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = _fixture(root / "pipeline.png") + out = root / "rebuild" + rebuild_editable( + reference, + out, + asset_mode="placeholder", + text_mode="manual", + layout_mode="heuristic", + control_mode="heuristic", + design_plan_mode="heuristic", + ) + + with patch("rfs.editable_rebuild.build_text_layer") as text_builder: + result = rebuild_editable( + reference, + out, + asset_mode="placeholder", + text_mode="ocr", + layout_mode="hybrid", + control_mode="hybrid", + design_plan_mode="vlm", + skip_analysis=True, + ) + + text_builder.assert_not_called() + self.assertTrue(result["ok"]) + self.assertTrue(result["text_layer_reused"]) + + def test_global_design_plan_artifacts_are_written(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = _fixture(root / "pipeline.png") + out = root / "rebuild" + + result = rebuild_editable( + reference, + out, + asset_mode="placeholder", + text_mode="off", + layout_mode="heuristic", + control_mode="heuristic", + design_plan_mode="heuristic", + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["design_plan_effective_mode"], "heuristic") + for name in [ + "reference_logic_plan.json", + "reference_layer_plan.json", + "reference_generation_plan.json", + "reference_flow_graph.json", + ]: + self.assertTrue((out / name).exists(), name) + def test_cli_placeholder_run_writes_required_outputs(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -501,6 +558,64 @@ def fake_repair(_reference, _preview, _dsl, _round): repair = json.loads((out / "professional_repair_round_1.json").read_text(encoding="utf-8")) self.assertEqual(repair["applied_count"], 1) + def test_smart_api_policy_filters_text_slots_and_disables_final_crop(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = _fixture(root / "pipeline.png") + out = root / "pro" + + def fake_planner(_reference, _baseline, _text_geometry): + return { + "canvas": {"width_px": 640, "height_px": 360, "width_in": 12.0, "height_in": 6.75, "background": "#F7F3EA"}, + "objects": [ + {"type": "canvas", "id": "canvas", "width_px": 640, "height_px": 360, "width_in": 12.0, "height_in": 6.75, "background": "#F7F3EA"}, + {"type": "text", "id": "text_label", "text": "Output", "bbox_percent": {"x": 0.55, "y": 0.20, "w": 0.20, "h": 0.08}, "font_size_pt": 12}, + {"type": "asset_slot", "id": "misread_text_as_icon", "asset_id": "misread_text_as_icon", "bbox_percent": {"x": 0.55, "y": 0.20, "w": 0.20, "h": 0.08}, "asset_type": "generic", "prompt_subject": "Output text"}, + {"type": "asset_slot", "id": "robot_icon", "asset_id": "robot_icon", "bbox_percent": {"x": 0.20, "y": 0.25, "w": 0.16, "h": 0.24}, "asset_type": "character", "prompt_subject": "friendly robot"}, + ], + } + + result = rebuild_editable_pro(reference, out, asset_mode="crop", asset_policy="smart-api", text_mode="off", repair_rounds=0, planner_adapter=fake_planner) + self.assertTrue(result["ok"]) + self.assertEqual(result["asset_policy"], "smart-api") + report = json.loads((out / "asset_generation_report.json").read_text(encoding="utf-8")) + self.assertFalse(report["final_crop_assets_allowed"]) + statuses = {item["slot_id"]: item["status"] for item in report["assets"]} + self.assertEqual(statuses["robot_icon"], "crop_disabled_by_smart_api_policy_placeholder") + self.assertNotIn("misread_text_as_icon", statuses) + text_filter = json.loads((out / "text_asset_filter_report.json").read_text(encoding="utf-8")) + self.assertEqual(text_filter["items"][0]["candidate_id"], "misread_text_as_icon") + program = json.loads((out / "figure_program.json").read_text(encoding="utf-8")) + self.assertEqual([slot["id"] for slot in program["slots"]], ["robot_icon"]) + + def test_smart_api_policy_reuses_duplicate_complex_icons(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = _fixture(root / "pipeline.png") + out = root / "pro" + + def fake_planner(_reference, _baseline, _text_geometry): + return { + "canvas": {"width_px": 640, "height_px": 360, "width_in": 12.0, "height_in": 6.75, "background": "#F7F3EA"}, + "objects": [ + {"type": "canvas", "id": "canvas", "width_px": 640, "height_px": 360, "width_in": 12.0, "height_in": 6.75, "background": "#F7F3EA"}, + {"type": "asset_slot", "id": "executor_a_icon", "asset_id": "executor_a_icon", "bbox_percent": {"x": 0.20, "y": 0.25, "w": 0.12, "h": 0.18}, "asset_type": "tool_combo", "prompt_subject": "terminal and database icon"}, + {"type": "asset_slot", "id": "executor_b_icon", "asset_id": "executor_b_icon", "bbox_percent": {"x": 0.40, "y": 0.25, "w": 0.12, "h": 0.18}, "asset_type": "tool_combo", "prompt_subject": "terminal and database icon"}, + ], + } + + with patch.dict("os.environ", {"API_BASE": "", "API_KEY": "", "GEMINI_API_KEY": "", "GEMINI_GEN_IMG_URL": ""}, clear=False): + result = rebuild_editable_pro(reference, out, asset_mode="api", asset_policy="smart-api", text_mode="off", repair_rounds=0, planner_adapter=fake_planner) + self.assertTrue(result["ok"]) + report = json.loads((out / "asset_generation_report.json").read_text(encoding="utf-8")) + self.assertEqual(report["api_requests_attempted"], 1) + statuses = {item["slot_id"]: item["status"] for item in report["assets"]} + self.assertEqual(statuses["executor_a_icon"], "api_failed_placeholder_fallback") + self.assertEqual(statuses["executor_b_icon"], "reused_from_group") + plan = json.loads((out / "api_asset_plan.json").read_text(encoding="utf-8")) + self.assertEqual(plan["unique_api_assets"], 1) + self.assertEqual(plan["reused_slots"], 1) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_validator.py b/tests/integration/test_validator.py similarity index 100% rename from tests/test_validator.py rename to tests/integration/test_validator.py diff --git a/tests/test_paper_to_image.py b/tests/test_paper_to_image.py deleted file mode 100644 index 95e51d9..0000000 --- a/tests/test_paper_to_image.py +++ /dev/null @@ -1,239 +0,0 @@ -from __future__ import annotations - -import base64 -import io -import json -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from PIL import Image - -from rfs.cli import build_parser -from rfs.paper_to_image import run_paper_to_image -from rfs.paper_to_image.critics import review_candidate -from rfs.paper_to_image.generator import generate_and_select -from rfs.paper_to_image.review import detect_domain_profile, validate_review_coverage -from rfs.paper_to_image.templates import build_template_profiles, render_layout_blueprint, select_template - - -PAPER_TEXT = """ -Evidence-Grounded Modular Reasoning for Scientific Documents - -Abstract -We introduce ModularTrace, a method that converts scientific documents into an evidence graph and then performs constrained reasoning over the graph. The method reduces unsupported conclusions while preserving exact paper terminology. - -1 Introduction -Scientific document assistants often generate claims without traceable evidence. ModularTrace addresses this problem by linking every extracted fact to a source passage. - -2 Method -The Document Parser splits the input paper into page-aware passages. The Evidence Graph Builder extracts entities and directed relations. The Constrained Reasoner consumes the graph and produces an answer with evidence identifiers. During training, a consistency loss penalizes unsupported relations. During inference, only retrieved evidence nodes may support an answer. - -3 Experiments -We evaluate on three scientific question-answering datasets using answer accuracy and evidence precision. - -4 Conclusion -ModularTrace improves evidence precision while maintaining answer accuracy. -""".strip() - - -def _png_b64(size=(1536, 1024), color=(235, 242, 247)) -> str: - buffer = io.BytesIO() - Image.new("RGB", size, color).save(buffer, format="PNG") - return base64.b64encode(buffer.getvalue()).decode("ascii") - - -class FakeResponse: - status_code = 200 - - def raise_for_status(self): - return None - - def json(self): - return {"data": [{"b64_json": _png_b64()}]} - - -def _passing_critic(_path, _blueprint, _prompt): - return { - "summary": "Mock passing critic.", - "ocr": {"detected_labels": ["Document Parser", "Evidence Graph Builder"], "missing_labels": [], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "score": 1.0, "passed": True}, - "scientific": {"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "innovation_visible": True, "score": 1.0, "passed": True}, - "template": {"macro_panel_match": 0.9, "reading_order_match": 0.9, "connector_rhythm_match": 0.9, "visual_density_match": 0.9, "copied_reference_content": [], "score": 0.9, "passed": True}, - "aesthetic": {"hierarchy": 0.9, "balance": 0.9, "whitespace": 0.9, "color": 0.9, "icon_consistency": 0.9, "readability": 0.9, "score": 0.9, "passed": True}, - "preserve": ["macro layout"], - "repair": [], - "remove": [], - "repair_regions": [], - "hard_errors": [], - } - - -def _plan() -> dict: - modules = [ - {"id": "parser", "name": "Document Parser", "role": "module", "evidence_ids": ["E0001"]}, - {"id": "graph", "name": "Evidence Graph Builder", "role": "module", "evidence_ids": ["E0001"]}, - ] - return { - "paper_summary": {"summary": "Paper summary.", "title": "ModularTrace"}, - "figure_specification": {"summary": "Scientific contract.", "figure_goal": "Show the method.", "modules": modules, "relations": [{"source": "parser", "target": "graph", "type": "data_flow", "evidence_ids": ["E0001"]}], "terminology": {"Document Parser": "Document Parser", "Evidence Graph Builder": "Evidence Graph Builder"}, "forbidden_inventions": []}, - "design_plan": {"summary": "Design.", "reading_order": ["parser", "graph"]}, - "layout_intent": {"summary": "Layout.", "pattern": "left_to_right"}, - "visual_metaphors": {"summary": "Metaphors.", "items": []}, - "style_plan": {"summary": "Style.", "medium": "academic"}, - } - - -class PaperToImageTests(unittest.TestCase): - def test_offline_workflow_is_engineering_only(self): - with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_KEY": "known-test-secret"}, clear=False): - root = Path(temp) - paper = root / "paper.md" - paper.write_text(PAPER_TEXT, encoding="utf-8") - out = root / "run" - result = run_paper_to_image(paper=paper, out=out, planner_mode="heuristic", asset_mode="placeholder", candidates=2, aspect_ratio="auto", review_mode="heuristic", ocr_engine="off") - - self.assertTrue(result["ok"]) - self.assertFalse(result["pptx_generated"]) - self.assertEqual(result["candidate_count"], 2) - self.assertIsNone(result["selected_image"]) - self.assertTrue((out / "engineering_preview.png").exists()) - self.assertFalse((out / "selected_image.png").exists()) - for name in ["document_index.json", "paper_review.json", "review_coverage_report.json", "domain_profile.json", "selected_template.json", "layout_blueprint.png", "image2_request_manifest.json", "ocr_review.json", "template_alignment_report.json", "scientific_critic_report.json", "aesthetic_critic_report.json"]: - self.assertTrue((out / name).exists(), name) - document_index = json.loads((out / "document_index.json").read_text(encoding="utf-8")) - self.assertTrue(document_index["sections"]) - manifest = json.loads((out / "image2_request_manifest.json").read_text(encoding="utf-8")) - self.assertEqual(manifest["requests"][0]["model"], "offline-placeholder") - self.assertIn("api_key_present", manifest) - self.assertNotIn("known-test-secret", json.dumps(manifest)) - - def test_cli_production_defaults(self): - parser = build_parser() - args = parser.parse_args(["paper-to-image", "--paper", "paper.pdf", "--out", "output/run"]) - self.assertEqual(args.command, "paper-to-image") - self.assertEqual(args.asset_mode, "image2") - self.assertEqual(args.candidates, 3) - self.assertEqual(args.review_mode, "vlm") - self.assertEqual(args.aspect_ratio, "auto") - self.assertEqual(args.domain_profile, "auto") - self.assertEqual(args.template, "auto") - self.assertEqual(args.repair_rounds, 1) - - def test_domain_profiles_detect_method_and_survey(self): - method = {"evidence": [{"text": "A neural model is optimized with a training loss and used during inference."}]} - survey = {"evidence": [{"text": "This systematic survey presents a taxonomy and review of the landscape."}]} - self.assertEqual(detect_domain_profile(method)["id"], "ai-ml-method") - self.assertEqual(detect_domain_profile(survey)["id"], "survey-review") - - def test_strict_review_rejects_ungrounded_relation(self): - parsed = {"evidence": [{"id": "E0001"}]} - profile = {"id": "general", "required_sections": ["research_questions", "central_claims", "contributions", "modules", "limitations"]} - fact = {"id": "f1", "statement": "fact", "status": "required", "evidence_ids": ["E0001"]} - review = {"research_questions": [fact], "central_claims": [fact], "contributions": [fact], "modules": [{**fact, "id": "m1"}], "limitations": [fact], "research_objects": [], "relations": [{"id": "r1", "source_id": "m1", "target_id": "missing", "evidence_ids": []}], "workflows": {"training": [], "inference": []}, "experiments": {}} - report = validate_review_coverage(review, parsed, profile, strict=True) - self.assertFalse(report["ok"]) - self.assertTrue(any("unknown endpoint" in item for item in report["errors"])) - - def test_four_reference_ratios_map_to_expected_templates(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - refs = [] - for name, size in [("arbor", (1831, 979)), ("linear", (1336, 306)), ("tripanel", (804, 277)), ("dense", (2498, 1086))]: - path = root / f"{name}.png" - Image.new("RGB", size, "white").save(path) - refs.append(str(path)) - profiles = build_template_profiles(refs, root / "profiles", mode="heuristic") - self.assertEqual([item["template_id"] for item in profiles], ["arbor", "linear", "tripanel", "dense-multimodal"]) - selected = select_template(profiles, {"modules": [{"statement": "multimodal retrieval"}] * 10, "inputs": [], "relations": [], "workflows": {"feedback": []}}, requested="auto", target_ratio="auto") - self.assertEqual(selected["template_id"], "dense-multimodal") - report = render_layout_blueprint(selected, root / "blueprint.png", "auto") - self.assertFalse(report["contains_reference_text"]) - self.assertTrue((root / "blueprint.png").exists()) - - @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) - def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): - with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): - root = Path(temp) - template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} - render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") - result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=_passing_critic) - self.assertTrue(result["selected_passed_all_checks"]) - self.assertEqual(result["successful_candidates"], 3) - self.assertTrue((root / "selected_image.png").exists()) - manifest_text = (root / "image2_request_manifest.json").read_text(encoding="utf-8") - manifest = json.loads(manifest_text) - self.assertEqual(manifest["model"], "gpt-image-2") - self.assertEqual(len(manifest["requests"]), 3) - self.assertNotIn("secret-value", manifest_text) - self.assertTrue(all(item["endpoint"].endswith("/images/edits") for item in manifest["requests"])) - - @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) - def test_failed_candidates_get_one_repair(self, _post): - calls = {"count": 0} - def critic(path, blueprint, prompt): - calls["count"] += 1 - if calls["count"] <= 3: - result = _passing_critic(path, blueprint, prompt) - result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.5, "passed": False}) - result["repair"] = ["Correct the Document Parser label"] - result["repair_regions"] = ["left panel"] - result["hard_errors"] = ["missing critical label"] - return result - return _passing_critic(path, blueprint, prompt) - with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): - root = Path(temp) - template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} - render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") - result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=critic) - self.assertEqual(result["repair_candidates"], 1) - self.assertEqual(result["selected_candidate_id"], "repair_01") - - @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) - @patch("rfs.paper_to_image.planner.call_vlm_json") - @patch("rfs.paper_to_image.review.call_vlm_json") - def test_mock_full_production_workflow(self, review_call, planner_call, _post): - fact = {"id": "fact_01", "statement": "Evidence-grounded reasoning", "status": "required", "importance": "critical", "confidence": 1.0, "evidence_ids": ["E0001"], "must_appear_in_figure": True, "visual_role": "module"} - review_call.return_value = { - "summary": "Universal structured paper review.", - "paper_identity": {"title": "ModularTrace", "paper_type": "method", "field": "AI"}, - "research_questions": [fact], - "central_claims": [{**fact, "id": "claim_01"}], - "inputs": [], "outputs": [], "research_objects": [], "concepts": [], - "modules": [{**fact, "id": "parser", "statement": "Document Parser"}, {**fact, "id": "graph", "statement": "Evidence Graph Builder"}], - "relations": [{**fact, "id": "relation_01", "source_id": "parser", "target_id": "graph", "relation_type": "data_flow", "visual_role": "relation"}], - "workflows": {"training": [], "inference": [], "offline": [], "online": [], "feedback": []}, - "contributions": [{**fact, "id": "contribution_01"}], "innovations": [], "assumptions": [], "limitations": [{**fact, "id": "limitation_01"}], - "experiments": {"datasets": [], "settings": [], "metrics": [], "baselines": [], "ablations": []}, "results": [], - "terminology": [{**fact, "id": "term_01", "statement": "Document Parser", "visible_label": "Document Parser"}, {**fact, "id": "term_02", "statement": "Evidence Graph Builder", "visible_label": "Evidence Graph Builder"}], - "forbidden_inventions": [], "unknowns": [], "contradictions": [], "ambiguities": [], "human_review_required": [], "figure_candidates": [], - } - planner_call.return_value = {"summary": "Planning result.", **_plan()} - with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "known-test-secret", "RFS_IMAGE_MODEL": "image-2"}, clear=False): - root = Path(temp) - paper = root / "paper.md" - paper.write_text(PAPER_TEXT, encoding="utf-8") - out = root / "run" - result = run_paper_to_image(paper=paper, out=out, planner_mode="vlm", domain_profile="general", template="linear", asset_mode="image2", candidates=3, aspect_ratio="1.5:1", review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=_passing_critic) - self.assertTrue(result["ok"]) - self.assertEqual(result["paper_review_mode"], "vlm") - self.assertEqual(result["template_id"], "linear") - self.assertTrue((out / "selected_image.png").exists()) - self.assertFalse(any(out.glob("*.pptx"))) - self.assertEqual(json.loads((out / "image2_request_manifest.json").read_text(encoding="utf-8"))["model"], "gpt-image-2") - - def test_local_ocr_can_block_vlm_pass(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - image = root / "candidate.png" - blueprint = root / "blueprint.png" - Image.new("RGB", (800, 600), "white").save(image) - Image.new("RGB", (800, 600), "white").save(blueprint) - review = review_candidate(image, blueprint, _plan(), {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="paddle", ocr_adapter=lambda _p, _l: [{"text": "Evidence Graph Builder"}], critic_adapter=_passing_critic) - self.assertFalse(review["ocr"]["passed"]) - self.assertIn("Document Parser", review["ocr"]["missing_labels"]) - self.assertFalse(review["production_pass"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..61a6105 --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1 @@ +"""Fast tests for isolated contracts, geometry, text, and composition behavior.""" diff --git a/tests/test_arrow_router.py b/tests/unit/test_arrow_router.py similarity index 100% rename from tests/test_arrow_router.py rename to tests/unit/test_arrow_router.py diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py new file mode 100644 index 0000000..6055d0b --- /dev/null +++ b/tests/unit/test_benchmarking.py @@ -0,0 +1,515 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from PIL import Image +from pptx import Presentation +from pptx.util import Inches + +from rfs.evaluation.benchmarking import ( + _score_planning_contract, + audit_paper_image_run, + list_benchmark_cases, + run_fast_benchmark_case, + run_fast_benchmark_suite, + score_benchmark_case, + validate_benchmark_case, +) +from rfs.evaluation.pdf_extraction_benchmark import run_pdf_extraction_stress_suite +from rfs.evaluation.scanned_pdf import rasterize_pdf_as_scan +from rfs.paper_to_image.analyzer import parse_paper + + +ROOT = Path(__file__).resolve().parents[2] + + +class BenchmarkingTests(unittest.TestCase): + def test_rasterized_scan_fixture_preserves_pages_and_removes_native_text(self): + import fitz + + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) / "source.pdf" + target = Path(temp) / "scanned.pdf" + second_target = Path(temp) / "scanned_again.pdf" + document = fitz.open() + for text in ("Abstract native text", "Method native text"): + page = document.new_page(width=600, height=800) + page.insert_text((50, 80), text, fontsize=14) + document.save(source) + document.close() + + rasterize_pdf_as_scan(source, target, dpi=110) + rasterize_pdf_as_scan(source, second_target, dpi=110) + parsed = parse_paper(target, ocr_engine="off") + first_hash = hashlib.sha256(target.read_bytes()).hexdigest() + second_hash = hashlib.sha256(second_target.read_bytes()).hexdigest() + + self.assertEqual(parsed["page_count"], 2) + self.assertEqual(parsed["extraction_report"]["pdf_type"], "scanned") + self.assertTrue(all(page["char_count"] == 0 for page in parsed["pages"])) + self.assertEqual(first_hash, second_hash) + def test_planning_contract_treats_relation_labels_as_intermediate_artifacts(self): + expected = { + "entities": [ + {"id": "generator", "label": "Generate"}, + {"id": "draft", "label": "Initial Output"}, + {"id": "critic", "label": "Feedback"}, + ], + "relations": [ + {"source": "generator", "target": "draft"}, + {"source": "draft", "target": "critic"}, + ], + } + specification = { + "modules": [ + {"id": "init", "name": "Generate"}, + {"id": "feedback", "name": "Feedback"}, + ], + "relations": [ + {"source": "init", "target": "feedback", "label": "Initial Output"}, + ], + } + + result = _score_planning_contract(expected, specification) + + self.assertEqual(result["entity_recall"], 1.0) + self.assertEqual(result["relation_recall"], 1.0) + + def test_planning_contract_prefers_real_entity_with_variable_suffix_over_relation_label(self): + expected = { + "entities": [ + {"id": "refiner", "label": "Refine"}, + {"id": "result", "label": "Refined Output"}, + ], + "relations": [{"source": "refiner", "target": "result"}], + } + specification = { + "modules": [{"id": "refine", "name": "Refine"}], + "outputs": [{"id": "final", "name": "Refined Output (yn)"}], + "relations": [{"source": "refine", "target": "final", "label": "Refined Output"}], + } + + result = _score_planning_contract(expected, specification) + + self.assertEqual(result["entity_mapping"]["result"], "final") + self.assertEqual(result["relation_recall"], 1.0) + + def test_planning_contract_uses_relations_to_disambiguate_equal_input_labels(self): + expected = { + "entities": [ + {"id": "task_input", "label": "Input"}, + {"id": "generator", "label": "Generate"}, + ], + "relations": [{"source": "task_input", "target": "generator"}], + } + specification = { + "inputs": [ + {"id": "iteration_input", "name": "Input (y0)"}, + {"id": "task_input", "name": "Input"}, + ], + "modules": [ + {"id": "model", "name": "Model"}, + {"id": "generate", "name": "Generate"}, + ], + "relations": [ + {"source": "iteration_input", "target": "model"}, + {"source": "task_input", "target": "generate"}, + ], + } + + result = _score_planning_contract(expected, specification) + + self.assertEqual(result["entity_mapping"]["task_input"], "task_input") + self.assertEqual(result["relation_recall"], 1.0) + + def test_bundled_cases_validate_and_list(self): + p2i = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + i2p = ROOT / "benchmarks" / "image-to-ppt" / "cases" / "001_three_stage_layout" + + self.assertTrue(validate_benchmark_case(p2i)["ok"]) + self.assertTrue(validate_benchmark_case(i2p)["ok"]) + listing = list_benchmark_cases(ROOT / "benchmarks") + self.assertGreaterEqual(len(listing["cases"]), 7) + + def test_real_paper_case_is_valid_with_source_metadata(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "101_vit_linear" + validation = validate_benchmark_case(case_dir) + + self.assertTrue(validation["ok"]) + self.assertTrue((case_dir / "source.json").exists()) + + def test_fast_benchmark_command_runs_without_image_generation(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + result = run_fast_benchmark_case(case_dir, tmp, planner_mode="heuristic", ocr_engine="off") + + self.assertTrue(result["ok"]) + self.assertTrue((Path(tmp) / "fast_benchmark_result.json").exists()) + self.assertFalse((Path(tmp) / "selected_image.png").exists()) + + def test_fast_suite_aggregates_reliability_and_timing_metrics(self): + with tempfile.TemporaryDirectory() as tmp: + result = run_fast_benchmark_suite( + ROOT / "benchmarks", + tmp, + case_ids=["001_linear_pipeline"], + planner_mode="heuristic", + ocr_engine="off", + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["aggregate"]["case_count"], 1) + self.assertIn("mean_total_seconds", result["aggregate"]) + self.assertIn("provider_success_rate", result["aggregate"]) + self.assertIn("mean_evidence_page_coverage_ratio", result["aggregate"]) + self.assertIn("ocr_scheduled_page_total", result["aggregate"]) + self.assertIn("ocr_attempted_page_total", result["aggregate"]) + self.assertIn("ocr_incomplete_run_count", result["aggregate"]) + self.assertIn("max_detected_column_count", result["aggregate"]) + self.assertIn("mean_section_count", result["aggregate"]) + self.assertIn("typographic_heading_total", result["aggregate"]) + self.assertIn("merged_heading_line_total", result["aggregate"]) + self.assertIn("figure_caption_total", result["aggregate"]) + self.assertIn("formula_total", result["aggregate"]) + self.assertIn("max_ocr_worker_count", result["aggregate"]) + self.assertIn("repeated_margin_noise_removed_total", result["aggregate"]) + self.assertIn("native_hyphenation_repair_total", result["aggregate"]) + self.assertIn("mean_ocr_latin_known_word_ratio", result["aggregate"]) + self.assertIn("ocr_spacing_repair_total", result["aggregate"]) + self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) + + def test_pdf_stress_suite_reports_ocr_stage_timings(self): + with tempfile.TemporaryDirectory() as temp: + result = run_pdf_extraction_stress_suite(Path(temp) / "pdf", ocr_engine="off") + + self.assertIn("ocr_stage_seconds", result["aggregate"]) + self.assertIn("recognition_seconds", result["aggregate"]["ocr_stage_seconds"]) + adapter_case = next(item for item in result["cases"] if item["case_id"] == "mixed_scan_fixture_adapter") + self.assertIn("ocr_page_durations", adapter_case) + self.assertIn("render_seconds", adapter_case["ocr_stage_seconds"]) + + def test_pdf_extraction_stress_suite_runs_deterministically_without_runtime_ocr(self): + with tempfile.TemporaryDirectory() as tmp: + result = run_pdf_extraction_stress_suite(tmp, ocr_engine="off") + + self.assertTrue(result["ok"]) + self.assertEqual(result["aggregate"]["case_count"], 10) + self.assertEqual(result["aggregate"]["passed_case_count"], 10) + self.assertTrue((Path(tmp) / "fixtures" / "native_two_column.pdf").exists()) + self.assertTrue((Path(tmp) / "fixtures" / "repeated_margin_noise.pdf").exists()) + self.assertTrue((Path(tmp) / "fixtures" / "native_chinese.pdf").exists()) + self.assertTrue((Path(tmp) / "fixtures" / "hyphenated_native.pdf").exists()) + self.assertTrue((Path(tmp) / "fixtures" / "rotated_repeated_margin.pdf").exists()) + self.assertTrue((Path(tmp) / "fixtures" / "formula_table_dense.pdf").exists()) + self.assertTrue((Path(tmp) / "mixed_scan_fixture_adapter" / "document_model.json").exists()) + self.assertTrue((Path(tmp) / "pdf_extraction_stress_report.json").exists()) + + def test_paper_to_image_score_uses_hard_scientific_gates(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "ocr": {"score": 1.0, "missing_labels": [], "misspelled_labels": [], "forbidden_labels_found": []}, + "aesthetic": {"score": 0.9, "readability": 0.9}, + "clarity": {"score": 0.9}, + "information": {"score": 0.95}, + "stability": {"production_pass_rate": 1.0}, + "hard_errors": [], + } + (run / "candidate_review.json").write_text(json.dumps({ + "selected_candidate_id": "candidate_01", + "candidates": [{"candidate_id": "candidate_01", "score": 0.95, "review": review}], + }), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertTrue(result["passed"]) + self.assertEqual(result["metrics"]["relation_recall"], 1.0) + + def test_paper_to_image_score_blocks_evidence_conflicts_and_reports_repair_metrics(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "paper_alignment": { + "score": 0.8, + "passed": False, + "priorities": [{"id": "GT_1", "status": "conflicting"}], + "missing_priorities": [], + "evidence_conflicts": [{"id": "GT_1", "evidence_ids": ["E1"], "reason": "visible claim reverses the paper statement"}], + "unsupported_visual_claims": [], + }, + "topology": {"score": 1.0, "passed": True, "missing_relations": [], "reversed_relations": [], "invented_relations": []}, + "ocr": {"score": 1.0, "missing_labels": [], "misspelled_labels": [], "forbidden_labels_found": []}, + "aesthetic": {"score": 0.9, "readability": 0.9}, + "clarity": {"score": 0.9}, + "information": {"score": 0.95}, + "stability": {"production_pass_rate": 1.0}, + "hard_errors": [], + } + (run / "candidate_review.json").write_text(json.dumps({ + "selected_candidate_id": "candidate_01", + "candidates": [{"candidate_id": "candidate_01", "score": 0.9, "review": review}], + }), encoding="utf-8") + (run / "review_ground_truth.json").write_text(json.dumps({ + "strict_evidence_required": True, + "priorities": [{"id": "GT_1", "required": True}], + }), encoding="utf-8") + (run / "repair_history.json").write_text(json.dumps({ + "rounds": [ + {"accepted": True}, + {"accepted": False, "rejection_reasons": ["scientific score decreased from 1.0 to 0.8"]}, + ] + }), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertFalse(result["passed"]) + self.assertEqual(result["metrics"]["evidence_conflict_count"], 1) + self.assertEqual(result["metrics"]["evidence_priority_coverage"], 0.0) + self.assertEqual(result["metrics"]["accepted_repair_count"], 1) + self.assertEqual(result["metrics"]["repair_regression_rejection_count"], 1) + self.assertIn("candidate conflicts with quoted paper evidence", result["hard_failures"]) + + def test_new_review_does_not_block_supported_extra_visible_labels(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "paper_alignment": {"score": 1.0, "passed": True, "priorities": [], "missing_priorities": [], "evidence_conflicts": [], "unsupported_visual_claims": []}, + "topology": {"score": 1.0, "passed": True, "missing_relations": [], "reversed_relations": [], "invented_relations": []}, + "ocr": { + "score": 1.0, + "detected_labels": ["Raw Document", "Document Encoding", "Final Prediction", "Add & Norm"], + "missing_labels": [], + "misspelled_labels": [], + "forbidden_labels_found": [], + "unexpected_labels": ["Add & Norm"], + "unsupported_unexpected_labels": [], + }, + "aesthetic": {"score": 1.0, "readability": 1.0}, + "clarity": {"score": 1.0}, + "information": {"score": 1.0}, + "stability": {"production_pass_rate": 1.0}, + "hard_errors": [], + } + (run / "candidate_review.json").write_text(json.dumps({"selected_candidate_id": "candidate_01", "requested_candidates": 1, "candidates": [{"candidate_id": "candidate_01", "production_pass": True, "score": 1.0, "review": review}]}), encoding="utf-8") + (run / "figure_specification.json").write_text(json.dumps({"required_labels": ["Raw Document", "Document Encoding", "Final Prediction"]}), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertNotIn("unexpected visible labels outside the normalized figure contract", result["hard_failures"]) + self.assertEqual(result["metrics"]["unexpected_label_count"], 0) + + def test_audit_existing_image_compiles_ground_truth_and_writes_frozen_review(self): + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + Image.new("RGB", (800, 600), "white").save(run / "selected_image.png") + Image.new("RGB", (800, 600), "white").save(run / "layout_blueprint.png") + specification = { + "summary": "Scientific contract.", + "inputs": [{"id": "raw", "name": "Raw Document", "evidence_ids": ["E1"]}], + "modules": [{"id": "encode", "name": "Document Encoding", "evidence_ids": ["E1"]}], + "outputs": [{"id": "prediction", "name": "Final Prediction", "evidence_ids": ["E1"]}], + "relations": [{"source": "raw", "target": "encode", "type": "data_flow", "evidence_ids": ["E1"]}], + "required_labels": ["Raw Document", "Document Encoding", "Final Prediction"], + } + (run / "figure_specification.json").write_text(json.dumps(specification), encoding="utf-8") + (run / "key_evidence.json").write_text(json.dumps({ + "evidence": [{"id": "E1", "page": 2, "section_hint": "Method", "text": "The Raw Document is transformed by Document Encoding into the Final Prediction.", "confidence": 1.0, "source": "plain"}] + }), encoding="utf-8") + (run / "selected_template.json").write_text(json.dumps({"template_id": "linear", "forbidden_copy_terms": []}), encoding="utf-8") + (run / "preferences.json").write_text(json.dumps({"aspect_ratio": "4:3"}), encoding="utf-8") + (run / "generation_parameters.json").write_text(json.dumps({"asset_mode": "placeholder"}), encoding="utf-8") + (run / "candidate_review.json").write_text(json.dumps({"selected_candidate_id": "candidate_01", "candidates": []}), encoding="utf-8") + + def critic(path, blueprint, prompt): + ground_truth = json.loads((run / "review_ground_truth.json").read_text(encoding="utf-8")) + return { + "ocr": {"detected_labels": ["Raw Document", "Document Encoding", "Final Prediction"], "missing_labels": [], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "score": 1.0, "passed": True}, + "scientific": {"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "role_mismatches": [], "containment_mismatches": [], "innovation_visible": True, "score": 1.0, "passed": True}, + "paper_alignment": {"priorities": [{"id": item["id"], "status": "supported", "evidence_ids": item.get("evidence_ids", [])} for item in ground_truth["priorities"] if item.get("required")], "missing_priorities": [], "evidence_conflicts": [], "unsupported_visual_claims": [], "score": 1.0, "passed": True}, + "template": {"copied_reference_content": [], "score": 0.9, "passed": True}, + "aesthetic": {"visual_information_density": 0.9, "mechanism_visualization": 0.9, "publication_polish": 0.9, "score": 0.9, "passed": True}, + "preserve": [], "repair": [], "remove": [], "repair_regions": [], "hard_errors": [], + } + + result = audit_paper_image_run(run, critic_adapter=critic) + + self.assertTrue(result["production_pass"]) + self.assertEqual(result["required_priority_count"], result["grounded_required_priority_count"]) + self.assertTrue((run / "review_ground_truth.json").exists()) + self.assertTrue((run / "benchmark_review.json").exists()) + self.assertTrue((run / "paper_image_reaudit.json").exists()) + self.assertTrue((run / "reaudit_repair_plan.json").exists()) + + def test_audit_failed_run_falls_back_to_highest_scoring_recorded_candidate(self): + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + Image.new("RGB", (800, 600), "white").save(run / "layout_blueprint.png") + weaker = run / "candidate_01.png" + stronger = run / "repair_02.png" + Image.new("RGB", (800, 600), "white").save(weaker) + Image.new("RGB", (800, 600), "white").save(stronger) + (run / "figure_specification.json").write_text(json.dumps({"required_labels": ["Input"]}), encoding="utf-8") + (run / "selected_template.json").write_text(json.dumps({"template_id": "linear", "forbidden_copy_terms": []}), encoding="utf-8") + (run / "preferences.json").write_text(json.dumps({"aspect_ratio": "4:3"}), encoding="utf-8") + (run / "generation_parameters.json").write_text(json.dumps({"asset_mode": "placeholder"}), encoding="utf-8") + (run / "candidate_review.json").write_text(json.dumps({ + "selected_candidate_id": None, + "candidates": [ + {"candidate_id": "candidate_01", "path": str(weaker), "score": 0.5, "production_pass": False}, + {"candidate_id": "repair_02", "path": str(stronger), "score": 0.9, "production_pass": False}, + ], + }), encoding="utf-8") + + def critic(path, blueprint, prompt): + self.assertEqual(Path(path), stronger) + return { + "ocr": {"detected_labels": ["Input"], "missing_labels": [], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "score": 1.0, "passed": True}, + "scientific": {"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "role_mismatches": [], "containment_mismatches": [], "innovation_visible": True, "score": 1.0, "passed": True}, + "paper_alignment": {"priorities": [], "missing_priorities": [], "evidence_conflicts": [], "unsupported_visual_claims": [], "score": 1.0, "passed": True}, + "template": {"copied_reference_content": [], "score": 1.0, "passed": True}, + "aesthetic": {"visual_information_density": 1.0, "mechanism_visualization": 1.0, "publication_polish": 1.0, "score": 1.0, "passed": True}, + "hard_errors": [], "preserve": [], "repair": [], "remove": [], "repair_regions": [], + } + + result = audit_paper_image_run(run, critic_adapter=critic) + + self.assertEqual(result["candidate_id"], "repair_02") + self.assertEqual(Path(result["candidate"]), stronger) + + def test_paper_to_image_score_checks_planning_contract(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "ocr": {"score": 1.0, "missing_labels": [], "misspelled_labels": [], "forbidden_labels_found": []}, + "aesthetic": {"score": 0.9, "readability": 0.9}, + "clarity": {"score": 0.9}, + "information": {"score": 0.95}, + "stability": {"production_pass_rate": 1.0}, + "hard_errors": [], + } + (run / "candidate_review.json").write_text(json.dumps({ + "selected_candidate_id": "candidate_01", + "candidates": [{"candidate_id": "candidate_01", "score": 0.95, "review": review}], + }), encoding="utf-8") + (run / "figure_specification.json").write_text(json.dumps({ + "modules": [{"id": "only_module", "name": "Feature Encoder"}], + "relations": [], + }), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertFalse(result["passed"]) + self.assertLess(result["metrics"]["plan_entity_recall"], 1.0) + self.assertEqual(result["metrics"]["plan_relation_recall"], 0.0) + self.assertIn("paper planning missed required benchmark entities", result["hard_failures"]) + self.assertIn("paper planning missed required benchmark relations", result["hard_failures"]) + + def test_paper_to_image_score_uses_generated_stability_report(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "ocr": {"score": 1.0, "missing_labels": [], "misspelled_labels": [], "forbidden_labels_found": []}, + "aesthetic": {"score": 0.9, "readability": 0.9}, + "clarity": {"score": 0.9}, + "information": {"score": 0.95}, + "hard_errors": [], + } + (run / "candidate_review.json").write_text(json.dumps({ + "selected_candidate_id": "candidate_01", + "candidates": [{"candidate_id": "candidate_01", "score": 0.95, "review": review}], + }), encoding="utf-8") + (run / "stability_report.json").write_text(json.dumps({ + "summary": "Generated stability.", + "seed_count": 3, + "mean_score": 0.94, + "worst_case_score": 0.91, + "standard_deviation": 0.02, + "production_pass_rate": 1.0, + }), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertEqual(result["metrics"]["stability_score"], 1.0) + self.assertEqual(result["total_score"], 0.9575) + + def test_stability_audit_rejects_candidates_with_unexpected_visible_labels(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + base_review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "aesthetic": {"score": 1.0, "readability": 1.0}, + "clarity": {"score": 1.0}, + "information": {"score": 1.0}, + "hard_errors": [], + } + required = ["Raw Document", "Document Encoding", "Evidence Aggregation", "Final Prediction"] + candidates = [] + for index in range(1, 4): + labels = [*required, "Contrastive Loss"] if index == 2 else required + review = {**base_review, "ocr": {"score": 1.0, "detected_labels": labels, "missing_labels": [], "misspelled_labels": [], "forbidden_labels_found": []}} + candidates.append({"candidate_id": f"candidate_{index:02d}", "score": 1.0, "production_pass": True, "review": review}) + (run / "candidate_review.json").write_text(json.dumps({ + "requested_candidates": 3, + "selected_candidate_id": "candidate_01", + "candidates": candidates, + "stability": {"production_pass_rate": 1.0}, + }), encoding="utf-8") + (run / "figure_specification.json").write_text(json.dumps({ + "inputs": [{"id": "raw", "name": "Raw Document"}], + "modules": [{"id": "encoding", "name": "Document Encoding"}, {"id": "aggregation", "name": "Evidence Aggregation"}], + "outputs": [{"id": "prediction", "name": "Final Prediction"}], + "relations": [ + {"source": "raw", "target": "encoding"}, + {"source": "encoding", "target": "aggregation"}, + {"source": "aggregation", "target": "prediction"}, + ], + "required_labels": required, + }), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertFalse(result["passed"]) + self.assertEqual(result["metrics"]["stability_score"], 0.6667) + self.assertIn("repeated generation stability below production threshold", result["hard_failures"]) + second = next(item for item in result["stability_audit"]["candidates"] if item["candidate_id"] == "candidate_02") + self.assertEqual(second["unexpected_labels"], ["Contrastive Loss"]) + + def test_full_slide_picture_is_detected_as_editability_cheating(self): + case_dir = ROOT / "benchmarks" / "image-to-ppt" / "cases" / "001_three_stage_layout" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + reference = case_dir / "reference.ppm" + with Image.open(reference) as image: + image.save(run / "rebuild_preview.png") + image.save(run / "reference.png") + prs = Presentation() + prs.slide_width = Inches(10) + prs.slide_height = Inches(5.625) + slide = prs.slides.add_slide(prs.slide_layouts[6]) + slide.shapes.add_picture(str(run / "reference.png"), 0, 0, width=prs.slide_width, height=prs.slide_height) + prs.save(run / "editable_composition.pptx") + (run / "composition_quality_report.json").write_text(json.dumps({"slots": [], "cards": [], "arrows": []}), encoding="utf-8") + (run / "rebuild_visual_quality_report.json").write_text(json.dumps({"blocking_issue_count": 0}), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertFalse(result["passed"]) + self.assertEqual(result["metrics"]["full_slide_image_count"], 1) + self.assertIn("full-slide reference image detected", result["hard_failures"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_contract_framework_bridge.py b/tests/unit/test_contract_framework_bridge.py new file mode 100644 index 0000000..d0ebea8 --- /dev/null +++ b/tests/unit/test_contract_framework_bridge.py @@ -0,0 +1,99 @@ +import tempfile +import unittest +from pathlib import Path + +from PIL import Image + +from rfs.contracts import apply_paper_semantic_contract, compile_paper_brief_from_semantic_contract +from rfs.reference_analyzer import analyze_reference + + +def _panel_contract() -> dict: + return { + "summary": "Test paper contract.", + "figure_specification": { + "summary": "Panel-local contract.", + "topology": "dense_multiframe", + "figure_goal": "Show training and architecture without merging repeated model instances.", + "panel_graphs": [ + { + "id": "panel_a", + "label": "Training and usage", + "nodes": [ + {"instance_id": "a_input", "name": "Dataset", "role": "input", "evidence_ids": ["E1"]}, + {"instance_id": "a_model", "name": "Model", "role": "module", "evidence_ids": ["E2"]}, + {"instance_id": "a_output", "name": "Prediction", "role": "output", "evidence_ids": ["E3"]}, + ], + "relations": [ + {"source": "a_input", "target": "a_model", "type": "data_flow", "evidence_ids": ["E4"]}, + {"source": "a_model", "target": "a_output", "type": "prediction", "evidence_ids": ["E5"]}, + ], + }, + { + "id": "panel_b", + "label": "Architecture", + "nodes": [ + {"instance_id": "b_group", "name": "Model block", "role": "group", "evidence_ids": ["E6"]}, + {"instance_id": "b_model", "name": "Model", "role": "module", "evidence_ids": ["E7"]}, + {"instance_id": "b_output", "name": "Output distribution", "role": "output", "evidence_ids": ["E8"]}, + ], + "relations": [ + {"source": "b_model", "target": "b_output", "type": "prediction", "evidence_ids": ["E9"]}, + ], + }, + ], + "relations": [ + {"source": "a_input", "target": "b_output", "type": "incorrect_global_compatibility_edge"} + ], + }, + } + + +class ContractFrameworkBridgeTests(unittest.TestCase): + def test_panel_graph_contract_drives_slots_and_relations_without_summary_fillers(self): + contract = _panel_contract() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = root / "reference.png" + Image.new("RGB", (1200, 800), "white").save(reference) + + brief = compile_paper_brief_from_semantic_contract(contract, root, source_path="paper.pdf") + inventory = analyze_reference( + reference, + brief, + root, + slot_count=25, + slot_source="reference-primary", + control_localizer_mode="off", + ) + + self.assertEqual(brief["planner"], "semantic_contract") + self.assertEqual(brief["semantic_authority"], "panel_graphs") + self.assertEqual(len(brief["slot_suggestions"]), 5) + self.assertNotIn("b_group", {item["id"] for item in brief["slot_suggestions"]}) + self.assertEqual(inventory["slot_count"], 25) + self.assertFalse(any("visual unit" in item["paper_concept"] for item in inventory["slots"])) + self.assertFalse(any("local detail" in item["paper_concept"] for item in inventory["slots"])) + self.assertTrue(all(item.get("semantic_instance_id") for item in inventory["slots"])) + self.assertTrue(any(item.get("slot_origin") == "reference_visual_split" for item in inventory["slots"])) + + program = { + "summary": "Test program.", + "slots": inventory["slots"], + "cards": [], + "panels": [], + "arrows": [], + "text_program": {"summary": "Empty text.", "items": []}, + } + updated, report = apply_paper_semantic_contract(program, contract, root) + + mappings = {item["entity_id"]: item["object_id"] for item in report["mappings"]} + self.assertEqual(report["contract_graph_source"], "panel_graphs") + self.assertEqual(report["mapped_relation_count"], 3) + self.assertNotEqual(mappings["a_model"], mappings["b_model"]) + self.assertEqual({item["semantic_panel_id"] for item in updated["arrows"]}, {"panel_a", "panel_b"}) + self.assertFalse(any(item["relation_type"] == "incorrect_global_compatibility_edge" for item in updated["arrows"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_control_localizer.py b/tests/unit/test_control_localizer.py new file mode 100644 index 0000000..8e23b1e --- /dev/null +++ b/tests/unit/test_control_localizer.py @@ -0,0 +1,19 @@ +import unittest + +import numpy as np + +from rfs.cv_compat import hough_line_coordinates + + +class ControlLocalizerTests(unittest.TestCase): + def test_hough_line_coordinates_accepts_opencv_shape_variants(self): + self.assertEqual(hough_line_coordinates(np.array([[1, 2, 3, 4]], dtype=np.int32)), (1, 2, 3, 4)) + self.assertEqual(hough_line_coordinates(np.array([1, 2, 3, 4], dtype=np.int32)), (1, 2, 3, 4)) + self.assertEqual(hough_line_coordinates([[1, 2, 3, 4]]), (1, 2, 3, 4)) + + def test_hough_line_coordinates_rejects_incomplete_rows(self): + self.assertIsNone(hough_line_coordinates(np.array([1, 2, 3], dtype=np.int32))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_overlay_renderer.py b/tests/unit/test_overlay_renderer.py new file mode 100644 index 0000000..51e0733 --- /dev/null +++ b/tests/unit/test_overlay_renderer.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from PIL import Image, ImageChops, ImageDraw + +from rfs.paper_to_image.generator import _repair_prompt +from rfs.paper_to_image.overlay_renderer import compose_semantic_overlay, normalize_visual_substrate_geometry, render_visual_substrate_blueprint +from rfs.paper_to_image.planner import compile_image_prompt +from rfs.paper_to_image.preparation import build_overlay_spec +from rfs.paper_to_image.semantic_blueprint import compile_semantic_blueprint + + +def _specification() -> dict: + return { + "summary": "Overlay test contract.", + "figure_goal": "Show a grounded linear pipeline.", + "topology": "linear", + "inputs": [{"id": "input", "name": "Input Image", "role": "input"}], + "modules": [ + {"id": "patches", "name": "Image Patches", "role": "intermediate"}, + {"id": "projection", "name": "Linear Projection", "role": "module"}, + ], + "outputs": [{"id": "output", "name": "Class", "role": "output"}], + "relations": [ + {"source": "input", "target": "patches", "type": "data_flow"}, + {"source": "patches", "target": "projection", "type": "data_flow"}, + {"source": "projection", "target": "output", "type": "prediction"}, + ], + "required_labels": ["Input Image", "Image Patches", "Linear Projection", "Class"], + } + + +class OverlayRendererTests(unittest.TestCase): + @staticmethod + def _drifted_visual_cards(semantic: dict, path: Path, *, omit_last: bool = False, duplicate_first: bool = False) -> None: + width, height = 900, 600 + image = Image.new("RGB", (width, height), "white") + draw = ImageDraw.Draw(image) + nodes = [item for item in semantic["nodes"] if isinstance(item, dict)] + if omit_last: + nodes = nodes[:-1] + for index, node in enumerate(nodes): + bbox = node["bbox_percent"] + center_x = (float(bbox["x"]) + float(bbox["w"]) / 2) * width + center_y = (float(bbox["y"]) + float(bbox["h"]) / 2) * height + card_w = float(bbox["w"]) * width * 1.22 + card_h = float(bbox["h"]) * height * 1.45 + shift_x = (-1 if index % 2 else 1) * width * 0.018 + shift_y = (-1 if index % 3 else 1) * height * 0.012 + box = ( + round(center_x - card_w / 2 + shift_x), + round(center_y - card_h / 2 + shift_y), + round(center_x + card_w / 2 + shift_x), + round(center_y + card_h / 2 + shift_y), + ) + draw.rounded_rectangle(box, radius=12, fill=(105 + index * 20, 155, 205 - index * 15), outline=(31, 80, 115), width=4) + separator = round(box[1] + (box[3] - box[1]) * 0.22) + draw.line((box[0] + 4, separator, box[2] - 4, separator), fill=(25, 70, 95), width=4) + draw.ellipse((box[0] + 18, separator + 12, box[0] + 55, separator + 49), fill=(235, 185, 72), outline=(40, 60, 70), width=3) + if duplicate_first and nodes: + left = round(width * 0.43) + top = round(height * 0.10) + draw.rounded_rectangle((left, top, left + 105, top + 85), radius=10, fill=(160, 130, 200), outline=(40, 40, 80), width=4) + image.save(path) + + def test_visual_substrate_contains_geometry_but_no_text_or_connectors(self): + semantic = compile_semantic_blueprint(_specification()) + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "visual_substrate.png" + + report = render_visual_substrate_blueprint(semantic, path, width=900, height=600) + + self.assertTrue(path.exists()) + self.assertFalse(report["contains_semantic_text"]) + self.assertFalse(report["contains_connectors"]) + self.assertEqual(report["node_count"], 4) + + def test_overlay_composes_every_exact_label_and_semantic_connector_path(self): + specification = _specification() + semantic = compile_semantic_blueprint(specification) + plan = { + "figure_specification": specification, + "paper_summary": {"summary": "Paper."}, + "design_plan": {"summary": "Design."}, + "layout_intent": {"summary": "Layout."}, + "visual_metaphors": {"summary": "Metaphors."}, + "style_plan": {"summary": "Style."}, + } + overlay = build_overlay_spec(plan) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + substrate = root / "substrate.png" + output = root / "composed.png" + report_path = root / "overlay.json" + render_visual_substrate_blueprint(semantic, substrate, width=900, height=600) + + report = compose_semantic_overlay(substrate, semantic, overlay, output, report_path=report_path) + + self.assertTrue(report["passed"]) + self.assertEqual(report["label_count"], 4) + self.assertEqual(report["connector_count"], 3) + patches_to_projection = next(item for item in report["connectors"] if item["source"] == "patches" and item["target"] == "projection") + semantic_relation = next(item for item in semantic["connectors"] if item["source"] == "patches" and item["target"] == "projection") + self.assertEqual(patches_to_projection["path_percent"], semantic_relation["path_percent"]) + saved = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual([item["text"] for item in saved["labels"]], specification["required_labels"]) + with Image.open(substrate) as before, Image.open(output) as after: + self.assertIsNotNone(ImageChops.difference(before.convert("RGB"), after.convert("RGB")).getbbox()) + + def test_visual_substrate_normalization_restores_exact_semantic_geometry_without_crop(self): + semantic = compile_semantic_blueprint(_specification()) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + guide = root / "guide.png" + raw = root / "raw.png" + normalized = root / "normalized.png" + report_path = root / "geometry.json" + render_visual_substrate_blueprint(semantic, guide, width=900, height=600) + self._drifted_visual_cards(semantic, raw) + + report = normalize_visual_substrate_geometry(raw, guide, semantic, normalized, report_path=report_path) + + self.assertTrue(normalized.exists()) + self.assertFalse(report["pre_normalization_alignment"]["passed"]) + self.assertTrue(report["post_normalization_alignment"]["passed"]) + self.assertTrue(report["post_normalization_alignment"]["unique_node_card_assignment"]) + self.assertFalse(report["post_normalization_alignment"]["semantic_crop_used"]) + self.assertEqual(report["matched_node_count"], len(semantic["nodes"])) + self.assertTrue(all(item["fit_policy"] == "contain_no_crop" for item in report["nodes"])) + self.assertTrue(all(not item["semantic_crop_used"] for item in report["nodes"])) + self.assertTrue(all(not item["background_trim"].get("semantic_crop_used", False) for item in report["nodes"])) + saved = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual(saved["post_normalization_alignment"]["node_geometry_source"], "semantic_plan_exact_bbox") + + def test_visual_substrate_normalization_rejects_missing_or_extra_cards(self): + semantic = compile_semantic_blueprint(_specification()) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + guide = root / "guide.png" + render_visual_substrate_blueprint(semantic, guide, width=900, height=600) + for name, options in (("missing", {"omit_last": True}), ("extra", {"duplicate_first": True})): + with self.subTest(name=name): + raw = root / f"{name}.png" + self._drifted_visual_cards(semantic, raw, **options) + with self.assertRaisesRegex(ValueError, "exact cardinality is required"): + normalize_visual_substrate_geometry(raw, guide, semantic, root / f"{name}_normalized.png") + + def test_visual_substrate_normalization_rejects_ambiguous_node_card_matching(self): + semantic = compile_semantic_blueprint(_specification()) + ambiguous = json.loads(json.dumps(semantic)) + ambiguous["nodes"][1]["bbox_percent"] = dict(ambiguous["nodes"][0]["bbox_percent"]) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + guide = root / "guide.png" + raw = root / "raw.png" + render_visual_substrate_blueprint(semantic, guide, width=900, height=600) + self._drifted_visual_cards(semantic, raw) + + with self.assertRaisesRegex(ValueError, "ambiguous"): + normalize_visual_substrate_geometry(raw, guide, ambiguous, root / "normalized.png") + + def test_overlay_prompt_forbids_image_model_text_and_arrows(self): + specification = _specification() + plan = { + "figure_specification": specification, + "design_plan": {}, + "layout_intent": {}, + "visual_metaphors": {}, + "style_plan": {}, + } + semantic = compile_semantic_blueprint(specification) + + prompt = compile_image_prompt( + plan, + {"aspect_ratio": "3:2", "language": "English"}, + selected_template={"template_id": "linear", "semantic_plan": semantic}, + deterministic_overlay=True, + ) + + self.assertIn("do not render any of them as visible text", prompt) + self.assertIn("do not render any arrow, line, connector, arrowhead", prompt) + self.assertIn("owned by overlay_spec.json", prompt) + self.assertNotIn("Every checklist item must appear at least once as clearly readable text", prompt) + + def test_overlay_repair_prompt_keeps_text_and_connectors_compiler_owned(self): + prompt = _repair_prompt( + {"preserve": ["correct visual substrate"], "remove": [], "repair_regions": [], "hard_errors": []}, + {"items": [{"issue_type": "aesthetic_quality", "instruction": "Improve visual hierarchy."}]}, + "Generate a visual-only substrate.", + {"figure_specification": _specification()}, + deterministic_overlay=True, + ) + + self.assertIn("Do not add, remove, rewrite, move, or repair any text", prompt) + self.assertIn("Repair only the image-rich visual substrate", prompt) + self.assertNotIn("Mandatory directed connector checklist after repair", prompt) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_package_boundaries.py b/tests/unit/test_package_boundaries.py new file mode 100644 index 0000000..bbecc7b --- /dev/null +++ b/tests/unit/test_package_boundaries.py @@ -0,0 +1,43 @@ +import unittest + + +class PackageBoundaryTests(unittest.TestCase): + def test_stable_package_entry_points_are_importable(self): + from rfs.analysis import parse_paper, plan_reference_layout + from rfs.composition import compile_ppt, render_rebuild_preview + from rfs.contracts import apply_paper_semantic_contract + from rfs.evaluation import run_rebuild_visual_quality_check + from rfs.generation import generate_and_select + from rfs.planning import plan_paper_image + from rfs.providers import call_vlm_json + from rfs.workflows import run_paper_to_editable + + for value in ( + parse_paper, + plan_reference_layout, + compile_ppt, + render_rebuild_preview, + apply_paper_semantic_contract, + run_rebuild_visual_quality_check, + generate_and_select, + plan_paper_image, + call_vlm_json, + run_paper_to_editable, + ): + self.assertTrue(callable(value)) + + def test_legacy_imports_remain_compatible(self): + from rfs.composition import compile_ppt as stable_compile + from rfs.contracts import apply_paper_semantic_contract as stable_contract + from rfs.paper_to_editable import run_paper_to_editable as legacy_workflow + from rfs.ppt_compiler import compile_ppt as legacy_compile + from rfs.semantic_contract import apply_paper_semantic_contract as legacy_contract + from rfs.workflows import run_paper_to_editable as stable_workflow + + self.assertIs(legacy_compile, stable_compile) + self.assertIs(legacy_contract, stable_contract) + self.assertIs(legacy_workflow, stable_workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py new file mode 100644 index 0000000..bc78d98 --- /dev/null +++ b/tests/unit/test_paper_analyzer.py @@ -0,0 +1,780 @@ +import json +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +import fitz + +from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _deadline_ocr_wave, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _repair_ocr_spacing, _section_coverage, _token_overlap_agreement, evidence_excerpt, overview_figure_panel_excerpt, overview_figure_spatial_excerpt, parse_paper +from rfs.paper_to_image.inspection import inspect_paper +from rfs.paper_to_image.document_cache import DOCUMENT_CACHE_VERSION, write_document_cache +from rfs.paper_to_image.preparation import _semantic_cache_safe +from rfs.reference_text_extractor import _positive_ocr_setting, _rapidocr_detector_limit, run_rapidocr_detailed + + +class PaperAnalyzerTests(unittest.TestCase): + def _pdf(self, draw) -> Path: + root = Path(tempfile.mkdtemp()) + target = root / "paper.pdf" + document = fitz.open() + page = document.new_page(width=600, height=800) + draw(page) + document.save(target) + document.close() + self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True)) + return target + + def _multipage_pdf(self, page_texts: list[list[str]]) -> Path: + root = Path(tempfile.mkdtemp()) + target = root / "paper.pdf" + document = fitz.open() + for blocks in page_texts: + page = document.new_page(width=600, height=800) + top = 50 + for text in blocks: + page.insert_textbox((45, top, 555, top + 110), text, fontsize=10) + top += 120 + document.save(target) + document.close() + self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True)) + return target + + def test_structured_pdf_preserves_page_blocks_and_captions(self): + def draw(page): + page.insert_text((50, 60), "1 Introduction", fontsize=14) + page.insert_textbox((50, 90, 550, 180), "A readable scientific paper paragraph with enough text for extraction quality checks.", fontsize=10) + page.insert_textbox((50, 220, 550, 280), "Figure 1: Overview of the proposed framework.", fontsize=10) + + parsed = parse_paper(self._pdf(draw)) + + self.assertEqual(parsed["page_count"], 1) + self.assertTrue(parsed["source_sha256"]) + self.assertTrue(parsed["pages"][0]["blocks"]) + self.assertEqual(parsed["document_index"]["figures"][0]["page"], 1) + self.assertIn("Introduction", parsed["headings"]) + self.assertTrue(all(item.get("block_id") for item in parsed["evidence"])) + + def test_bold_unnumbered_headings_create_section_boundaries(self): + def draw(page): + page.insert_text((50, 55), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((50, 75, 550, 145), "This paper introduces a reliable architecture with enough text for extraction quality checks.", fontsize=10) + page.insert_text((50, 185), "Model Architecture", fontname="hebo", fontsize=11) + page.insert_textbox((50, 205, 550, 285), "The encoder maps the input sequence into a representation used by the decoder and output layer.", fontsize=10) + page.insert_text((50, 325), "Encoder and Decoder Stacks", fontname="hebo", fontsize=10) + page.insert_textbox((50, 345, 550, 425), "Each stack contains attention and feed-forward modules with residual connections.", fontsize=10) + + parsed = parse_paper(self._pdf(draw)) + evidence_by_text = {item["text"]: item for item in parsed["evidence"]} + architecture_text = next(text for text in evidence_by_text if text.startswith("The encoder maps")) + stack_text = next(text for text in evidence_by_text if text.startswith("Each stack contains")) + + self.assertIn("Model Architecture", parsed["headings"]) + self.assertIn("Encoder and Decoder Stacks", parsed["headings"]) + self.assertEqual(evidence_by_text[architecture_text]["section_hint"], "Model Architecture") + self.assertEqual(evidence_by_text[stack_text]["section_hint"], "Encoder and Decoder Stacks") + self.assertGreaterEqual(parsed["extraction_report"]["typographic_heading_count"], 3) + + def test_multiline_figure_caption_is_combined_into_priority_evidence(self): + def draw(page): + page.insert_textbox((50, 80, 550, 150), "Figure 1: Three interconnected components: a promptable task,\na model, and a data engine for collecting a large dataset.", fontsize=10) + page.insert_textbox((50, 180, 550, 250), "1 Introduction\nA sufficiently long introduction paragraph for quality validation.", fontsize=10) + + parsed = parse_paper(self._pdf(draw)) + + caption = parsed["document_index"]["figures"][0]["caption"] + self.assertIn("data engine", caption) + self.assertTrue(any(item["kind"] == "caption" and "data engine" in item["text"] for item in parsed["evidence"])) + + def test_cross_column_caption_continuation_is_recovered(self): + def draw(page): + page.insert_textbox((40, 330, 290, 345), "Fig. 1 | Overview of the method. a, High-level pre-training and", fontsize=7) + page.insert_textbox((40, 345, 290, 360), "usage. b, The model architecture is adapted from the", fontsize=7) + page.insert_textbox((310, 330, 560, 345), "standard transformer encoder for two-dimensional tables.", fontsize=7) + page.insert_textbox((40, 390, 290, 470), "Body text in the left column must not enter the caption.", fontsize=9) + page.insert_textbox((310, 390, 560, 470), "Body text in the right column must not enter the caption.", fontsize=9) + + parsed = parse_paper(self._pdf(draw)) + captions = parsed["document_index"]["figures"] + + self.assertEqual(len(captions), 1) + self.assertIn("standard transformer encoder", captions[0]["caption"]) + self.assertNotIn("Body text", captions[0]["caption"]) + + def test_evidence_excerpt_prioritizes_labels_inside_overview_figure_region(self): + caption = "Figure 1: Overview of the proposed method and model architecture." + parsed = { + "pages": [{ + "page": 2, + "width": 600, + "height": 800, + "blocks": [ + {"id": "P002_B001", "reading_order": 1}, + {"id": "P002_B002", "reading_order": 2}, + {"id": "P002_B003", "reading_order": 3}, + {"id": "P002_B004", "reading_order": 4}, + ], + }], + "document_index": {"figures": [{"page": 2, "block_id": "P002_B004", "caption": caption}]}, + "evidence": [ + {"id": "E_CAP", "page": 2, "block_id": "P002_B004", "bbox": [40, 340, 560, 360], "kind": "caption", "text": caption}, + {"id": "E_FEATURE", "page": 2, "block_id": "P002_B001", "bbox": [310, 180, 410, 195], "kind": "paragraph", "text": "1D feature attention"}, + {"id": "E_SAMPLE", "page": 2, "block_id": "P002_B002", "bbox": [420, 180, 520, 195], "kind": "paragraph", "text": "1D sample attention"}, + {"id": "E_OUTPUT", "page": 2, "block_id": "P002_B003", "bbox": [500, 260, 550, 275], "kind": "paragraph", "text": "Predictions"}, + *[ + {"id": f"E_OTHER_{index}", "page": 3 + index, "block_id": f"P{3 + index:03d}_B001", "bbox": [40, 50, 560, 120], "kind": "caption", "text": "Figure 9: Architecture visualization. " + ("irrelevant analysis " * 20)} + for index in range(8) + ], + ], + } + + excerpt = evidence_excerpt(parsed, max_chars=700) + + self.assertIn("1D feature attention", excerpt) + self.assertIn("1D sample attention", excerpt) + self.assertIn("Predictions", excerpt) + spatial = json.loads(overview_figure_spatial_excerpt(parsed)) + by_text = {item["text"]: item for item in spatial} + self.assertEqual(by_text["1D feature attention"]["bbox_percent"], [0.5167, 0.225, 0.6833, 0.2437]) + + def test_panel_excerpt_treats_descriptive_dataset_headings_as_groups(self): + caption = "Figure 1: Overview. a, Pre-training and use. b, Model architecture." + parsed = { + "pages": [{"page": 1, "width": 600, "height": 800, "blocks": []}], + "document_index": {"figures": [{"page": 1, "block_id": "P001_B009", "caption": caption}]}, + "evidence": [ + {"id": "E_A", "page": 1, "block_id": "P001_B001", "bbox": [40, 80, 48, 92], "kind": "paragraph", "text": "a"}, + {"id": "E_A_GROUP", "page": 1, "block_id": "P001_B002", "bbox": [55, 105, 180, 120], "kind": "paragraph", "text": "A synthetic dataset"}, + {"id": "E_A_INPUT", "page": 1, "block_id": "P001_B003", "bbox": [55, 135, 120, 150], "kind": "paragraph", "text": "X train"}, + {"id": "E_B", "page": 1, "block_id": "P001_B004", "bbox": [310, 80, 318, 92], "kind": "paragraph", "text": "b"}, + {"id": "E_B_GROUP", "page": 1, "block_id": "P001_B005", "bbox": [330, 105, 500, 120], "kind": "paragraph", "text": "An arbitrary real-world dataset"}, + {"id": "E_B_INPUT", "page": 1, "block_id": "P001_B006", "bbox": [330, 135, 420, 150], "kind": "paragraph", "text": "Input dataset"}, + {"id": "E_CAP", "page": 1, "block_id": "P001_B009", "bbox": [40, 380, 560, 410], "kind": "caption", "text": caption}, + ], + } + + excerpt = json.loads(overview_figure_panel_excerpt(parsed)) + labels = { + item["text"]: item + for panel in excerpt["panels"] + for item in panel["label_inventory"] + } + + self.assertEqual(labels["A synthetic dataset"]["role_hint"], "group") + self.assertEqual(labels["An arbitrary real-world dataset"]["role_hint"], "group") + self.assertEqual(labels["X train"]["role_hint"], "input") + self.assertEqual(labels["Input dataset"]["role_hint"], "input") + + def test_two_column_blocks_are_sorted_left_then_right(self): + def draw(page): + page.insert_textbox((330, 100, 560, 180), "RIGHT COLUMN SECOND with enough words to form a complete paragraph.", fontsize=10) + page.insert_textbox((40, 100, 270, 180), "LEFT COLUMN FIRST with enough words to form a complete paragraph.", fontsize=10) + page.insert_textbox((40, 40, 560, 75), "2 Method", fontsize=14) + page.insert_textbox((330, 220, 560, 300), "RIGHT COLUMN FOURTH with enough words to form a complete paragraph.", fontsize=10) + page.insert_textbox((40, 220, 270, 300), "LEFT COLUMN THIRD with enough words to form a complete paragraph.", fontsize=10) + + parsed = parse_paper(self._pdf(draw)) + text = parsed["pages"][0]["text"] + + self.assertLess(text.index("LEFT COLUMN FIRST"), text.index("LEFT COLUMN THIRD")) + self.assertLess(text.index("LEFT COLUMN THIRD"), text.index("RIGHT COLUMN SECOND")) + self.assertGreaterEqual(parsed["pages"][0]["reading_order_confidence"], 0.8) + self.assertEqual(parsed["pages"][0]["column_count"], 2) + + def test_cross_extractor_token_agreement_ignores_column_order(self): + native = "left column first passage left column second passage right column first passage" + poppler = "left column first passage right column first passage left column second passage" + + self.assertEqual(_token_overlap_agreement(native, poppler), 1.0) + + def test_cross_extractor_token_agreement_ignores_equation_variable_layout(self): + native = "The projected point gives the origin and direction. ax x z ay y z az bz." + poppler = "The projected point gives the origin and direction. a x x/z a y y/z a z + b z." + + self.assertEqual(_token_overlap_agreement(native, poppler), 1.0) + + def test_cross_extractor_token_agreement_allows_poppler_hidden_text(self): + native = "The network architecture contains an encoder and decoder." + poppler = native + " hidden latex accessibility expansion with many additional formula tokens" + + self.assertEqual(_token_overlap_agreement(native, poppler), 1.0) + + def test_unicode_quality_gates_and_cjk_parser_agreement(self): + native = "摘要 本文提出一个多模态框架，用于图像编码与文本生成。" + poppler = "摘 要 本 文 提 出 一 个 多 模 态 框 架，用 于 图 像 编 码 与 文 本 生 成。" + + self.assertGreater(len(_normalized_compare_text(native)), 20) + self.assertEqual(_token_overlap_agreement(native, poppler), 1.0) + + def test_cjk_section_coverage_and_caption_kinds(self): + pages = [{"page": 1, "text": "摘要\n1 引言\n2 方法\n3 实验\n4 结论"}] + coverage = _section_coverage(pages, []) + + self.assertTrue(all(coverage.values())) + self.assertEqual(_block_kind("图 1 系统框架概览"), "caption") + self.assertEqual(_block_kind("表 2 实验结果"), "table") + self.assertEqual(_block_kind("2 方法"), "heading") + + def test_implicit_first_page_abstract_is_recognized(self): + lead = ( + "Tabular data are used across scientific fields. " + "Here we present a foundation model that learns an efficient prediction algorithm. " + + "This lead summary reports the problem, method, evidence and implications. " * 14 + ) + coverage = _section_coverage([{"page": 1, "text": lead}], [{"title": "Methods"}]) + + self.assertTrue(coverage["abstract"]) + self.assertTrue(coverage["method"]) + + def test_localized_caption_prefix_requires_a_real_identifier(self): + self.assertEqual(_block_kind("Figura 1: Arquitectura del sistema"), "caption") + self.assertEqual(_block_kind("Tabelle A2: Ablation results"), "table") + self.assertEqual(_block_kind("figura editable"), "paragraph") + self.assertEqual(_block_kind("tableau comparatif"), "paragraph") + self.assertEqual(_block_kind("Figures 1 and 2 outline our approach"), "paragraph") + + def test_three_column_blocks_are_sorted_column_major(self): + def draw(page): + page.insert_textbox((400, 100, 560, 170), "COLUMN THREE TOP complete scientific paragraph.", fontsize=9) + page.insert_textbox((220, 100, 380, 170), "COLUMN TWO TOP complete scientific paragraph.", fontsize=9) + page.insert_textbox((40, 100, 200, 170), "COLUMN ONE TOP complete scientific paragraph.", fontsize=9) + page.insert_textbox((400, 220, 560, 290), "COLUMN THREE BOTTOM complete scientific paragraph.", fontsize=9) + page.insert_textbox((220, 220, 380, 290), "COLUMN TWO BOTTOM complete scientific paragraph.", fontsize=9) + page.insert_textbox((40, 220, 200, 290), "COLUMN ONE BOTTOM complete scientific paragraph.", fontsize=9) + page.insert_textbox((40, 40, 560, 75), "3 Method Architecture and System Overview Across Three Columns", fontsize=13) + + parsed = parse_paper(self._pdf(draw)) + text = parsed["pages"][0]["text"] + + self.assertLess(text.index("COLUMN ONE TOP"), text.index("COLUMN ONE BOTTOM")) + self.assertLess(text.index("COLUMN ONE BOTTOM"), text.index("COLUMN TWO TOP")) + self.assertLess(text.index("COLUMN TWO BOTTOM"), text.index("COLUMN THREE TOP")) + self.assertEqual(parsed["pages"][0]["column_count"], 3) + self.assertEqual(parsed["extraction_report"]["max_column_count"], 3) + + def test_invalid_three_column_split_falls_back_to_two_columns(self): + items = [] + for index in range(6): + items.append({"bbox": [95, 300 + index * 30, 395, 322 + index * 30], "text": f"Complete left column scientific sentence number {index}."}) + items.append({"bbox": [412, 300 + index * 30, 713, 322 + index * 30], "text": f"Complete right column scientific sentence number {index}."}) + items.extend([ + {"bbox": [157, 96, 650, 120], "text": "BERT Pre-training of Deep Bidirectional Transformers"}, + {"bbox": [294, 125, 512, 148], "text": "Language Understanding"}, + ]) + + groups = _column_groups(items, 804) + + self.assertEqual(len(groups), 2) + + def test_rotated_page_uses_display_coordinate_space(self): + def draw(page): + page.insert_textbox((40, 40, 560, 85), "Rotated Paper Architecture Overview", fontsize=14) + page.insert_textbox((40, 120, 270, 220), "First scientific block with enough evidence for extraction.", fontsize=10) + page.insert_textbox((330, 120, 560, 220), "Second scientific block with enough evidence for extraction.", fontsize=10) + page.set_rotation(90) + + parsed = parse_paper(self._pdf(draw)) + page = parsed["pages"][0] + + self.assertEqual(page["rotation"], 90) + self.assertEqual((page["width"], page["height"]), (800.0, 600.0)) + self.assertTrue(all(0 <= block["bbox"][0] <= block["bbox"][2] <= page["width"] for block in page["blocks"])) + self.assertTrue(all(0 <= block["bbox"][1] <= block["bbox"][3] <= page["height"] for block in page["blocks"])) + self.assertEqual(parsed["extraction_report"]["rotated_pages"], [1]) + + def test_unicode_normalization_keeps_math_symbols(self): + def draw(page): + page.insert_text((50, 60), "3 Method", fontsize=14) + page.insert_textbox((50, 100, 550, 180), "Model accuracy is approximately 95 percent and beta parameters are preserved.", fontsize=10) + + parsed = parse_paper(self._pdf(draw)) + + self.assertEqual(parsed["extraction_report"]["replacement_character_rate"], 0.0) + self.assertEqual(parsed["extraction_report"]["mojibake_rate"], 0.0) + + def test_ocr_spacing_repair_restores_scientific_english_boundaries(self): + mapping = { + "Thescannedencoderpassesevidencetothescanned": ["The", "scanned", "encoder", "passes", "evidence", "to", "the", "scanned"], + "architectureoverviewwithencoder": ["architecture", "overview", "with", "encoder"], + "decoderandoutput": ["decoder", "and", "output"], + } + + repaired, count = _repair_ocr_spacing( + "2Method Thescannedencoderpassesevidencetothescanned decoderandoutput. Figure1:Scanned architectureoverviewwithencoder.", + splitter=lambda token: mapping.get(token, [token]), + ) + + self.assertIn("2 Method", repaired) + self.assertIn("The scanned encoder passes evidence to the scanned decoder and output", repaired) + self.assertIn("Figure 1: Scanned architecture overview with encoder", repaired) + self.assertGreaterEqual(count, 10) + + def test_ocr_spacing_repair_preserves_cjk_acronyms_and_known_compounds(self): + source = "多模态 ImageBind BIDIRECTIONALTRANSFORMER multimodal representation" + repaired, _ = _repair_ocr_spacing(source, splitter=lambda token: ["multi", "modal"]) + + self.assertEqual(repaired, source) + + def test_ocr_spacing_repair_restores_numbered_chinese_section_heading(self): + repaired, count = _repair_ocr_spacing("2\u65b9\u6cd5", splitter=lambda token: [token]) + + self.assertEqual(repaired, "2 \u65b9\u6cd5") + self.assertEqual(count, 1) + + def test_ocr_spacing_repair_restores_numbered_spanish_section_headings(self): + repaired, count = _repair_ocr_spacing("2Metodo 3Experimentos 4Conclusion Figura1", splitter=lambda token: [token]) + + self.assertEqual(repaired, "2 Metodo 3 Experimentos 4 Conclusion Figura 1") + self.assertEqual(count, 4) + + def test_ocr_spacing_repair_does_not_split_spanish_word_on_single_english_anchor(self): + repaired, count = _repair_ocr_spacing("procesamiento", splitter=lambda token: ["proc", "esa", "mien", "to"]) + + self.assertEqual(repaired, "procesamiento") + self.assertEqual(count, 0) + + def test_ocr_spacing_repair_restores_sentence_boundary_before_accented_latin_text(self): + repaired, count = _repair_ocr_spacing("documentos.El módulo continúa.Éste termina.", splitter=lambda token: [token]) + + self.assertEqual(repaired, "documentos. El módulo continúa. Éste termina.") + self.assertEqual(count, 2) + + def test_scanned_chinese_numbered_sections_form_real_section_boundaries(self): + path = self._multipage_pdf([[]]) + + def adapter(_image_path, _lang): + lines = [ + ("\u6458\u8981", 30), + ("\u672c\u6587\u63d0\u51fa\u6587\u6863\u7f16\u7801\u5668\u548c\u5173\u7cfb\u89e3\u7801\u5668,\u751f\u6210\u53ef\u7f16\u8f91\u6846\u67b6\u56fe\u3002", 90), + ("2\u65b9\u6cd5", 190), + ("\u8f93\u5165\u8bba\u6587\u8fdb\u5165\u6587\u6863\u7f16\u7801\u5668,\u7136\u540e\u7531\u5173\u7cfb\u89e3\u7801\u5668\u751f\u6210\u8f93\u51fa\u3002", 250), + ("3\u5b9e\u9a8c", 390), + ("\u5b9e\u9a8c\u8bc4\u4f30\u5b9e\u4f53\u53ec\u56de\u7387\u548c\u5173\u7cfb\u53ec\u56de\u7387\u3002", 450), + ("4\u7ed3\u8bba", 590), + ("\u8be5\u65b9\u6cd5\u4fdd\u6301\u8bc1\u636e\u9875\u7801\u548c\u6a21\u5757\u5173\u7cfb\u3002", 650), + ] + return [ + {"text": text, "confidence": 0.99, "quad": [[40, top], [900, top], [900, top + 45], [40, top + 45]]} + for text, top in lines + ] + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=adapter) + + self.assertEqual(parsed["headings"], ["\u6458\u8981", "\u65b9\u6cd5", "\u5b9e\u9a8c", "\u7ed3\u8bba"]) + self.assertEqual(parsed["extraction_report"]["section_count"], 4) + self.assertGreaterEqual(parsed["extraction_report"]["ocr_spacing_repair_count"], 3) + + def test_truncated_pdf_fails_before_parser_noise(self): + path = self._multipage_pdf([["Abstract", "A valid document before truncation."]]) + payload = path.read_bytes() + path.write_bytes(payload[: max(16, len(payload) // 2)]) + + with self.assertRaisesRegex(ValueError, "EOF marker is missing"): + parse_paper(path, ocr_engine="off") + + def test_non_pdf_payload_with_pdf_extension_fails_preflight(self): + root = Path(tempfile.mkdtemp()) + self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True)) + path = root / "not_really.pdf" + path.write_bytes(b"this is not a PDF document") + + with self.assertRaisesRegex(ValueError, "valid PDF header"): + parse_paper(path, ocr_engine="off") + + def test_encrypted_pdf_returns_explicit_password_error(self): + from pypdf import PdfReader, PdfWriter + + source = self._multipage_pdf([["Abstract", "Encrypted scientific content."]]) + target = source.parent / "encrypted.pdf" + reader = PdfReader(str(source)) + writer = PdfWriter() + for page in reader.pages: + writer.add_page(page) + writer.encrypt("secret") + with target.open("wb") as handle: + writer.write(handle) + + with self.assertRaisesRegex(ValueError, "encrypted"): + parse_paper(target, ocr_engine="off") + + def test_high_confidence_garbled_english_ocr_fails_lexical_gate(self): + path = self._multipage_pdf([[]]) + + def adapter(_image_path, _lang): + lines = [ + "Abstract", + "2 Method", + "LEFT TOP encoder evidenoe begins themetiod ecndevidn letang clumn", + "Figur oveview TOerevl edne beong complene lett comn", + "RIGHT BOTTOM outputvid finihes rnading order clumn evidn", + "3 Conclusion", + ] + return [ + {"text": text, "confidence": 0.92, "quad": [[40, 30 + index * 80], [900, 30 + index * 80], [900, 75 + index * 80], [40, 75 + index * 80]]} + for index, text in enumerate(lines) + ] + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=adapter) + report = parsed["extraction_report"] + + self.assertTrue(report["ocr_latin_lexical_gate_applied"]) + self.assertLess(report["ocr_latin_known_word_ratio"], 0.5) + self.assertEqual(report["status"], "fail") + + def test_low_text_page_uses_local_ocr_adapter(self): + path = self._pdf(lambda _page: None) + + parsed = parse_paper( + path, + ocr_engine="easyocr", + ocr_adapter=lambda _image, _lang: [{ + "text": "Abstract Method Image Encoder produces the final representation with reliable evidence.", + "confidence": 0.93, + "quad": [[10, 10], [900, 10], [900, 80], [10, 80]], + }], + ) + + self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1]) + self.assertTrue(parsed["pages"][0]["used_ocr"]) + self.assertIn("Image Encoder", parsed["pages"][0]["text"]) + + def test_cross_extractor_content_disagreement_still_triggers_ocr(self): + path = self._pdf(lambda page: page.insert_textbox( + (40, 80, 560, 220), + "Abstract Method The native parser returns a sufficiently long but unsupported passage about an encoder and decoder.", + fontsize=10, + )) + ocr_text = "Abstract Method The verified page contains an image encoder, transformer decoder, training objective, inference output, and complete scientific evidence for the recovered framework." + with patch("rfs.paper_to_image.analyzer._poppler_pages", return_value=(["Completely unrelated alternate parser content about chemistry molecules and laboratory measurements."], None)): + parsed = parse_paper( + path, + ocr_engine="easyocr", + ocr_adapter=lambda _image, _lang: [{ + "text": ocr_text, + "confidence": 0.96, + "quad": [[20, 20], [950, 20], [950, 120], [20, 120]], + }], + ) + + self.assertEqual(parsed["extraction_report"]["ocr_candidate_pages"], [1]) + self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1]) + self.assertIn("verified page", parsed["pages"][0]["text"]) + + def test_empty_ocr_result_returns_failed_document_model_instead_of_raising(self): + path = self._pdf(lambda _page: None) + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=lambda _image, _lang: []) + + self.assertEqual(parsed["extraction_report"]["status"], "fail") + self.assertEqual(parsed["evidence"], []) + self.assertEqual(parsed["extraction_report"]["pdf_type"], "scanned") + + def test_ocr_lines_are_grouped_into_complete_figure_caption(self): + path = self._pdf(lambda _page: None) + records = [ + {"text": "Figure 1: Overview of the proposed framework.", "confidence": 0.95, "quad": [[40, 100], [560, 100], [560, 125], [40, 125]]}, + {"text": "The image encoder feeds a transformer decoder.", "confidence": 0.94, "quad": [[40, 130], [560, 130], [560, 155], [40, 155]]}, + {"text": "1 Introduction", "confidence": 0.98, "quad": [[40, 210], [250, 210], [250, 240], [40, 240]]}, + {"text": "A sufficiently long scientific paragraph provides reliable evidence for the method.", "confidence": 0.96, "quad": [[40, 250], [560, 250], [560, 275], [40, 275]]}, + ] + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=lambda _image, _lang: records) + + self.assertEqual(parsed["document_index"]["figures"][0]["caption"], "Figure 1: Overview of the proposed framework. The image encoder feeds a transformer decoder.") + self.assertGreater(parsed["extraction_report"]["mean_ocr_confidence"], 0.9) + + def test_rapidocr_caption_group_survives_sentence_boundaries(self): + lines = [ + {"bbox": [40, 100, 560, 120], "text": "Fig. 1: Overview of the model.", "confidence": 0.98}, + {"bbox": [40, 124, 560, 144], "text": "The encoder produces features.", "confidence": 0.98}, + {"bbox": [40, 148, 560, 168], "text": "A decoder predicts the output.", "confidence": 0.98}, + {"bbox": [40, 210, 560, 230], "text": "We next describe the training objective.", "confidence": 0.98}, + ] + + grouped = _assign_ocr_parent_blocks(lines, 600) + + self.assertEqual({item["parent_block"] for item in grouped[:3]}, {1}) + self.assertNotEqual(grouped[2]["parent_block"], grouped[3]["parent_block"]) + + def test_ocr_margin_filter_removes_vertical_watermark_fragments(self): + records = [ + {"bbox": [130, 100 + index * 30, 730, 124 + index * 30], "text": f"Complete scientific body line number {index} with reliable method evidence."} + for index in range(6) + ] + records.extend([ + {"bbox": [35, 120, 70, 190], "text": "May"}, + {"bbox": [38, 210, 68, 245], "text": "28"}, + {"bbox": [90, 400, 180, 425], "text": "CNN"}, + ]) + + filtered, removed = _filter_ocr_margin_noise(records, 842) + + self.assertEqual(removed, 2) + self.assertIn("CNN", [item["text"] for item in filtered]) + + def test_ocr_margin_filter_removes_fragments_even_when_body_starts_left(self): + records = [ + {"bbox": [70, 300 + index * 30, 770, 324 + index * 30], "text": f"Long body line {index} establishes the left boundary near the page margin."} + for index in range(6) + ] + records.extend([ + {"bbox": [35, 120, 70, 190], "text": "May"}, + {"bbox": [40, 220, 66, 246], "text": "S"}, + {"bbox": [63, 40, 790, 75], "text": "End-to-End Object Detection with Transformers"}, + ]) + + filtered, removed = _filter_ocr_margin_noise(records, 842) + + self.assertEqual(removed, 2) + self.assertIn("End-to-End Object Detection with Transformers", [item["text"] for item in filtered]) + + def test_inspection_reuses_matching_document_cache(self): + path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 180), "Abstract Method A sufficiently long scientific paragraph for deterministic PDF inspection.", fontsize=10)) + with tempfile.TemporaryDirectory() as temp: + first = inspect_paper(path, temp, ocr_engine="off") + second = inspect_paper(path, temp, ocr_engine="off") + + self.assertTrue(first["document_model"]) + self.assertEqual(second["status"], "cached") + + def test_inspection_invalidates_stale_local_document_model(self): + path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 180), "Abstract Method A sufficiently long scientific paragraph for deterministic PDF inspection.", fontsize=10)) + with tempfile.TemporaryDirectory() as temp: + first = inspect_paper(path, temp, ocr_engine="off") + model_path = Path(first["document_model"]) + model = json.loads(model_path.read_text(encoding="utf-8")) + model["document_cache_version"] = DOCUMENT_CACHE_VERSION - 1 + model_path.write_text(json.dumps(model), encoding="utf-8") + + second = inspect_paper(path, temp, ocr_engine="off") + + self.assertNotEqual(second["status"], "cached") + refreshed = json.loads(model_path.read_text(encoding="utf-8")) + self.assertEqual(refreshed["document_cache_version"], DOCUMENT_CACHE_VERSION) + + def test_global_document_cache_reuses_parse_across_output_directories(self): + path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 220), "Abstract Method A sufficiently long scientific paragraph for global cache validation. The document contains repeated evidence, page-aware terminology, a complete method description, and enough characters to satisfy the readable-page quality gate.", fontsize=10)) + with tempfile.TemporaryDirectory() as cache, tempfile.TemporaryDirectory() as first_out, tempfile.TemporaryDirectory() as second_out, patch.dict("os.environ", {"RFS_CACHE_DIR": cache}, clear=False): + first = inspect_paper(path, first_out, ocr_engine="off") + second = inspect_paper(path, second_out, ocr_engine="off") + + self.assertFalse(first["document_cache_hit"]) + self.assertTrue(second["document_cache_hit"]) + + def test_evidence_ids_are_stable_across_repeated_parses(self): + path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 180), "1 Method\nA stable block of scientific evidence for repeated parsing.", fontsize=10)) + + first = parse_paper(path) + second = parse_paper(path) + + self.assertEqual([item["id"] for item in first["evidence"]], [item["id"] for item in second["evidence"]]) + self.assertTrue(all(item["id"].startswith("E_P") for item in first["evidence"])) + + def test_evidence_budget_preserves_late_document_coverage(self): + path = self._multipage_pdf([ + ["Abstract", "Early context " * 90], + ["1 Introduction", "Our data engine has three stages:", "assisted-manual, semi-automatic, and fully automatic.", "Introduction evidence " * 24], + ["2 Method", "The encoder transforms inputs into representations. " * 18], + ["3 Experiments", "Experimental settings and evaluation metrics. " * 18], + ["4 Analysis", "Ablation and error analysis evidence. " * 18], + ["5 Conclusion", "The proposed method improves the final prediction while preserving scientific evidence. " * 12], + ]) + + parsed = parse_paper(path, max_chars=1800) + evidence_pages = {item["page"] for item in parsed["evidence"]} + + self.assertEqual(evidence_pages, {1, 2, 3, 4, 5, 6}) + self.assertTrue(any(item["page"] == 6 and "Conclusion" in item["text"] for item in parsed["evidence"])) + self.assertTrue(any("three stages" in item["text"] for item in parsed["evidence"])) + self.assertTrue(any("assisted-manual" in item["text"] for item in parsed["evidence"])) + self.assertEqual(parsed["extraction_report"]["evidence_page_coverage_ratio"], 1.0) + self.assertLessEqual(parsed["extraction_report"]["evidence_char_count"], 1800) + + def test_ocr_priority_spreads_fully_scanned_long_document(self): + pages = [{"page": index, "text": ""} for index in range(1, 13)] + + selected, details = _prioritize_ocr_candidates(pages, list(range(1, 13)), 6) + + self.assertEqual(selected, [1, 2, 3, 4, 10, 6]) + self.assertEqual([item["rank"] for item in details], [1, 2, 3, 4, 5, 6]) + + def test_rapidocr_worker_count_is_bounded_and_adapter_safe(self): + with patch.dict("os.environ", {"RFS_OCR_WORKERS": "3"}, clear=False): + self.assertEqual(_rapidocr_worker_count("rapidocr", None, 6), min(3, __import__("os").cpu_count() or 1)) + self.assertEqual(_rapidocr_worker_count("rapidocr", lambda *_args: [], 6), 1) + self.assertEqual(_rapidocr_worker_count("easyocr", None, 6), 1) + + with patch.dict("os.environ", {"RFS_OCR_WORKERS": ""}, clear=False), patch("rfs.paper_to_image.analyzer.os.cpu_count", return_value=12): + self.assertEqual(_rapidocr_worker_count("rapidocr", None, 6), 4) + with patch.dict("os.environ", {"RFS_OCR_WORKERS": ""}, clear=False), patch("rfs.paper_to_image.analyzer.os.cpu_count", return_value=4): + self.assertEqual(_rapidocr_worker_count("rapidocr", None, 6), 2) + + def test_invalid_rapidocr_environment_setting_falls_back(self): + with patch.dict("os.environ", {"RFS_RAPIDOCR_THREADS": "invalid"}, clear=False): + self.assertEqual(_positive_ocr_setting(None, "RFS_RAPIDOCR_THREADS", 1), 1) + + def test_rapidocr_detector_limit_is_bounded_and_overridable(self): + with patch.dict("os.environ", {"RFS_RAPIDOCR_DET_LIMIT": "384"}, clear=False): + self.assertEqual(_rapidocr_detector_limit(), 384) + self.assertEqual(_rapidocr_detector_limit(128), 256) + self.assertEqual(_rapidocr_detector_limit(4096), 2048) + + def test_rapidocr_detailed_exposes_stage_timings(self): + fake_engine = lambda _path: ([([[0, 0], [10, 0], [10, 10], [0, 10]], "Method", 0.95)], [0.2, 0.0, 0.3]) + local = __import__("rfs.reference_text_extractor", fromlist=["_RAPIDOCR_LOCAL"])._RAPIDOCR_LOCAL + prior = getattr(local, "engines", None) + local.engines = {(1, 6, 384): fake_engine} + try: + records, diagnostics = run_rapidocr_detailed("unused.png", "en", detector_limit=384) + finally: + if prior is None: + delattr(local, "engines") + else: + local.engines = prior + + self.assertEqual(records[0]["text"], "Method") + self.assertEqual(diagnostics["detector_limit"], 384) + self.assertEqual(diagnostics["detection_seconds"], 0.2) + self.assertEqual(diagnostics["recognition_seconds"], 0.3) + self.assertEqual(diagnostics["inference_seconds"], 0.5) + + def test_ocr_priority_prefers_semantic_pages_before_coverage_anchors(self): + pages = [{"page": index, "text": ""} for index in range(1, 11)] + pages[3]["text"] = "Figure 1: Overview of the proposed architecture and system pipeline." + pages[7]["text"] = "8 Conclusion We summarize the method and its limitations." + + selected, details = _prioritize_ocr_candidates(pages, [1, 4, 6, 8, 10], 3) + + self.assertEqual(selected[:2], [4, 8]) + self.assertIn("overview_figure", details[0]["reasons"]) + self.assertIn("conclusion", details[1]["reasons"]) + + def test_scanned_pdf_report_records_adaptive_ocr_schedule(self): + path = self._multipage_pdf([[] for _ in range(12)]) + + def adapter(image_path, _lang): + page_number = int(Path(image_path).stem.rsplit("_", 1)[-1]) + return [{ + "text": f"Page {page_number} Abstract Method architecture evidence with enough reliable text for adaptive OCR scheduling and document recovery.", + "confidence": 0.97, + "quad": [[20, 20], [560, 20], [560, 70], [20, 70]], + }] + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=adapter, max_ocr_pages=6) + + self.assertEqual(parsed["extraction_report"]["ocr_priority_pages"], [1, 2, 3, 4, 10, 6]) + self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1, 2, 3, 4, 10, 6]) + self.assertEqual([item["page"] for item in parsed["extraction_report"]["ocr_priority"]], [1, 2, 3, 4, 10, 6]) + self.assertEqual(parsed["extraction_report"]["status"], "warning") + self.assertEqual(parsed["extraction_report"]["pdf_type"], "scanned") + self.assertEqual(parsed["extraction_report"]["semantic_scope"], "sampled_pages_only") + self.assertFalse(parsed["extraction_report"]["scientific_scope_complete"]) + self.assertTrue(parsed["extraction_report"]["ocr_schedule_complete"]) + self.assertTrue(parsed["extraction_report"]["ocr_run_complete"]) + + def test_three_high_confidence_scan_pages_are_usable_partial_evidence(self): + path = self._multipage_pdf([[] for _ in range(12)]) + + def adapter(image_path, _lang): + page_number = int(Path(image_path).stem.rsplit("_", 1)[-1]) + return [{ + "text": f"Page {page_number} Abstract Method architecture evidence with enough reliable text for a partial engineering contract.", + "confidence": 0.97, + "quad": [[20, 20], [560, 20], [560, 70], [20, 70]], + }] + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=adapter, max_ocr_pages=3) + report = parsed["extraction_report"] + + self.assertEqual(report["ocr_pages"], [1, 2, 3]) + self.assertTrue(report["sampled_scan_ready"]) + self.assertEqual(report["status"], "warning") + self.assertEqual(report["semantic_scope"], "sampled_pages_only") + self.assertFalse(report["scientific_scope_complete"]) + + def test_deadline_limited_ocr_is_not_cached_as_complete(self): + path = self._multipage_pdf([[] for _ in range(3)]) + + def adapter(_image_path, _lang): + return [{ + "text": "Abstract Method architecture evidence recovered by OCR.", + "confidence": 0.97, + "quad": [[20, 20], [560, 20], [560, 70], [20, 70]], + }] + + parsed = parse_paper(path, deadline_at=time.monotonic() - 1, ocr_engine="easyocr", ocr_adapter=adapter, max_ocr_pages=3) + report = parsed["extraction_report"] + + self.assertFalse(report["ocr_schedule_complete"]) + self.assertFalse(report["ocr_run_complete"]) + self.assertEqual(report["ocr_attempted_pages"], []) + self.assertTrue(any("schedule was not completed" in warning for warning in report["warnings"])) + with tempfile.TemporaryDirectory() as cache, patch.dict("os.environ", {"RFS_CACHE_DIR": cache}, clear=False): + self.assertIsNone(write_document_cache(path, parsed, ocr_engine="easyocr", ocr_lang="en_ch")) + self.assertFalse(_semantic_cache_safe(parsed)) + + def test_deadline_ocr_wave_preserves_completed_pages_and_terminates_slow_workers(self): + class FakeProcess: + def __init__(self, command, **_kwargs): + self.page = int(command[command.index("--page") + 1]) + self.threads = int(command[command.index("--threads") + 1]) + self.output = Path(command[command.index("--out") + 1]) + self.returncode = None + self.terminated = False + if self.page == 1: + self.output.write_text(json.dumps({ + "blocks": [{"text": "Method", "bbox": [0, 0, 10, 10], "confidence": 0.9}], + "engine": "rapidocr", + "error": None, + "diagnostics": {"recognition_seconds": 0.1}, + "elapsed_seconds": 0.2, + }), encoding="utf-8") + self.returncode = 0 + + def poll(self): + return self.returncode + + def terminate(self): + self.terminated = True + self.returncode = -15 + + def wait(self, timeout=None): + del timeout + return self.returncode + + def kill(self): + self.returncode = -9 + + def communicate(self, timeout=None): + del timeout + return "", "" + + created = [] + + def fake_popen(command, **kwargs): + process = FakeProcess(command, **kwargs) + created.append(process) + return process + + with patch("rfs.paper_to_image.analyzer.subprocess.Popen", side_effect=fake_popen): + results = _deadline_ocr_wave(Path("paper.pdf"), [1, 2], "rapidocr", "en", time.monotonic() + 20.05) + + by_page = {page: result for page, result, _elapsed in results} + self.assertIsNone(by_page[1][2]) + self.assertEqual(by_page[1][0][0]["text"], "Method") + self.assertTrue(by_page[1][3]["worker_process"]) + self.assertEqual(by_page[2][2], "OCR exceeded extraction deadline") + self.assertTrue(by_page[2][3]["timed_out"]) + self.assertTrue(next(item for item in created if item.page == 2).terminated) + + created.clear() + with patch("rfs.paper_to_image.analyzer.subprocess.Popen", side_effect=fake_popen): + single = _deadline_ocr_wave(Path("paper.pdf"), [1], "rapidocr", "en", time.monotonic() + 20.05) + self.assertIsNone(single[0][1][2]) + self.assertEqual(created[0].page, 1) + self.assertEqual(created[0].threads, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py new file mode 100644 index 0000000..b5811ff --- /dev/null +++ b/tests/unit/test_paper_contract.py @@ -0,0 +1,1983 @@ +import re +import unittest + +from rfs.paper_to_image.planner import validate_plan_grounding +from rfs.paper_to_image.preparation import _demote_unconnected_completion_entities, _detect_dense_overview_panels, _extract_overview_panels, _fast_cache_path, _merge_isolated_architecture_entities, _normalize_panel_graphs, _repair_cross_script_entity_labels, _repair_relation_evidence_after_grounding, build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract +from rfs.paper_to_image.contract_completion import _overview_candidates + + +class PaperContractTests(unittest.TestCase): + def test_panel_graph_normalization_keeps_local_instances_and_repairs_relation_evidence(self): + parsed = {"evidence": [ + {"id": "E_DATA", "page": 2, "text": "Dataset"}, + {"id": "E_MODEL", "page": 2, "text": "Model"}, + ]} + spec = {"panel_graphs": [{ + "id": "panel_a", + "panel_label": "a", + "label": "Workflow", + "nodes": [ + {"instance_id": "data", "name": "Dataset", "role": "input", "evidence_ids": ["E_DATA"]}, + {"instance_id": "model", "name": "Model", "role": "module", "evidence_ids": ["E_MODEL"]}, + ], + "relations": [{"source": "data", "target": "model", "type": "data_flow", "evidence_ids": []}], + }]} + + report = _normalize_panel_graphs(spec, parsed) + + self.assertEqual(report["node_count"], 2) + self.assertEqual(report["relation_count"], 1) + self.assertEqual(spec["panel_graphs"][0]["relations"][0]["evidence_ids"], ["E_DATA", "E_MODEL"]) + + def test_two_panel_overview_is_preserved_as_dense_multiframe(self): + parsed = { + "document_index": {"figures": [{ + "page": 2, + "block_id": "P002_B010", + "caption": "Fig. 1 | Overview of the method. a, High-level pre-training and usage. b, The model architecture.", + }]}, + "pages": [{ + "page": 2, + "height": 790, + "blocks": [ + {"id": "P002_B010", "reading_order": 10, "bbox": [40, 335, 290, 344], "font_size": 7, "text": "Fig. 1 | Overview of the method. a, High-level pre-training and usage."}, + {"id": "P002_B011", "reading_order": 11, "bbox": [306, 335, 560, 344], "font_size": 7, "text": "b, The model architecture."}, + ], + }], + "evidence": [ + {"id": "E_A", "block_id": "P002_B010", "page": 2, "text": "Fig. 1 | Overview of the method. a, High-level pre-training and usage."}, + {"id": "E_B", "block_id": "P002_B011", "page": 2, "text": "b, The model architecture."}, + ], + } + spec = { + "inputs": [{"id": "input", "name": "Input dataset", "evidence_ids": []}], + "modules": [{"id": "encoder", "name": "attention over features", "evidence_ids": []}], + "outputs": [{"id": "prediction", "name": "Prediction", "evidence_ids": []}], + "innovations": [], + } + + dense = _detect_dense_overview_panels(parsed) + panels = _extract_overview_panels(parsed, spec) + + self.assertTrue(dense["detected"]) + self.assertEqual(dense["panel_labels"], ["a", "b"]) + self.assertEqual([item["panel_label"] for item in panels], ["a", "b"]) + self.assertEqual(panels[0]["entity_ids"], ["input", "prediction"]) + self.assertEqual(panels[1]["entity_ids"], ["encoder"]) + + def test_extracts_cross_column_figure_one_panel_descriptions(self): + parsed = { + "document_index": {"figures": [{ + "page": 3, + "block_id": "P003_B010", + "caption": "Fig. 1 | Overview of Example. a, Flow chart showing the model architecture of Example.", + }]}, + "pages": [{ + "page": 3, + "height": 790, + "blocks": [ + {"id": "P003_B010", "reading_order": 10, "bbox": [40, 365, 290, 374], "font_size": 7, "text": "Fig. 1 | Overview of Example. a, Flow chart showing the model architecture of Example."}, + {"id": "P003_B011", "reading_order": 11, "bbox": [40, 376, 290, 385], "font_size": 7, "text": "The model maps the input through two encoders."}, + {"id": "P003_B067", "reading_order": 67, "bbox": [306, 366, 560, 375], "font_size": 7, "text": "b, Image tile-level pretraining using DINOv2."}, + {"id": "P003_B068", "reading_order": 68, "bbox": [306, 376, 560, 385], "font_size": 7, "text": "c, Slide-level pretraining with LongNet using masked autoencoder."}, + {"id": "P003_B069", "reading_order": 69, "bbox": [306, 435, 560, 445], "font_size": 8.25, "text": "Body text must not enter the caption."}, + ], + }], + "evidence": [ + {"id": "E_A", "block_id": "P003_B010", "text": "Fig. 1 | Overview of Example. a, Flow chart showing the model architecture of Example."}, + {"id": "E_A2", "block_id": "P003_B011", "text": "The model maps the input through two encoders."}, + {"id": "E_B", "block_id": "P003_B067", "text": "b, Image tile-level pretraining using DINOv2."}, + {"id": "E_C", "block_id": "P003_B068", "text": "c, Slide-level pretraining with LongNet using masked autoencoder."}, + ], + } + spec = { + "inputs": [{"id": "input", "name": "Input", "evidence_ids": ["E_A2"]}], + "modules": [], "outputs": [], "innovations": [], + } + + panels = _extract_overview_panels(parsed, spec) + + self.assertEqual([item["panel_label"] for item in panels], ["a", "b", "c"]) + self.assertEqual(panels[0]["label"], "Model architecture of Example") + self.assertEqual(panels[1]["label"], "Image tile-level pretraining using DINOv2") + self.assertEqual(panels[2]["label"], "Slide-level pretraining with LongNet using masked autoencoder") + self.assertEqual(panels[0]["entity_ids"], ["input"]) + + def test_supporting_details_are_exact_deduplicated_and_weak_verbs_are_dropped(self): + spec = { + "inputs": [], + "modules": [ + {"id": "class_token", "name": "Class Token", "evidence_ids": ["E_CLASS"]}, + {"id": "generate", "name": "Generate", "evidence_ids": ["E_GENERATE"]}, + ], + "outputs": [], "innovations": [], "relations": [], + "supporting_details": [ + {"id": "class_token", "name": "Class Token", "evidence_ids": ["E_CLASS"]}, + {"id": "class_token", "name": "Class Token", "evidence_ids": ["E_CLASS"]}, + ], + } + completion = {"added_entities": ["class_token", "generate"]} + parsed = {"evidence": [ + {"id": "E_CLASS", "page": 3, "text": "classification token."}, + {"id": "E_SYMBOL", "page": 3, "text": "[CLS] is the classification token."}, + {"id": "E_GENERATE", "page": 3, "text": "to generate contextualized embeddings, which can serve as the basis for downstream applications"}, + ]} + + _demote_unconnected_completion_entities(spec, completion, parsed) + _demote_unconnected_completion_entities(spec, completion, parsed) + + self.assertEqual(len(spec["supporting_details"]), 1) + self.assertEqual(spec["supporting_details"][0]["name"], "classification token") + self.assertEqual(spec["supporting_details"][0]["visual_cue"], "one distinct highlighted token block among repeated token blocks") + self.assertEqual(spec["supporting_details"][0]["optional_visible_label"], "[CLS]") + + def test_dense_overview_panels_and_local_details_are_not_flattened_into_peer_nodes(self): + parsed = { + "document_index": { + "figures": [{"page": 3, "caption": "Fig. 1 | Overview of Example. a, Model architecture."}], + }, + "pages": [{ + "page": 3, + "blocks": [ + {"text": "a"}, + {"text": "b"}, + {"text": "c"}, + {"text": "b, Tile-level pretraining."}, + {"text": "c, Slide-level pretraining."}, + ], + }], + "evidence": [ + {"id": "E_MAIN", "page": 3, "text": "Slide Encoder based on the LongNet architecture"}, + {"id": "E_EXACT", "page": 3, "text": "Slide Encoder (LongNet)"}, + {"id": "E_DETAIL", "page": 3, "text": "Classification token"}, + ], + } + spec = { + "inputs": [{"id": "input", "name": "Input", "evidence_ids": ["E_MAIN"]}], + "modules": [ + {"id": "encoder", "name": "Slide Encoder", "evidence_ids": ["E_MAIN"]}, + {"id": "longnet", "name": "LongNet", "evidence_ids": ["E_MAIN"]}, + {"id": "class_token", "name": "Class Token", "evidence_ids": ["E_DETAIL"]}, + ], + "outputs": [{"id": "output", "name": "Output", "evidence_ids": ["E_MAIN"]}], + "innovations": [], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow", "evidence_ids": ["E_MAIN"]}, + {"source": "encoder", "target": "output", "type": "data_flow", "evidence_ids": ["E_MAIN"]}, + ], + } + completion = {"added_entities": ["class_token"]} + + dense = _detect_dense_overview_panels(parsed) + merged = _merge_isolated_architecture_entities(spec, parsed) + demoted = _demote_unconnected_completion_entities(spec, completion) + + self.assertTrue(dense["detected"]) + self.assertEqual(dense["panel_labels"], ["a", "b", "c"]) + self.assertEqual(merged, ["longnet->encoder"]) + self.assertEqual(spec["modules"][0]["name"], "Slide Encoder (LongNet)") + self.assertEqual(demoted, ["class_token"]) + self.assertEqual(spec["supporting_details"][0]["visibility"], "unlabeled_visual_detail_not_global_node") + + def test_late_exact_grounding_repairs_boundary_relation_evidence(self): + parsed = { + "document_index": {"figures": [{"page": 2, "caption": "Figure 1: Model overview."}]}, + "pages": [], + "evidence": [ + {"id": "E_INPUT", "page": 2, "text": "Input dataset"}, + {"id": "E_MODEL", "page": 2, "text": "Input dataset enters the model architecture"}, + {"id": "E_OUTPUT", "page": 2, "text": "model architecture produces Predictions"}, + ], + } + spec = { + "inputs": [{"id": "input", "name": "Input dataset", "evidence_ids": ["E_INPUT", "E_MODEL"]}], + "modules": [{"id": "model", "name": "model architecture", "evidence_ids": ["E_MODEL", "E_OUTPUT"]}], + "outputs": [{"id": "output", "name": "Predictions", "evidence_ids": ["E_OUTPUT"]}], + "innovations": [], + "relations": [ + {"source": "input", "target": "model", "type": "data_flow", "evidence_ids": []}, + {"source": "model", "target": "output", "type": "prediction", "evidence_ids": []}, + ], + } + + repaired = _repair_relation_evidence_after_grounding(spec, parsed) + + self.assertTrue(all(item["evidence_ids"] for item in spec["relations"])) + self.assertEqual(repaired, ["input->model", "model->output"]) + + def test_peer_generic_inputs_converge_on_existing_shared_target(self): + evidence = [ + {"id": "E_XTRAIN", "page": 2, "text": "X train"}, + {"id": "E_YTRAIN", "page": 2, "text": "y train"}, + {"id": "E_XTEST", "page": 2, "text": "X test"}, + {"id": "E_LINEAR", "page": 2, "text": "linear encoders"}, + {"id": "E_OUTPUT", "page": 2, "text": "y test"}, + ] + parsed = { + "document_index": {"figures": [{"page": 2, "caption": "Figure 1: Model overview."}]}, + "pages": [], + "evidence": evidence, + } + plan = { + "paper_summary": {}, + "figure_specification": { + "inputs": [ + {"id": "xtrain", "name": "X train", "evidence_ids": ["E_XTRAIN"]}, + {"id": "ytrain", "name": "y train", "evidence_ids": ["E_YTRAIN"]}, + {"id": "xtest", "name": "X test", "evidence_ids": ["E_XTEST"]}, + ], + "modules": [{"id": "linear", "name": "linear encoders", "evidence_ids": ["E_LINEAR"]}], + "outputs": [{"id": "output", "name": "y test", "evidence_ids": ["E_OUTPUT"]}], + "innovations": [], + "relations": [ + {"source": "xtrain", "target": "linear", "type": "encoding", "evidence_ids": ["E_XTRAIN", "E_LINEAR"]}, + {"source": "ytrain", "target": "linear", "type": "encoding", "evidence_ids": ["E_YTRAIN", "E_LINEAR"]}, + {"source": "linear", "target": "output", "type": "prediction", "evidence_ids": ["E_LINEAR", "E_OUTPUT"]}, + ], + }, + } + + normalize_figure_contract(plan, parsed) + relations = plan["figure_specification"]["relations"] + + self.assertTrue(any(item["source"] == "xtest" and item["target"] == "linear" for item in relations)) + + def test_cross_script_symbol_hallucination_is_repaired_from_exact_evidence_span(self): + spec = { + "inputs": [{"id": "tiles", "name": "256 脳 256 image tiles", "evidence_ids": ["E1"]}], + "modules": [], + "outputs": [], + "innovations": [], + } + parsed = {"evidence": [{"id": "E1", "text": "serializes each WSI into a sequence of 256 × 256 image tiles in row-major order"}]} + + repaired = _repair_cross_script_entity_labels(spec, parsed) + + self.assertEqual(spec["inputs"][0]["name"], "256 × 256 image tiles") + self.assertEqual(repaired, ["tiles:256 脳 256 image tiles->256 × 256 image tiles"]) + + def test_cross_script_repair_restores_exact_multiplication_symbol_label(self): + parsed = { + "evidence": [{"id": "E_LAYER", "page": 2, "text": "2D TabPFN layer (12×)"}], + } + spec = { + "inputs": [], + "modules": [{"id": "layer", "name": "2D TabPFN layer (12脳)", "evidence_ids": ["E_LAYER"]}], + "outputs": [], + "innovations": [], + } + + repaired = _repair_cross_script_entity_labels(spec, parsed) + + self.assertEqual(spec["modules"][0]["name"], "2D TabPFN layer (12×)") + self.assertTrue(repaired) + + def test_overview_selection_rejects_late_scaling_figure_when_figure_one_is_a_true_model_overview(self): + parsed = { + "document_index": { + "figures": [ + {"page": 3, "caption": "Figure 1: Model overview. Images become patches and enter a Transformer encoder."}, + {"page": 16, "caption": "Figure 8: Scaling different model dimensions of the Vision Transformer."}, + ] + } + } + + candidates = _overview_candidates(parsed) + + self.assertEqual([item["page"] for item in candidates], [3]) + + def test_contract_prunes_appendix_only_diagram_label_and_ungrounded_innovation(self): + overview = "Figure 1: Model overview. We split an input image into image patches, linearly embed them, add position embeddings and a class token, and feed them to a Transformer Encoder for classification." + parsed = { + "page_count": 18, + "document_index": { + "figures": [ + {"page": 3, "caption": overview}, + {"page": 16, "caption": "Figure 8: Scaling different model dimensions of the Vision Transformer."}, + ] + }, + "evidence": [ + {"id": "E_OVERVIEW", "page": 3, "kind": "caption", "text": overview, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E_DEPTH", "page": 16, "kind": "paragraph", "text": "Depth", "section_hint": "APPENDIX", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "figure_goal": "Show the Vision Transformer model overview.", + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "Images are processed as patch sequences.", "evidence_ids": ["E_OVERVIEW"]}, + "inputs": [ + {"id": "image", "name": "Input Image", "evidence_ids": ["E_OVERVIEW"]}, + {"id": "depth", "name": "Depth", "evidence_ids": ["E_DEPTH"]}, + ], + "modules": [{"id": "encoder", "name": "Transformer Encoder", "evidence_ids": ["E_OVERVIEW"]}], + "outputs": [{"id": "classification", "name": "Classification", "evidence_ids": ["E_OVERVIEW"]}], + "innovations": [{"id": "innovation_class_token", "name": "Learnable Class Token", "evidence_ids": []}], + "relations": [ + {"source": "image", "target": "encoder", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + {"source": "depth", "target": "encoder", "type": "data_flow", "evidence_ids": ["E_DEPTH"]}, + {"source": "encoder", "target": "classification", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertNotIn("Depth", [item["name"] for item in spec["inputs"]]) + self.assertEqual(spec["innovations"], []) + self.assertFalse(any(item["source"] == "depth" or item["target"] == "depth" for item in spec["relations"])) + self.assertIn("Depth", plan["contract_completion_report"]["removed_out_of_scope_entities"]) + self.assertIn("Learnable Class Token", plan["contract_completion_report"]["removed_ungrounded_innovations"]) + + def test_vit_mlp_is_upgraded_to_one_mlp_head_instead_of_duplicating_the_prediction_path(self): + overview = "Figure 1: Model overview. The Transformer encoder representation is passed through an MLP head for classification." + parsed = { + "page_count": 4, + "document_index": {"figures": [{"page": 2, "caption": overview}]}, + "evidence": [{"id": "E_OVERVIEW", "page": 2, "kind": "caption", "text": overview, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "figure_goal": "Show image classification.", + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [ + {"id": "encoder", "name": "Transformer Encoder", "evidence_ids": ["E_OVERVIEW"]}, + {"id": "mlp", "name": "MLP", "role": "module", "evidence_ids": ["E_OVERVIEW"]}, + ], + "outputs": [{"id": "classification", "name": "Classification", "evidence_ids": ["E_OVERVIEW"]}], + "innovations": [], + "relations": [ + {"source": "encoder", "target": "mlp", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + {"source": "mlp", "target": "classification", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + labels = [item["name"] for item in spec["modules"]] + + self.assertEqual(labels.count("MLP Head"), 1) + self.assertNotIn("MLP", labels) + self.assertEqual(len([item for item in spec["relations"] if item["target"] == "classification"]), 1) + + def test_explicit_input_intermediate_path_removes_redundant_input_shortcut(self): + overview = "Figure 1: Input Image is split into Image Patches, followed by Linear Projection and a Transformer Encoder." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 2, "caption": overview}]}, + "evidence": [{"id": "E_OVERVIEW", "page": 2, "kind": "caption", "text": overview, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "figure_goal": "Show the patch pipeline.", + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "image", "name": "Input Image", "evidence_ids": ["E_OVERVIEW"]}], + "modules": [ + {"id": "patches", "name": "Image Patches", "role": "intermediate", "evidence_ids": ["E_OVERVIEW"]}, + {"id": "projection", "name": "Linear Projection", "evidence_ids": ["E_OVERVIEW"]}, + ], + "outputs": [], + "innovations": [], + "relations": [ + {"source": "image", "target": "projection", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + {"source": "image", "target": "patches", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + {"source": "patches", "target": "projection", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertNotIn(("image", "projection"), pairs) + self.assertIn(("image", "patches"), pairs) + self.assertIn(("patches", "projection"), pairs) + self.assertIn("image->projection", plan["contract_completion_report"]["removed_input_shortcuts"]) + + def _parsed(self): + return { + "evidence": [ + {"id": "E0001", "confidence": 1.0}, + {"id": "E0002", "confidence": 0.7, "source": "easyocr"}, + ] + } + + def test_contract_adds_topology_labels_and_separate_flows(self): + plan = { + "paper_summary": { + "research_problem": {"text": "Understand an input.", "evidence_ids": ["E0001"]}, + "central_claim": {"text": "The method refines its output.", "evidence_ids": ["E0001"]}, + "inputs": [{"id": "input", "name": "Input", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "Refined Output", "evidence_ids": ["E0001"]}], + "training_flow": [{"step": "Train", "evidence_ids": ["E0001"]}], + "inference_flow": [{"step": "Refine", "evidence_ids": ["E0001"]}], + "unknowns": [], + }, + "figure_specification": { + "modules": [{"id": "refiner", "name": "Refiner", "evidence_ids": ["E0001"]}], + "relations": [{"source": "output", "target": "refiner", "type": "feedback", "evidence_ids": ["E0001"]}], + "terminology": {"Refiner": "Refiner"}, + }, + "design_plan": {"reading_order": ["input", "refiner", "output"], "groups": []}, + } + + spec = normalize_figure_contract(plan, self._parsed()) + overlay = build_overlay_spec(plan) + + self.assertEqual(spec["topology"], "feedback") + self.assertEqual(spec["training_flow"][0]["step"], "Train") + self.assertEqual(spec["inference_flow"][0]["step"], "Refine") + self.assertIn("Refiner", spec["required_labels"]) + self.assertEqual(overlay["connectors"][0]["source"], "output") + self.assertTrue(any(item["text"] == "Refined Output" for item in overlay["labels"])) + + def test_contract_normalizes_scalar_claim_and_grounds_it_from_evidence(self): + parsed = { + "evidence": [ + { + "id": "E0001", + "page": 3, + "text": "The same architecture is used for pre-training and fine-tuning with task-specific output layers.", + "confidence": 1.0, + } + ] + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": "Unified architecture for pre-training and fine-tuning.", + "central_claim": "The same architecture is used for pre-training and fine-tuning with task-specific output layers.", + "modules": [{"id": "encoder", "name": "Encoder", "evidence_ids": ["E0001"]}], + "inputs": [], + "outputs": [], + "innovations": [], + "relations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + validation = validate_plan_grounding(plan, parsed) + + self.assertEqual(spec["central_claim"]["evidence_ids"], ["E0001"]) + self.assertEqual(spec["research_problem"]["evidence_ids"], ["E0001"]) + self.assertTrue(validation["ok"]) + + def test_validation_rejects_cross_script_translation_of_visible_labels(self): + parsed = { + "evidence": [ + {"id": "E0001", "text": "输入论文首先进入文档编码器,关系解码器连接实体,最后输出可编辑框架图。", "confidence": 1.0}, + ] + } + plan = { + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "Input document", "evidence_ids": ["E0001"]}], + "modules": [{"id": "encoder", "name": "Document encoder", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "可编辑框架图", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow", "label": "进入", "evidence_ids": ["E0001"]}, + {"source": "encoder", "target": "output", "type": "prediction", "label": "produces", "evidence_ids": ["E0001"]}, + ], + "terminology": {}, + } + } + + validation = validate_plan_grounding(plan, parsed) + + self.assertFalse(validation["ok"]) + self.assertTrue(any("Input document" in item for item in validation["errors"])) + self.assertTrue(any("Document encoder" in item for item in validation["errors"])) + self.assertTrue(any("produces" in item for item in validation["errors"])) + + def test_validation_accepts_verbatim_non_english_visible_labels(self): + parsed = { + "evidence": [ + {"id": "E0001", "text": "输入论文首先进入文档编码器,关系解码器连接实体,最后输出可编辑框架图。", "confidence": 1.0}, + ] + } + plan = { + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "输入论文", "evidence_ids": ["E0001"]}], + "modules": [{"id": "encoder", "name": "文档编码器", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "可编辑框架图", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow", "label": "进入", "evidence_ids": ["E0001"]}, + {"source": "encoder", "target": "output", "type": "prediction", "label": "", "evidence_ids": ["E0001"]}, + ], + "terminology": {"文档编码器": "文档编码器"}, + } + } + + self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + + def test_contract_repairs_cross_script_terminology_to_verbatim_source_key(self): + parsed = { + "evidence": [ + {"id": "E0001", "text": "输入论文首先进入文档编码器,关系解码器连接实体,最后输出可编辑框架图。", "confidence": 1.0}, + ] + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "输入论文", "evidence_ids": ["E0001"]}], + "modules": [{"id": "encoder", "name": "文档编码器", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "可编辑框架图", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [{"source": "input", "target": "encoder", "type": "data_flow", "label": "进入", "evidence_ids": ["E0001"]}], + "terminology": {"文档编码器": "Document encoder"}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["terminology"]["文档编码器"], "文档编码器") + self.assertIn("文档编码器", spec["required_labels"]) + self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + + def test_contract_grounds_exact_innovation_only_from_novelty_evidence(self): + parsed = { + "evidence": [ + {"id": "E0001", "text": "本文提出论文编码器和关系解码器,生成可编辑框架图。", "section_hint": "摘要", "confidence": 1.0}, + {"id": "E0002", "text": "A baseline also contains a relation decoder.", "section_hint": "Related Work", "confidence": 1.0}, + ] + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [{"id": "decoder", "name": "关系解码器", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [{"id": "innovation_decoder", "name": "关系解码器", "evidence_ids": []}], + "relations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["innovations"][0]["evidence_ids"], ["E0001"]) + self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + + def test_validation_rejects_visible_input_without_evidence(self): + parsed = {"evidence": [{"id": "E0001", "text": "The encoder produces an output.", "confidence": 1.0}]} + plan = { + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "Input Image", "evidence_ids": []}], + "modules": [{"id": "encoder", "name": "encoder", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [], + "relations": [], + "terminology": {}, + } + } + + validation = validate_plan_grounding(plan, parsed) + + self.assertFalse(validation["ok"]) + self.assertIn("inputs[0] has no evidence_ids", validation["errors"]) + + def test_contract_grounds_short_diagram_labels_only_on_overview_figure_page(self): + parsed = { + "document_index": {"figures": [{"page": 3, "caption": "Figure 1: Overall pre-training and fine-tuning procedures."}]}, + "evidence": [ + {"id": "E0001", "page": 3, "text": "T 1", "confidence": 1.0}, + {"id": "E0002", "page": 3, "text": "C", "confidence": 1.0}, + {"id": "E0003", "page": 8, "text": "C", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [{"id": "encoder", "name": "Encoder", "evidence_ids": ["E0001"]}], + "outputs": [ + {"id": "token_output", "name": "T 1", "evidence_ids": []}, + {"id": "class_output", "name": "C", "evidence_ids": []}, + ], + "innovations": [], + "relations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["outputs"][0]["evidence_ids"], ["E0001"]) + self.assertEqual(spec["outputs"][1]["evidence_ids"], ["E0002"]) + self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + + def test_validation_rejects_english_translation_in_spanish_paper(self): + evidence_text = "Proponemos un codificador de documentos que transforma el articulo en evidencia estructurada. El documento de entrada pasa al codificador y el decodificador de relaciones produce una salida editable para los resultados." + parsed = {"evidence": [{"id": "E0001", "text": evidence_text, "confidence": 1.0}]} + plan = { + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "Input Document", "evidence_ids": ["E0001"]}], + "modules": [{"id": "encoder", "name": "Document Encoder", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "Editable Output", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [], + "terminology": {}, + } + } + + validation = validate_plan_grounding(plan, parsed) + + self.assertFalse(validation["ok"]) + self.assertEqual(validation["detected_source_language"], "spanish") + self.assertTrue(any("translated from spanish" in item for item in validation["errors"])) + + def test_validation_accepts_verbatim_spanish_labels(self): + evidence_text = "Proponemos un codificador de documentos que transforma el articulo en evidencia estructurada. El documento de entrada pasa al codificador y el decodificador de relaciones produce una salida editable para los resultados." + parsed = {"evidence": [{"id": "E0001", "text": evidence_text, "confidence": 1.0}]} + plan = { + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "documento de entrada", "evidence_ids": ["E0001"]}], + "modules": [{"id": "encoder", "name": "codificador de documentos", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "salida editable", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [], + "terminology": {}, + } + } + + validation = validate_plan_grounding(plan, parsed) + + self.assertTrue(validation["ok"]) + self.assertEqual(validation["detected_source_language"], "spanish") + + def test_contract_clears_translated_relation_label_when_direction_is_grounded(self): + evidence_text = "El codificador de documentos produce evidencia estructurada para el decodificador de relaciones." + parsed = {"evidence": [{"id": "E0001", "text": evidence_text, "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [ + {"id": "encoder", "name": "codificador de documentos", "evidence_ids": ["E0001"]}, + {"id": "decoder", "name": "decodificador de relaciones", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [{"source": "encoder", "target": "decoder", "type": "data_flow", "label": "structured evidence", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["relations"][0]["label"], "") + self.assertEqual(plan["contract_completion_report"]["removed_unverbatim_relation_labels"], ["encoder->decoder:structured evidence"]) + self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + + def test_contract_preserves_verbatim_relation_label(self): + evidence_text = "El codificador produce evidencia estructurada para el decodificador." + parsed = {"evidence": [{"id": "E0001", "text": evidence_text, "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [ + {"id": "encoder", "name": "codificador", "evidence_ids": ["E0001"]}, + {"id": "decoder", "name": "decodificador", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [{"source": "encoder", "target": "decoder", "type": "data_flow", "label": "evidencia estructurada", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["relations"][0]["label"], "evidencia estructurada") + self.assertEqual(plan["contract_completion_report"]["removed_unverbatim_relation_labels"], []) + + def test_overlay_deduplicates_innovation_label_that_repeats_a_module(self): + parsed = {"evidence": [{"id": "E0001", "text": "Proponemos un codificador de documentos para procesar la entrada.", "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [{"id": "encoder", "name": "codificador de documentos", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "salida", "evidence_ids": ["E0001"]}], + "innovations": [{"id": "innovation_encoder", "name": "Codificador de documentos", "evidence_ids": ["E0001"]}], + "relations": [{"source": "innovation_encoder", "target": "output", "type": "data_flow", "label": "", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + overlay = build_overlay_spec(plan) + + self.assertEqual(len(spec["innovations"]), 1) + self.assertEqual([item["text"].casefold() for item in overlay["labels"]].count("codificador de documentos"), 1) + self.assertEqual(overlay["connectors"][0]["source"], "encoder") + self.assertEqual(overlay["connectors"][0]["target"], "output") + + def test_contract_grounds_declared_multword_entity_by_exact_paper_term(self): + parsed = { + "page_count": 6, + "evidence": [ + {"id": "E0001", "page": 6, "text": "Transformer encoder. First, a convolution reduces the channel dimension.", "confidence": 0.98}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "modules": [{"id": "encoder", "name": "Transformer Encoder", "evidence_ids": []}], + "inputs": [], + "outputs": [], + "innovations": [], + "relations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["modules"][0]["evidence_ids"], ["E0001"]) + self.assertIn("encoder", plan["contract_completion_report"]["grounded_entities"]) + + def test_contract_does_not_add_cross_modal_edges_to_connected_inputs(self): + parsed = {"evidence": [{"id": "E0001", "page": 1, "text": "图像进入图像编码器，论文文本进入文本编码器。", "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [ + {"id": "image", "name": "图像", "evidence_ids": ["E0001"]}, + {"id": "text", "name": "论文文本", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "image_encoder", "name": "图像编码器", "evidence_ids": ["E0001"]}, + {"id": "text_encoder", "name": "文本编码器", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [ + {"source": "image", "target": "image_encoder", "evidence_ids": ["E0001"]}, + {"source": "text", "target": "text_encoder", "evidence_ids": ["E0001"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertEqual(pairs, {("image", "image_encoder"), ("text", "text_encoder")}) + + def test_contract_leaves_unknown_input_unconnected_instead_of_guessing(self): + parsed = {"evidence": [{"id": "E0001", "page": 1, "text": "A scientific system contains two modules.", "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "signal", "name": "Scientific Signal", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "first", "name": "First Module", "evidence_ids": ["E0001"]}, + {"id": "second", "name": "Second Module", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertFalse(any(item["source"] == "signal" for item in spec["relations"])) + + def test_contract_normalizes_string_entities_and_relation_alias_fields(self): + parsed = { + "evidence": [ + {"id": "E0001", "page": 1, "text": "The Image Encoder sends representations to the Mask Decoder.", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": { + "unknowns": [], + "core_modules": ["Image Encoder", "Mask Decoder"], + }, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [], + "outputs": [], + "innovations": [], + "relations": [ + {"source_id": "Image Encoder", "target_id": "Mask Decoder", "relation_type": "feature_flow", "evidence_ids": ["E0001"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + validation = validate_plan_grounding(plan, parsed) + + self.assertEqual([item["name"] for item in spec["modules"]], ["Image Encoder", "Mask Decoder"]) + self.assertEqual(spec["relations"][0]["source"], spec["modules"][0]["id"]) + self.assertEqual(spec["relations"][0]["target"], spec["modules"][1]["id"]) + self.assertEqual(spec["relations"][0]["type"], "feature_flow") + self.assertTrue(validation["ok"]) + + def test_grounding_rejects_relation_without_evidence(self): + plan = { + "figure_specification": { + "research_problem": {"text": "Problem", "evidence_ids": ["E0001"]}, + "central_claim": {"text": "Claim", "evidence_ids": ["E0001"]}, + "modules": [ + {"id": "a", "name": "A", "evidence_ids": ["E0001"]}, + {"id": "b", "name": "B", "evidence_ids": ["E0001"]}, + ], + "relations": [{"source": "a", "target": "b", "type": "data_flow", "evidence_ids": []}], + "inputs": [], + "outputs": [], + "innovations": [], + } + } + + result = validate_plan_grounding(plan, self._parsed()) + + self.assertFalse(result["ok"]) + self.assertIn("relation a -> b has no evidence_ids", result["errors"]) + + def test_low_confidence_evidence_is_reported_as_uncertain(self): + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "modules": [{"id": "ocr_module", "name": "OCR Module", "evidence_ids": ["E0002"]}], + "relations": [], + "inputs": [], + "outputs": [], + "innovations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, self._parsed()) + + self.assertIn("ocr_module relies only on low-confidence OCR evidence", spec["uncertainties"]) + + def test_review_fact_ids_expand_to_page_evidence_and_encoder_group(self): + parsed = {"evidence": [{"id": "E0001", "confidence": 1.0}]} + review = {"modules": [{"id": "review_encoders", "evidence_ids": ["E0001"]}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "modules": [ + {"id": "image_encoder", "name": "Image Encoder", "evidence_ids": ["review_encoders"]}, + {"id": "text_encoder", "name": "Text Encoder", "evidence_ids": ["review_encoders"]}, + {"id": "audio_encoder", "name": "Audio Encoder", "evidence_ids": ["review_encoders"]}, + {"id": "joint", "name": "Joint Embedding Space", "evidence_ids": ["E0001"]}, + ], + "inputs": [{"id": "image", "name": "Image", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [], + "relations": [ + {"source": "image", "target": "image_encoder", "evidence_ids": ["E0001"]}, + {"source": "image_encoder", "target": "joint", "evidence_ids": ["E0001"]}, + ], + "terminology": {}, + }, + } + + expand_plan_evidence(plan, review, parsed) + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(plan["figure_specification"]["modules"][0]["evidence_ids"], ["E0001"]) + self.assertTrue(any(item.get("id") == "modality_encoders_group" for item in spec["modules"])) + self.assertTrue(any(item.get("source") == "image" and item.get("target") == "modality_encoders_group" for item in spec["relations"])) + + def test_contract_completion_adds_required_conditioning_and_output_boundary(self): + parsed = {"evidence": [{"id": "E0001", "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "modules": [ + {"id": "patches", "name": "Image Patches", "evidence_ids": ["E0001"]}, + {"id": "encoder", "name": "Transformer Encoder", "evidence_ids": ["E0001"]}, + {"id": "head", "name": "MLP Head", "evidence_ids": ["E0001"]}, + ], + "inputs": [{"id": "image", "name": "Input Image", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "prediction", "name": "Class Prediction", "evidence_ids": ["E0001"]}], + "relations": [ + {"source": "patches", "target": "encoder", "evidence_ids": ["E0001"]}, + {"source": "encoder", "target": "head", "evidence_ids": ["E0001"]}, + ], + "must_show": [{"text": "Class Token", "evidence_ids": ["E0001"]}], + "innovations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + class_token = next(item for item in spec["modules"] if item["name"] == "Class Token") + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + self.assertIn(("image", "patches"), pairs) + self.assertIn((class_token["id"], "encoder"), pairs) + self.assertIn(("head", "prediction"), pairs) + + def test_zero_shot_prediction_is_not_upgraded_to_classifier(self): + caption = "Fig. 1: The model offers zero-shot prediction of reporter assay readout." + parsed = { + "page_count": 2, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "modules": [{"id": "head", "name": "Prediction Head", "evidence_ids": ["E0001"]}], + "inputs": [], + "outputs": [ + {"id": "output_zero_shot_classifier", "name": "Zero-Shot Classifier", "evidence_ids": ["E0001"]}, + {"id": "output_zero_shot_prediction", "name": "Zero-shot Prediction", "evidence_ids": ["E0001"]}, + ], + "relations": [], + "innovations": [], + "must_show": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + output_names = {item["name"] for item in spec["outputs"]} + + self.assertIn("Zero-shot Prediction", output_names) + self.assertNotIn("Zero-Shot Classifier", output_names) + self.assertEqual(sum(item["name"] == "Zero-shot Prediction" for item in spec["outputs"]), 1) + + def test_feedback_contract_merges_formula_and_display_name_duplicates(self): + caption = "Figure 1: Input x is used to Generate an Initial Output, obtain Self-Feedback, Refine it into a Refined Output, and repeat the feedback loop." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [ + {"id": "input_x", "name": "input x", "evidence_ids": ["E0001"]}, + {"id": "input_input", "name": "Input", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "generate", "name": "Generate", "evidence_ids": ["E0001"]}, + {"id": "initial_output", "name": "Initial Output", "evidence_ids": ["E0001"]}, + {"id": "feedback", "name": "Self-Feedback", "evidence_ids": ["E0001"]}, + {"id": "refine", "name": "Refine", "evidence_ids": ["E0001"]}, + ], + "outputs": [ + {"id": "y_0", "name": "y 0", "evidence_ids": ["E0001"]}, + {"id": "y_t_plus_1", "name": "y t + 1", "evidence_ids": ["E0001"]}, + {"id": "refined_output", "name": "Refined Output", "evidence_ids": ["E0001"]}, + ], + "relations": [ + {"source": "input_x", "target": "generate", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "generate", "target": "y_0", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "y_0", "target": "feedback", "type": "feedback", "evidence_ids": ["E0001"]}, + {"source": "refine", "target": "y_t_plus_1", "type": "data_flow", "evidence_ids": ["E0001"]}, + ], + "innovations": [], + "must_show": [], + "terminology": {"y 0": "Initial Output", "y t + 1": "Refined Output"}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + all_entities = [item for field in ("inputs", "modules", "outputs") for item in spec[field]] + normalized_names = [re.sub(r"[^a-z0-9]+", "", item["name"].casefold()) for item in all_entities] + endpoint_ids = {item["id"] for item in all_entities} + + self.assertEqual(len(spec["inputs"]), 1) + self.assertTrue(normalized_names[0].startswith("input")) + self.assertEqual(sum(name == "initialoutput" for name in normalized_names), 1) + self.assertEqual(sum(name == "refinedoutput" for name in normalized_names), 1) + self.assertTrue(all(item["source"] in endpoint_ids and item["target"] in endpoint_ids for item in spec["relations"])) + self.assertEqual(len(spec["required_labels"]), len({_normalized.casefold() for _normalized in spec["required_labels"]})) + + def test_feedback_contract_removes_parallel_model_path_and_keeps_one_eight_edge_loop(self): + caption = "Figure 1: Input x is used to Generate an Initial Output, obtain Self-Feedback from FEEDBACK, REFINE it into a Refined Output, and iterate." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "feedback", + "inputs": [{"id": "input", "name": "input x", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "model", "name": "Model M", "evidence_ids": ["E0001"]}, + {"id": "generate", "name": "Generate", "evidence_ids": ["E0001"]}, + {"id": "initial", "name": "Initial Output", "evidence_ids": ["E0001"]}, + {"id": "feedback", "name": "FEEDBACK", "evidence_ids": ["E0001"]}, + {"id": "self_feedback", "name": "Self-Feedback", "evidence_ids": ["E0001"]}, + {"id": "refine", "name": "REFINE", "evidence_ids": ["E0001"]}, + ], + "outputs": [{"id": "refined", "name": "Refined output", "evidence_ids": ["E0001"]}], + "innovations": [{"id": "iteration", "name": "Iterative self-refinement", "evidence_ids": ["E0001"]}], + "relations": [ + {"source": "input", "target": "model", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "model", "target": "initial", "type": "prediction", "evidence_ids": ["E0001"]}, + {"source": "feedback", "target": "model", "type": "conditioning", "evidence_ids": ["E0001"]}, + {"source": "model", "target": "refine", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "input", "target": "generate", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "generate", "target": "initial", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "initial", "target": "feedback", "type": "evaluation", "evidence_ids": ["E0001"]}, + {"source": "feedback", "target": "self_feedback", "type": "feedback", "evidence_ids": ["E0001"]}, + {"source": "initial", "target": "refine", "type": "revision_input", "evidence_ids": ["E0001"]}, + {"source": "self_feedback", "target": "refine", "type": "feedback", "evidence_ids": ["E0001"]}, + {"source": "refine", "target": "refined", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "refined", "target": "feedback", "type": "feedback_loop", "evidence_ids": ["E0001"]}, + ], + "must_show": [], + "terminology": {}, + "repeatable_labels": ["Model M"], + }, + "design_plan": {"summary": "Old plan", "reading_order": ["iteration"], "preserve": ["Iterative self-refinement"]}, + "layout_intent": {"summary": "Old layout"}, + "visual_metaphors": {"summary": "Old metaphors", "items": [{"module_id": "iteration", "metaphor": "Iterative self-refinement"}]}, + } + + spec = normalize_figure_contract(plan, parsed) + synchronize_plan_to_contract(plan) + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertEqual(len(spec["relations"]), 8) + self.assertEqual(spec["innovations"], []) + self.assertFalse(any("model" in pair for pair in pairs)) + self.assertIn(("input", "generate"), pairs) + self.assertIn(("refined", "feedback"), pairs) + self.assertNotIn("Iterative self-refinement", spec["must_show"]) + self.assertEqual(plan["design_plan"]["preserve"], spec["required_labels"]) + + def test_branch_contract_merges_head_output_aliases_and_removes_cross_branch_edges(self): + caption = "Figure 1: An Input Image passes through a Backbone and RPN to RoIAlign, followed by parallel classification, bounding-box regression, and mask branches." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "branch", + "inputs": [{"id": "image", "name": "Input Image", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "backbone", "name": "Backbone", "evidence_ids": ["E0001"]}, + {"id": "rpn", "name": "RPN", "evidence_ids": ["E0001"]}, + {"id": "proposals", "name": "Region Proposals", "evidence_ids": ["E0001"]}, + {"id": "roi", "name": "RoIAlign", "evidence_ids": ["E0001"]}, + {"id": "class_branch", "name": "classification branch", "evidence_ids": ["E0001"]}, + {"id": "box_branch", "name": "bounding-box regression", "evidence_ids": ["E0001"]}, + {"id": "mask_branch", "name": "mask branch", "evidence_ids": ["E0001"]}, + ], + "outputs": [ + {"id": "class_output", "name": "Classification", "evidence_ids": ["E0001"]}, + {"id": "box_output", "name": "bounding-box offset", "evidence_ids": ["E0001"]}, + {"id": "mask_output", "name": "binary mask", "evidence_ids": ["E0001"]}, + ], + "relations": [ + {"source": "image", "target": "backbone", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "backbone", "target": "rpn", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "proposals", "target": "roi", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "roi", "target": "class_branch", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "roi", "target": "box_branch", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "roi", "target": "mask_branch", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "mask_branch", "target": "box_output", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "mask_branch", "target": "mask_output", "type": "data_flow", "evidence_ids": ["E0001"]}, + ], + "innovations": [], + "must_show": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + names = [re.sub(r"[^a-z0-9]+", "", item["name"].casefold()) for field in ("inputs", "modules", "outputs") for item in spec[field]] + branch_names = [name for name in names if name in {"classification", "classificationbranch", "boundingboxregression", "boxbranch", "maskbranch"}] + endpoint_by_name = {re.sub(r"[^a-z0-9]+", "", item["name"].casefold()): item["id"] for field in ("inputs", "modules", "outputs") for item in spec[field]} + branch_ids = {endpoint_by_name[name] for name in branch_names} + + self.assertEqual(len(spec["inputs"]), 1) + self.assertEqual(sum(name in {"rpn", "regionproposals"} for name in names), 1) + self.assertEqual(len(branch_names), 3) + self.assertFalse(any(item["source"] in branch_ids and item["target"] in branch_ids and item["source"] != item["target"] for item in spec["relations"])) + + def test_multimodal_contract_keeps_shared_embedding_story_and_removes_training_detail(self): + caption = "Figure 1: ImageBind uses images to bind text, audio, depth, thermal, and IMU in one joint embedding space with emergent cross-modal alignment." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "branch", + "inputs": [ + *[{"id": name.lower(), "name": name, "evidence_ids": ["E0001"]} for name in ("Image", "Text", "Audio", "Depth", "Thermal", "IMU")], + {"id": "images_alias", "name": "Images", "evidence_ids": ["E0001"]}, + {"id": "class_descriptions", "name": "Class Descriptions", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "modality_encoders", "name": "Modality encoders", "evidence_ids": ["E0001"]}, + {"id": "common_space", "name": "Common embedding space", "evidence_ids": ["E0001"]}, + {"id": "text_encoder", "name": "Text Encoder", "evidence_ids": ["E0001"]}, + {"id": "contrastive", "name": "Contrastive Learning", "evidence_ids": ["E0001"]}, + ], + "outputs": [{"id": "joint_space", "name": "Joint Embedding Space", "evidence_ids": ["E0001"]}], + "innovations": [{"id": "emergent", "name": "Emergent Cross-modal Alignment", "evidence_ids": ["E0001"]}], + "relations": [], + "must_show": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + labels = [_normalized for field in ("inputs", "modules", "outputs") for item in spec[field] if (_normalized := re.sub(r"[^a-z0-9]+", "", item["name"].casefold()))] + + self.assertEqual(spec["topology"], "multimodal") + self.assertEqual(len(spec["inputs"]), 6) + self.assertEqual(labels.count("modalityencoders"), 1) + self.assertEqual(labels.count("jointembeddingspace"), 1) + self.assertEqual(labels.count("emergentcrossmodalalignment"), 1) + self.assertNotIn("classdescriptions", labels) + self.assertNotIn("textencoder", labels) + self.assertNotIn("contrastivelearning", labels) + self.assertEqual(len(spec["relations"]), 8) + self.assertIn("Emergent Cross-modal Alignment", spec["required_labels"]) + + def test_dense_multiframe_contract_keeps_task_model_and_data_engine_panels(self): + caption = "Figure 1: Three interconnected components: a promptable segmentation task, a segmentation model (SAM), and a data engine for collecting SA-1B." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "branch", + "inputs": [ + {"id": "image", "name": "image", "evidence_ids": ["E0001"]}, + {"id": "images_alias", "name": "Images", "evidence_ids": ["E0001"]}, + {"id": "prompts", "name": "input prompts", "evidence_ids": ["E0001"]}, + {"id": "prompt", "name": "Prompt", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "task", "name": "Prompt-able segmentation task", "role": "overview_panel", "evidence_ids": ["E0001"]}, + {"id": "sam", "name": "Segmentation model (SAM)", "role": "overview_panel", "evidence_ids": ["E0001"]}, + {"id": "engine", "name": "Data engine", "role": "overview_panel", "evidence_ids": ["E0001"]}, + {"id": "image_encoder", "name": "heavyweight image encoder", "evidence_ids": ["E0001"]}, + {"id": "prompt_encoder", "name": "prompt encoder", "evidence_ids": ["E0001"]}, + {"id": "mask_decoder", "name": "lightweight mask decoder", "evidence_ids": ["E0001"]}, + {"id": "image_embeddings", "name": "Image Embeddings", "evidence_ids": ["E0001"]}, + {"id": "transformer", "name": "Transformer Encoder", "evidence_ids": ["E0001"]}, + {"id": "position", "name": "Positional Encoding", "evidence_ids": ["E0001"]}, + {"id": "modality_group", "name": "Modality Encoders", "evidence_ids": ["E0001"]}, + {"id": "assisted", "name": "Assisted-Manual", "evidence_ids": ["E0001"]}, + {"id": "semi", "name": "Semi-Automatic", "evidence_ids": ["E0001"]}, + {"id": "fully", "name": "Fully Automatic", "evidence_ids": ["E0001"]}, + ], + "outputs": [ + {"id": "mask", "name": "Valid Segmentation Mask", "evidence_ids": ["E0001"]}, + {"id": "sa1b", "name": "SA-1B", "evidence_ids": ["E0001"]}, + {"id": "confidence", "name": "confidence scores", "evidence_ids": ["E0001"]}, + ], + "innovations": [], + "relations": [], + "must_show": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + labels = [re.sub(r"[^a-z0-9]+", "", item["name"].casefold()) for field in ("inputs", "modules", "outputs") for item in spec[field]] + + self.assertEqual(spec["topology"], "dense_multiframe") + self.assertEqual(len(labels), 13) + self.assertEqual(len(spec["relations"]), 9) + self.assertIn("promptablesegmentationtask", labels) + self.assertIn("segmentanythingmodel", labels) + self.assertIn("dataengine", labels) + self.assertIn("imageencoder", labels) + self.assertIn("maskdecoder", labels) + self.assertNotIn("imageembeddings", labels) + self.assertNotIn("transformerencoder", labels) + self.assertNotIn("positionalencoding", labels) + self.assertNotIn("modalityencoders", labels) + self.assertNotIn("confidencescores", labels) + + def test_overview_caption_and_stage_list_complete_missing_panels(self): + caption = "Figure 1: Three interconnected components: a promptable segmentation task, a segmentation model (SAM) that powers data annotation, and a data engine for collecting SA-1B, our dataset." + parsed = { + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}, + {"id": "E0002", "page": 2, "kind": "paragraph", "text": "Our data engine has three stages:", "confidence": 1.0}, + {"id": "E0003", "page": 2, "kind": "paragraph", "text": "assisted-manual, semi-automatic, and fully automatic.", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "modules": [], "inputs": [], "outputs": [], "innovations": [], "relations": [], "must_show": [], "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + labels = {_item["name"] for _item in spec["modules"]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn("Promptable segmentation task", labels) + self.assertIn("Segmentation model (SAM)", labels) + self.assertIn("Data engine", labels) + self.assertIn("Assisted-Manual", labels) + self.assertTrue(any(item.get("name") == "SA-1B" for item in spec["outputs"])) + self.assertTrue(any("stage_assisted_manual" in source and "stage_semi_automatic" in target for source, target in pairs)) + + def test_generic_completion_recovers_detr_without_bibliography_noise(self): + captions = [ + "Fig. 1: DETR directly predicts the final set of detections by combining a common CNN with a transformer architecture. During training, bipartite matching uniquely assigns predictions with ground truth boxes and class predictions.", + "Fig. 2: DETR uses a conventional CNN backbone for an input image, supplements features with a positional encoding, passes them into a transformer encoder, and uses object queries in a transformer decoder. A feed forward network predicts a class and bounding box.", + ] + parsed = { + "page_count": 12, + "document_index": {"figures": [{"page": index + 1, "caption": text} for index, text in enumerate(captions)]}, + "evidence": [ + {"id": f"E000{index + 1}", "page": index + 1, "kind": "caption", "text": text, "confidence": 1.0} + for index, text in enumerate(captions) + ] + [{"id": "E0099", "page": 11, "kind": "heading", "text": "Learning non-maximum suppression. In: ECCV", "section_hint": "References", "confidence": 1.0}], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + labels = {_item.get("name") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + + self.assertIn("Object Queries", labels) + self.assertIn("Bipartite Matching", labels) + self.assertIn("Bounding Box Predictions", labels) + self.assertNotIn("Non-Maximum Suppression", labels) + self.assertIn((ids["Object Queries"], ids["Transformer Decoder"]), pairs) + self.assertIn((ids["Feed Forward Network"], ids["Bounding Box Predictions"]), pairs) + + def test_generic_completion_keeps_clip_modalities_and_encoders_distinct(self): + caption = "Figure 1. Summary of our approach. CLIP jointly trains an image encoder and a text encoder to predict the correct pairings of a batch of (image, text) training examples. At test time the text encoder synthesizes a zero-shot linear classifier by embedding the names or descriptions of the target classes." + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 3, "kind": "paragraph", "text": "The image encoder and text encoder maximize similarity of image and text embeddings using a contrastive objective.", "section_hint": "Method", "confidence": 1.0}, + ], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [{"id": "fallback_contrastive", "name": "Contrastive Pre-training", "role": "paper-derived stage requiring VLM verification", "evidence_ids": ["E0002"]}], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertNotEqual(ids["Text"], ids["Text Encoder"]) + self.assertIn((ids["Text"], ids["Text Encoder"]), pairs) + self.assertIn((ids["Class Descriptions"], ids["Text Encoder"]), pairs) + self.assertIn((ids["Text Encoder"], ids["Text Embeddings"]), pairs) + self.assertTrue(any(item.get("id") == "fallback_contrastive" for item in spec["modules"])) + self.assertEqual(spec["topology"], "multimodal") + + def test_rich_clip_contract_reuses_singular_embedding_entities(self): + caption = "Figure 1. CLIP trains an image encoder and a text encoder on paired image and text inputs, then embeds class descriptions for a zero-shot classifier." + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 3, "kind": "paragraph", "text": "The image and text embeddings are aligned by a contrastive objective.", "section_hint": "Method", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [ + {"id": "images", "name": "Image", "evidence_ids": ["E0001"]}, + {"id": "texts", "name": "Text", "evidence_ids": ["E0001"]}, + {"id": "classes", "name": "Class Descriptions", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "image_encoder", "name": "Image Encoder", "evidence_ids": ["E0001"]}, + {"id": "text_encoder", "name": "Text Encoder", "evidence_ids": ["E0001"]}, + {"id": "contrastive", "name": "Contrastive Learning", "evidence_ids": ["E0002"]}, + ], + "outputs": [ + {"id": "image_embedding", "name": "image embedding", "evidence_ids": ["E0002"]}, + {"id": "text_embedding", "name": "text embedding", "evidence_ids": ["E0002"]}, + {"id": "classifier", "name": "Zero-Shot Classifier", "evidence_ids": ["E0001"]}, + ], + "relations": [{"source": "image_embedding", "target": "classifier", "type": "classification", "evidence_ids": ["E0001", "E0002"]}], + "innovations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + embedding_labels = [ + item.get("name") + for field in ("inputs", "modules", "outputs") + for item in spec[field] + if "embedding" in str(item.get("name") or "").casefold() + ] + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertEqual(len([label for label in embedding_labels if "image" in label.casefold()]), 1) + self.assertEqual(len([label for label in embedding_labels if "text" in label.casefold()]), 1) + self.assertIn(("text_embedding", "classifier"), pairs) + + def test_rich_bert_contract_uses_selected_caption_for_context_rules(self): + caption = "Figure 2: BERT input representation. The input embeddings are the sum of the token embeddings, the segmentation embeddings and the position embeddings." + parsed = { + "page_count": 12, + "document_index": {"figures": [{"page": 5, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 5, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [{"id": "tokens", "name": "Input Sequence", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "token_embeddings", "name": "Token Embeddings", "evidence_ids": ["E0001"]}, + {"id": "position_embeddings", "name": "Position Embeddings", "evidence_ids": ["E0001"]}, + {"id": "bert", "name": "Bidirectional Transformer Encoder", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [{"source": "tokens", "target": "token_embeddings", "type": "embedding", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + ids = {item.get("name"): item.get("id") for field in ("inputs", "modules", "outputs") for item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn("Segment Embeddings", ids) + self.assertIn("Input Representation", ids) + self.assertIn((ids["Segment Embeddings"], ids["Input Representation"]), pairs) + + def test_rich_nerf_contract_reuses_parenthesized_mlp_and_recovers_distant_method_step(self): + overview = "Figure 2: A neural radiance field samples 5D coordinates along camera rays and uses an MLP to predict color and volume density for volume rendering." + encoding = "We transform each sampled 3D point with a positional encoding before passing it into the MLP." + parsed = { + "page_count": 10, + "document_index": {"figures": [{"page": 2, "caption": overview}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": overview, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 6, "kind": "paragraph", "text": encoding, "section_hint": "Method", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [{"id": "ray", "name": "Camera Rays", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "points", "name": "Sampled 3D Points", "evidence_ids": ["E0001", "E0002"]}, + {"id": "field", "name": "MLP (F_theta)", "evidence_ids": ["E0001", "E0002"]}, + {"id": "render", "name": "Volume Rendering", "evidence_ids": ["E0001"]}, + ], + "outputs": [{"id": "image", "name": "Rendered Image", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [{"source": "ray", "target": "points", "type": "sampling", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + mlps = [item for item in spec["modules"] if str(item.get("name") or "").casefold().startswith("mlp")] + positional = next(item for item in spec["modules"] if item.get("name") == "Positional Encoding") + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertEqual(len(mlps), 1) + self.assertIn(("points", positional["id"]), pairs) + self.assertIn((positional["id"], "field"), pairs) + + def test_rich_transformer_contract_recovers_connected_steps_from_distant_method_evidence(self): + caption = "Figure 1: The Transformer follows an encoder-decoder architecture." + method = "We use learned embeddings to convert the input tokens and output tokens to vectors. Output embeddings are offset by one position. A linear transformation and softmax convert the decoder output to predicted next-token probabilities." + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 6, "kind": "paragraph", "text": method, "section_hint": "Method", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [ + {"id": "input_embedding", "name": "Input Embedding", "evidence_ids": ["E0002"]}, + {"id": "output_embedding", "name": "Output Embedding", "evidence_ids": ["E0002"]}, + {"id": "target_sequence", "name": "output sequence", "evidence_ids": ["E0002"]}, + ], + "modules": [ + {"id": "encoder", "name": "Encoder Stack", "evidence_ids": ["E0001"]}, + {"id": "decoder", "name": "Decoder Stack", "evidence_ids": ["E0001"]}, + {"id": "softmax", "name": "Softmax", "evidence_ids": ["E0002"]}, + ], + "outputs": [], + "innovations": [], + "relations": [{"source": "encoder", "target": "decoder", "type": "cross_attention", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + ids = {item.get("name"): item.get("id") for field in ("inputs", "modules", "outputs") for item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn((ids["Inputs"], "input_embedding"), pairs) + self.assertEqual(ids["Outputs (shifted right)"], "target_sequence") + self.assertIn(("target_sequence", "output_embedding"), pairs) + self.assertIn(("softmax", ids["Output Probabilities"]), pairs) + + def test_generic_completion_recovers_nerf_branch_without_false_multimodal_topology(self): + caption = "Fig. 2: An overview of our neural radiance field and differentiable rendering procedure. We synthesize images by sampling 5D coordinates along camera rays, feeding locations and viewing direction into an MLP to produce a color and volume density, and using volume rendering to composite these values into an image." + positional = "Fig. 4: Our full model passes input coordinates through a high-frequency positional encoding." + parsed = { + "page_count": 10, + "document_index": {"figures": [{"page": 2, "caption": caption}, {"page": 4, "caption": positional}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Method", "confidence": 1.0}, + {"id": "E0002", "page": 4, "kind": "caption", "text": positional, "section_hint": "Method", "confidence": 1.0}, + ], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn((ids["Viewing Direction"], ids["MLP"]), pairs) + self.assertIn((ids["MLP"], ids["Color"]), pairs) + self.assertIn((ids["Volume Rendering"], ids["Rendered Image"]), pairs) + self.assertEqual(spec["topology"], "branch") + + def test_generic_completion_recovers_encoder_decoder_from_method_lines(self): + caption = "Figure 1: The model architecture." + lines = [ + "Here, the encoder maps an input sequence of symbol representations to continuous representations.", + "The encoder is composed of a stack of identical layers.", + "The decoder is also composed of a stack of identical layers and attends over the output of the encoder stack.", + "We use learned embeddings to convert the input", + "tokens and output tokens to vectors. We also use the usual learned linear transfor-", + "mation and softmax function to convert the decoder output to predicted next-token probabilities.", + "The output embeddings are offset by one position.", + "We add positional encodings to the input embeddings at the bottoms of the encoder and decoder stacks.", + ] + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 2, "kind": "caption", "text": caption, "confidence": 1.0}] + + [{"id": f"E00{index + 2:02d}", "page": 3 + index // 3, "kind": "paragraph", "text": text, "section_hint": "Method", "confidence": 1.0} for index, text in enumerate(lines)], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Inputs", "Input Embedding", "Encoder Stack", "Outputs (shifted right)", "Output Embedding", "Decoder Stack", "Linear", "Softmax", "Output Probabilities"): + self.assertIn(label, ids) + self.assertIn((ids["Encoder Stack"], ids["Decoder Stack"]), pairs) + self.assertIn((ids["Linear"], ids["Softmax"]), pairs) + + def test_generic_completion_treats_refinement_action_as_refined_output(self): + caption = "Figure 1: Given an input, the system starts by generating an output, gets feedback, and then refines the previously generated output." + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn("Refine", ids) + self.assertIn("Refined Output", ids) + self.assertIn((ids["Refine"], ids["Refined Output"]), pairs) + + def test_generic_completion_recovers_embedding_pretraining_and_finetuning(self): + captions = [ + "Figure 1: Overall pre-training and fine-tuning procedures. The same pre-trained model parameters are used to initialize models for different down-stream tasks.", + "Figure 2: Input representation. The input embeddings are the sum of the token embeddings, the segmentation embeddings and the position embeddings.", + ] + method = [ + "The model architecture is a multi-layer bidirectional Transformer encoder.", + "Task #1: Masked LM", + "Task #2: Next Sentence Prediction (NSP)", + "We represent the input sequence and fine-tune using labeled data from the downstream tasks.", + ] + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": index + 2, "caption": text} for index, text in enumerate(captions)]}, + "evidence": [{"id": f"E000{index + 1}", "page": index + 2, "kind": "caption", "text": text, "section_hint": "Figure Captions", "confidence": 1.0} for index, text in enumerate(captions)] + + [{"id": f"E001{index}", "page": 4, "kind": "paragraph", "text": text, "section_hint": "Method", "confidence": 1.0} for index, text in enumerate(method)], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Input Sequence", "Token Embeddings", "Segment Embeddings", "Position Embeddings", "Input Representation", "Bidirectional Transformer Encoder", "Masked LM", "Next Sentence Prediction", "Fine-tuning", "Downstream Tasks"): + self.assertIn(label, ids) + self.assertIn((ids["Token Embeddings"], ids["Input Representation"]), pairs) + self.assertIn((ids["Bidirectional Transformer Encoder"], ids["Fine-tuning"]), pairs) + + def test_generic_completion_recovers_elided_embedding_list_across_ocr_blocks(self): + caption = "Figure 1: Overall pre-training and fine-tuning procedures." + parsed = { + "page_count": 16, + "document_index": {"figures": [{"page": 3, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 3, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 0.99}, + {"id": "E0002", "page": 4, "kind": "paragraph", "text": "For a given token, its input representation is", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0003", "page": 4, "kind": "paragraph", "text": "constructed by summing the corresponding token,", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0004", "page": 4, "kind": "paragraph", "text": "segment, and position embeddings.", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0005", "page": 4, "kind": "paragraph", "text": "The architecture is a multi-layer bidirectional Transformer encoder.", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0006", "page": 4, "kind": "paragraph", "text": "We pre-train with Masked LM and Next Sentence Prediction before fine-tuning on downstream tasks.", "section_hint": "Method", "confidence": 0.98}, + ], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Input Sequence", "Token Embeddings", "Segment Embeddings", "Position Embeddings", "Input Representation", "Bidirectional Transformer Encoder"): + self.assertIn(label, ids) + self.assertIn((ids["Token Embeddings"], ids["Input Representation"]), pairs) + self.assertIn((ids["Segment Embeddings"], ids["Input Representation"]), pairs) + self.assertIn((ids["Position Embeddings"], ids["Input Representation"]), pairs) + + def test_generic_completion_uses_selected_architecture_page_context_and_diagram_labels(self): + captions = [ + "Figure 1: Direct set prediction combines a CNN with a transformer and bipartite matching.", + "Figure 10: Architecture of the proposed transformer's subsystem. See the detailed description on this page.", + ] + parsed = { + "page_count": 24, + "document_index": {"figures": [{"page": 2, "caption": captions[0]}, {"page": 20, "caption": captions[1]}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "bbox": [40, 300, 500, 320], "text": captions[0], "section_hint": "Figure Captions", "confidence": 0.98}, + {"id": "E0002", "page": 4, "kind": "paragraph", "bbox": [40, 220, 500, 240], "text": "The input image is processed for direct prediction with bipartite matching.", "section_hint": "Object Detection Method", "confidence": 0.98}, + {"id": "E0003", "page": 20, "kind": "caption", "bbox": [40, 600, 500, 620], "text": captions[1], "section_hint": "Figure Captions", "confidence": 0.98}, + {"id": "E0004", "page": 20, "kind": "paragraph", "bbox": [40, 100, 500, 120], "text": "Image features from the CNN backbone are passed through the transformer encoder.", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0005", "page": 20, "kind": "paragraph", "bbox": [40, 150, 500, 170], "text": "The decoder receives object queries and encoder memory.", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0006", "page": 20, "kind": "paragraph", "bbox": [340, 280, 380, 300], "text": "FFN", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0007", "page": 20, "kind": "paragraph", "bbox": [140, 390, 190, 410], "text": "Encoder", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0008", "page": 20, "kind": "paragraph", "bbox": [310, 310, 360, 330], "text": "Add & Norm", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0009", "page": 20, "kind": "paragraph", "bbox": [40, 180, 500, 200], "text": "The final class labels and bounding boxes are predicted in parallel.", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0010", "page": 20, "kind": "paragraph", "bbox": [40, 710, 500, 730], "text": "For example, a separate text encoder uses class descriptions and contrastive learning to create text embeddings.", "section_hint": "Related Work", "confidence": 0.98}, + ], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Input Image", "CNN Backbone", "Transformer Encoder", "Object Queries", "Transformer Decoder", "Feed Forward Network", "Class Predictions", "Bounding Box Predictions", "Bipartite Matching"): + self.assertIn(label, ids) + self.assertIn((ids["CNN Backbone"], ids["Transformer Encoder"]), pairs) + self.assertIn((ids["Object Queries"], ids["Transformer Decoder"]), pairs) + self.assertIn((ids["Feed Forward Network"], ids["Class Predictions"]), pairs) + self.assertNotIn("Text Encoder", ids) + self.assertNotIn("Contrastive Learning", ids) + + def test_dataset_sources_are_separated_from_modalities_when_caption_states_provenance(self): + caption = "Figure 1: Stage one constructs RETFound by means of SSL, using CFP and OCT from MEH-MIDAS and public datasets. Stage two adapts RETFound by supervised fine-tuning for internal and external evaluation." + parsed = { + "page_count": 4, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 3, "kind": "paragraph", "text": "Models were adapted with labelled training data and internally evaluated on hold-out test data from the MEH-AlzEye dataset.", "section_hint": "Evaluation", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "linear", + "inputs": [ + {"id": "cfp", "name": "CFP", "evidence_ids": ["E0001"]}, + {"id": "oct", "name": "OCT", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "ssl", "name": "SSL", "evidence_ids": ["E0001"]}, + {"id": "supervised", "name": "supervised fine-tuning", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [ + {"source": "cfp", "target": "ssl", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "oct", "target": "ssl", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "ssl", "target": "supervised", "type": "data_flow", "evidence_ids": ["E0001"]}, + ], + }, + } + + spec = normalize_figure_contract(plan, parsed) + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + by_name = {item["name"]: item for item in spec["inputs"]} + source_id = by_name["MEH-MIDAS + public datasets"]["id"] + + self.assertEqual(by_name["MEH-MIDAS + public datasets"]["role"], "data_source") + self.assertEqual(by_name["CFP"]["role"], "input modality") + self.assertEqual(by_name["OCT"]["role"], "input modality") + self.assertIn((source_id, "cfp"), pairs) + self.assertIn((source_id, "oct"), pairs) + self.assertNotIn((source_id, "ssl"), pairs) + self.assertIn(source_id, plan["contract_completion_report"]["added_data_sources"]) + self.assertNotIn("MEH-MIDAS", by_name) + self.assertNotIn("public datasets", by_name) + self.assertNotIn("MEH-AlzEye dataset", by_name) + output_names = {item["name"]: item["id"] for item in spec["outputs"]} + self.assertIn(("supervised", output_names["internal evaluation"]), pairs) + self.assertIn(("supervised", output_names["external evaluation"]), pairs) + + def test_fast_contract_cache_is_scoped_by_scientific_domain_not_visual_preferences(self): + parsed = {"source_sha256": "abc123"} + + first = _fast_cache_path(parsed, "planner-model", "ai-ml-method") + second = _fast_cache_path(parsed, "planner-model", "ai-ml-method") + + self.assertEqual(first, second) + self.assertNotEqual(first, _fast_cache_path(parsed, "planner-model", "empirical-science")) + + def test_generic_foundation_model_label_is_replaced_by_explicit_paper_name(self): + caption = "Figure 1: Schematic of development and evaluation of the foundation models (RETFound)." + parsed = { + "page_count": 4, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "linear", + "inputs": [], + "modules": [{"id": "model", "name": "foundation model", "role": "module", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [], + "relations": [], + "terminology": {"RETFound": "foundation model"}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["modules"][0]["name"], "RETFound") + self.assertEqual(spec["terminology"], {"RETFound": "RETFound"}) + self.assertIn("foundation model->RETFound", plan["contract_completion_report"]["repaired_generic_named_modules"]) + + def test_explicit_named_ssl_technique_is_restored_as_required_innovation(self): + parsed = { + "page_count": 4, + "document_index": {"figures": []}, + "evidence": [{"id": "E0001", "page": 2, "kind": "paragraph", "text": "We use an advanced SSL technique (masked autoencoder 15) on retinal images.", "section_hint": "Method", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "linear", + "inputs": [], + "modules": [{"id": "ssl", "name": "SSL", "role": "training_objective", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [], + "relations": [], + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["innovations"][0]["name"], "masked autoencoder") + self.assertTrue(spec["innovations"][0]["visible_label_required"]) + self.assertTrue(spec["innovations"][0]["must_appear_in_figure"]) + self.assertTrue(plan["contract_completion_report"]["added_explicit_named_techniques"]) + + def test_parallel_leaf_heads_are_promoted_to_consistent_output_roles(self): + caption = "The mask branch is in parallel with the classification and bounding-box regression heads." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "branch", + "inputs": [], + "modules": [ + {"id": "roialign", "name": "RoIAlign", "role": "module", "evidence_ids": ["E0001"]}, + {"id": "box", "name": "bounding-box regression", "role": "module", "evidence_ids": ["E0001"]}, + {"id": "mask", "name": "mask branch", "role": "module", "evidence_ids": ["E0001"]}, + ], + "outputs": [{"id": "classification", "name": "Classification", "role": "output", "evidence_ids": ["E0001"]}], + "innovations": [{"id": "innovation_mask", "name": "mask branch", "evidence_ids": ["E0001"]}], + "relations": [ + {"source": "proposal", "target": "roialign", "type": "conditioning", "evidence_ids": ["E0001"]}, + {"source": "roialign", "target": "classification", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "roialign", "target": "box", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "roialign", "target": "mask", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "proposal", "target": "mask", "type": "data_flow", "evidence_ids": ["E0001"]}, + ], + }, + } + plan["figure_specification"]["modules"].insert(0, {"id": "proposal", "name": "Region Proposals", "role": "module", "evidence_ids": ["E0001"]}) + + spec = normalize_figure_contract(plan, parsed) + output_by_id = {item["id"]: item for item in spec["outputs"]} + + self.assertEqual(set(output_by_id), {"classification", "box", "mask"}) + self.assertNotIn("box", {item["id"] for item in spec["modules"]}) + self.assertNotIn("mask", {item["id"] for item in spec["modules"]}) + self.assertTrue(all(output_by_id[item_id]["role"] == "output head" for item_id in ("classification", "box", "mask"))) + self.assertEqual(set(plan["contract_completion_report"]["promoted_parallel_output_heads"]), {"box", "mask"}) + self.assertNotIn(("proposal", "mask"), {(item["source"], item["target"]) for item in spec["relations"]}) + self.assertIn("proposal->mask", plan["contract_completion_report"]["removed_branch_shortcuts"]) + + def test_generic_completion_recovers_retrieval_conditioned_generation(self): + caption = "Figure 1: Overview of our approach. We combine a pre-trained retriever (Query Encoder + Document Index) with a pre-trained seq2seq model (Generator). For query x, we use MIPS to find the top-K documents. For final prediction y, the generator produces the output sequence." + result_caption = "Figure 2: Posterior for each generated token with five retrieved documents." + parsed = { + "page_count": 6, + "document_index": {"figures": [{"page": 2, "caption": caption}, {"page": 5, "caption": result_caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 5, "kind": "caption", "text": result_caption, "section_hint": "Figure Captions", "confidence": 1.0}, + ], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Input Query", "Query Encoder", "Document Index", "Retriever", "Top-K Documents", "Generator", "Output Sequence"): + self.assertIn(label, ids) + self.assertNotIn("Generate", ids) + self.assertIn((ids["Document Index"], ids["Retriever"]), pairs) + self.assertIn((ids["Top-K Documents"], ids["Generator"]), pairs) + + def test_rag_contract_repairs_latent_endpoint_alias_and_acronym_evidence(self): + caption = "Figure 1: Overview of our approach. Query Encoder and Document Index use Maximum Inner Product Search (MIPS) to find the top-K documents z. The Generator conditions on those latent documents." + parsed = { + "page_count": 4, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [ + {"id": "query_encoder", "name": "Query Encoder", "evidence_ids": ["E0001"]}, + {"id": "doc_index", "name": "Document Index", "evidence_ids": ["E0001"]}, + {"id": "generator", "name": "Generator", "evidence_ids": ["E0001"]}, + {"id": "mips", "name": "MIPS", "evidence_ids": []}, + ], + "outputs": [], + "innovations": [], + "relations": [ + {"source": "mips", "target": "latent_z", "type": "branch", "label": "Top-K Documents", "evidence_ids": []}, + {"source": "latent_z", "target": "generator", "type": "conditioning", "label": "context", "evidence_ids": []}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + validation = validate_plan_grounding(plan, parsed) + ids = {item["name"]: item["id"] for item in spec["modules"]} + pairs = {(item["source"], item["target"]): item for item in spec["relations"]} + + self.assertTrue(next(item for item in spec["modules"] if item["name"] == "MIPS")["evidence_ids"]) + self.assertIn(("mips", ids["Top-K Documents"]), pairs) + self.assertIn((ids["Top-K Documents"], "generator"), pairs) + self.assertTrue(pairs[("mips", ids["Top-K Documents"])]["evidence_ids"]) + self.assertFalse(any("latent_z" in (item["source"], item["target"]) for item in spec["relations"])) + self.assertTrue(validation["ok"]) + + def test_rich_sam_contract_does_not_import_unrelated_clip_components(self): + sam_caption = "Figure 1: The promptable segmentation model contains an image encoder, prompt encoder, and mask decoder that produces a valid segmentation mask." + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": sam_caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": sam_caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 1, "kind": "paragraph", "text": "CLIP uses class descriptions, a text encoder, text embeddings, and contrastive learning.", "section_hint": "Introduction", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "image", "name": "Image", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "image_encoder", "name": "Image Encoder", "evidence_ids": ["E0001"]}, + {"id": "prompt_encoder", "name": "Prompt Encoder", "evidence_ids": ["E0001"]}, + {"id": "mask_decoder", "name": "Mask Decoder", "evidence_ids": ["E0001"]}, + ], + "outputs": [{"id": "mask", "name": "Valid Segmentation Mask", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [ + {"source": "image", "target": "image_encoder", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "image_encoder", "target": "mask_decoder", "type": "feature_flow", "evidence_ids": ["E0001"]}, + {"source": "prompt_encoder", "target": "mask_decoder", "type": "conditioning", "evidence_ids": ["E0001"]}, + {"source": "mask_decoder", "target": "mask", "type": "prediction", "evidence_ids": ["E0001"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + labels = {_item.get("name") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + + self.assertNotIn("Class Descriptions", labels) + self.assertNotIn("Text Encoder", labels) + self.assertNotIn("Contrastive Learning", labels) + self.assertTrue(plan["contract_completion_report"]["conservative_expansion"]) + + def test_generic_completion_repairs_vlm_ids_labels_and_relation_grounding(self): + caption = "Figure 1: Pre-training and fine-tuning procedures." + parsed = { + "page_count": 5, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 3, "kind": "paragraph", "text": "The input embeddings are the sum of the token embeddings, segmentation embeddings and position embeddings.", "section_hint": "Method", "confidence": 1.0}, + {"id": "E0003", "page": 3, "kind": "paragraph", "text": "The architecture is a multi-layer bidirectional Transformer encoder.", "section_hint": "Method", "confidence": 1.0}, + {"id": "E0004", "page": 4, "kind": "paragraph", "text": "We train with a masked language model and next sentence prediction.", "section_hint": "Method", "confidence": 1.0}, + {"id": "E0005", "page": 4, "kind": "paragraph", "text": "We represent the input sequence for downstream tasks.", "section_hint": "Method", "confidence": 1.0}, + {"id": "E0006", "page": 1, "kind": "paragraph", "text": "A major limitation is that standard language models are", "section_hint": "Introduction", "confidence": 1.0}, + {"id": "E0007", "page": 1, "kind": "paragraph", "text": "unidirectional.", "section_hint": "Introduction", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [{"id": "input_tokens", "name": "input example", "evidence_ids": ["E0005"]}], + "modules": [ + {"id": "embedding_layer", "name": "input embeddings", "evidence_ids": ["E0002"]}, + {"id": "transformer_encoder", "name": "Bidirectional Transformer Encoder", "evidence_ids": ["E0003"]}, + {"id": "mlm_head", "name": "MLM objective", "evidence_ids": []}, + ], + "research_problem": {"text": "Standard language models are unidirectional.", "evidence_ids": []}, + "outputs": [], "innovations": [], "must_show": [], "terminology": {}, + "relations": [ + {"source": "embedding_layer", "target": "transformer_encoder", "type": "data_flow", "evidence_ids": []}, + {"source": "transformer_encoder", "target": "mlm_head", "type": "prediction", "evidence_ids": []}, + ], + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(next(item for item in spec["inputs"] if item["id"] == "input_tokens")["name"], "Input Sequence") + self.assertEqual(next(item for item in spec["modules"] if item["id"] == "mlm_head")["name"], "Masked LM") + self.assertTrue(next(item for item in spec["modules"] if item["id"] == "mlm_head")["evidence_ids"]) + self.assertTrue(all(item.get("evidence_ids") for item in spec["relations"] if item.get("source") in {"embedding_layer", "transformer_encoder"} and item.get("target") in {"transformer_encoder", "mlm_head"})) + self.assertEqual(spec["research_problem"]["evidence_ids"], ["E0006", "E0007"]) + + def test_rich_contract_expands_newly_grounded_intermediate_to_evidence_neighbors(self): + caption = "Figure 1: Overall pre-training and fine-tuning procedures." + parsed = { + "page_count": 16, + "document_index": {"figures": [{"page": 3, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 3, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 0.99}, + {"id": "E0002", "page": 4, "kind": "paragraph", "text": "For a given token, its input representation is", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0003", "page": 4, "kind": "paragraph", "text": "constructed by summing the corresponding token,", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0004", "page": 4, "kind": "paragraph", "text": "segment, and position embeddings.", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0005", "page": 4, "kind": "paragraph", "text": "The architecture is a multi-layer bidirectional Transformer encoder.", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0006", "page": 4, "kind": "paragraph", "text": "We train with Masked LM and Next Sentence Prediction before fine-tuning on downstream tasks.", "section_hint": "Method", "confidence": 0.98}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [{"id": "sentence_pair", "name": "Sentence Pair", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "bert", "name": "Bidirectional Transformer Encoder", "evidence_ids": ["E0005"]}, + {"id": "mlm", "name": "Masked LM", "evidence_ids": ["E0006"]}, + {"id": "nsp", "name": "Next Sentence Prediction", "evidence_ids": ["E0006"]}, + ], + "outputs": [{"id": "tasks", "name": "Downstream Tasks", "evidence_ids": ["E0006"]}], + "relations": [{"source": "bert", "target": "mlm", "type": "training_objective", "evidence_ids": ["E0006"]}], + "innovations": [], "must_show": [], "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Input Sequence", "Token Embeddings", "Segment Embeddings", "Position Embeddings", "Input Representation"): + self.assertIn(label, ids) + self.assertIn((ids["Input Representation"], ids["Bidirectional Transformer Encoder"]), pairs) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_paper_to_editable_route.py b/tests/unit/test_paper_to_editable_route.py new file mode 100644 index 0000000..f05c7f8 --- /dev/null +++ b/tests/unit/test_paper_to_editable_route.py @@ -0,0 +1,51 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from rfs.workflows.paper_to_editable import run_paper_to_editable + + +class PaperToEditableRouteTests(unittest.TestCase): + def test_framework_route_reuses_paper_to_image_contract_as_authority(self): + contract = {"figure_specification": {"panel_graphs": [{"id": "p", "label": "Panel", "nodes": []}]}} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = root / "selected.png" + reference.write_bytes(b"not-read-by-mocked-compiler") + image_result = {"ok": True, "selected_image": str(reference)} + framework_result = { + "ok": True, + "pptx": str(root / "editable" / "editable_composition.pptx"), + "png": None, + "semantic_binding": {"status": "pass"}, + } + + with ( + patch("rfs.workflows.paper_to_editable.run_paper_to_image", return_value=image_result), + patch("rfs.workflows.paper_to_editable.load_paper_semantic_contract", return_value=contract), + patch("rfs.workflows.paper_to_editable.make_framework", return_value=framework_result) as compiler, + ): + result = run_paper_to_editable( + paper=root / "paper.pdf", + out=root / "run", + editable_route="framework", + framework_asset_mode="placeholder", + framework_slot_count=25, + framework_candidates_per_slot=1, + framework_asset_workers=3, + export_preview=False, + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["editable_route"], "framework") + self.assertEqual(result["semantic_binding_status"], "pass") + kwargs = compiler.call_args.kwargs + self.assertIs(kwargs["semantic_contract"], contract) + self.assertEqual(kwargs["slot_count"], 25) + self.assertEqual(kwargs["asset_mode"], "placeholder") + self.assertEqual(kwargs["slot_source"], "reference-primary") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ppt_compiler.py b/tests/unit/test_ppt_compiler.py similarity index 86% rename from tests/test_ppt_compiler.py rename to tests/unit/test_ppt_compiler.py index e86f6a2..459abe9 100644 --- a/tests/test_ppt_compiler.py +++ b/tests/unit/test_ppt_compiler.py @@ -29,6 +29,17 @@ def test_multisegment_and_dashed_arrows_are_rendered_as_ppt_connectors(self): "bbox_percent": {"x": 0.05, "y": 0.05, "w": 0.90, "h": 0.80}, "editable_in": "pptx", }], + "cards": [{ + "id": "card_a", + "semantic_role": "outer_group_boundary", + "bbox_percent": {"x": 0.08, "y": 0.16, "w": 0.76, "h": 0.54}, + "shape_kind": "rect", + "stroke_color": "#59AFCB", + "stroke_width_pt": 1.5, + "dash_style": "dash", + "fill_transparency": 1.0, + "z_index": 12, + }], "slots": [ { "id": "slot_a", @@ -87,6 +98,8 @@ def test_multisegment_and_dashed_arrows_are_rendered_as_ppt_connectors(self): by_id = {item["arrow_id"]: item for item in report["arrows"]} self.assertEqual(by_id["multi_a"]["segment_count"], 3) self.assertEqual(by_id["loop_a"]["segment_count"], 4) + self.assertEqual(report["cards"][0]["card_id"], "card_a") + self.assertEqual(report["cards"][0]["render_policy"], "ppt_shape_not_image_asset") with zipfile.ZipFile(pptx) as archive: slide_xml = archive.read("ppt/slides/slide1.xml").decode("utf-8") diff --git a/tests/unit/test_ppt_roundtrip.py b/tests/unit/test_ppt_roundtrip.py new file mode 100644 index 0000000..23c4b86 --- /dev/null +++ b/tests/unit/test_ppt_roundtrip.py @@ -0,0 +1,38 @@ +import tempfile +import unittest +from pathlib import Path + +from rfs.paper_to_image.editable_ppt import compile_semantic_ppt +from rfs.paper_to_image.ppt_roundtrip import evaluate_ppt_roundtrip +from rfs.utils import read_json + + +class PptRoundtripTests(unittest.TestCase): + def test_semantic_ppt_object_tree_preserves_labels_nodes_and_connectors(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Encoder", "Output"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "encoder", "name": "Encoder"}], + "outputs": [{"id": "output", "name": "Output"}], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow"}, + {"source": "encoder", "target": "output", "type": "data_flow"}, + ], + } + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + compiled = compile_semantic_ppt(specification, root) + program = read_json(root / "figure_program.json") + + report = evaluate_ppt_roundtrip(program, compiled["pptx"], root, render=False) + + self.assertTrue(report["ok"]) + self.assertEqual(report["structure"]["editable_label_ratio"], 1.0) + self.assertEqual(report["structure"]["node_editability_ratio"], 1.0) + self.assertEqual(report["structure"]["connector_editability_ratio"], 1.0) + self.assertEqual(report["structure"]["flattened_picture_area_ratio"], 0.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_program_builder.py b/tests/unit/test_program_builder.py new file mode 100644 index 0000000..18b0b0d --- /dev/null +++ b/tests/unit/test_program_builder.py @@ -0,0 +1,23 @@ +import tempfile +import unittest +from pathlib import Path + +from rfs.program_builder import build_figure_program + + +class ProgramBuilderTests(unittest.TestCase): + def test_canvas_preserves_reference_aspect_ratio(self): + inventory = {"canvas_aspect_ratio": 1.5, "reference_path": "reference.png"} + layout_plan = {"locator_mode": "heuristic", "panels": [], "slots": [], "arrows": []} + style = {"palette": [], "color_tokens": []} + + with tempfile.TemporaryDirectory() as tmp: + program = build_figure_program({}, inventory, style, Path(tmp), layout_plan) + + canvas = program["canvas"] + self.assertAlmostEqual(canvas["width_in"] / canvas["height_in"], 1.5, places=3) + self.assertEqual(canvas["ratio"], 1.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_rebuild_visual_critic.py b/tests/unit/test_rebuild_visual_critic.py new file mode 100644 index 0000000..0dc6da7 --- /dev/null +++ b/tests/unit/test_rebuild_visual_critic.py @@ -0,0 +1,97 @@ +import tempfile +import unittest +from pathlib import Path + +from rfs.rebuild_visual_critic import run_rebuild_visual_quality_check + + +class RebuildVisualCriticTests(unittest.TestCase): + def test_deterministic_report_flags_text_overlap_bounds_and_missing_arrow_path(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + program = { + "text_program": { + "items": [ + { + "id": "text_a", + "source_reference_text_id": "ref_a", + "text": "A", + "bbox_percent": {"x": 0.10, "y": 0.10, "w": 0.20, "h": 0.05}, + }, + { + "id": "text_b", + "source_reference_text_id": "ref_b", + "text": "B", + "bbox_percent": {"x": 0.12, "y": 0.11, "w": 0.20, "h": 0.05}, + }, + { + "id": "text_bad", + "source_reference_text_id": "ref_bad", + "text": "bad", + "bbox_percent": {"x": 0.98, "y": 0.10, "w": 0.10, "h": 0.05}, + }, + ] + }, + "panels": [], + "slots": [], + "arrows": [{"id": "arrow_missing", "source_id": "a", "target_id": "b"}], + } + + report = run_rebuild_visual_quality_check(out, program) + + self.assertEqual(report["status"], "blocked") + issue_types = {item["type"] for item in report["issues"]} + self.assertIn("text_overlap", issue_types) + self.assertIn("text_bbox_out_of_bounds", issue_types) + self.assertIn("arrow_missing_path", issue_types) + self.assertTrue((out / "rebuild_visual_quality_report.json").exists()) + + def test_ownership_conflict_is_reported(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + program = { + "text_program": { + "items": [{ + "id": "text_a", + "source_reference_text_id": "ref_a", + "text": "A", + "bbox_percent": {"x": 0.10, "y": 0.10, "w": 0.20, "h": 0.05}, + }] + }, + "panels": [], + "slots": [], + "arrows": [], + } + ownership = { + "items": [{ + "text_id": "ref_a", + "layer_ownership": "raster_asset_layer", + "included_in_text_program": False, + }] + } + + report = run_rebuild_visual_quality_check(out, program, ownership_report=ownership) + + self.assertEqual(report["ownership_issue_count"], 1) + self.assertEqual(report["issues"][0]["type"], "text_layer_ownership_conflict") + + def test_same_target_role_without_explicit_group_is_not_forced_aligned(self): + with tempfile.TemporaryDirectory() as tmp: + program = { + "text_program": {"items": [ + {"id": "a", "target_id": "panel", "role": "body_label", "bbox_percent": {"x": 0.1, "y": 0.1, "w": 0.1, "h": 0.03}}, + {"id": "b", "target_id": "panel", "role": "body_label", "bbox_percent": {"x": 0.3, "y": 0.4, "w": 0.1, "h": 0.03}}, + {"id": "c", "target_id": "panel", "role": "body_label", "bbox_percent": {"x": 0.5, "y": 0.7, "w": 0.1, "h": 0.03}}, + ]}, + "panels": [], + "slots": [], + "arrows": [], + } + + report = run_rebuild_visual_quality_check(Path(tmp), program) + + self.assertNotIn("text_group_misaligned", {item["type"] for item in report["issues"]}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_review_contract.py b/tests/unit/test_review_contract.py new file mode 100644 index 0000000..e9c3ed6 --- /dev/null +++ b/tests/unit/test_review_contract.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import unittest + +from rfs.paper_to_image.review_contract import build_repair_plan, compile_review_ground_truth + + +class ReviewContractTests(unittest.TestCase): + def test_ground_truth_embeds_page_section_and_original_excerpt(self): + plan = { + "figure_specification": { + "central_claim": {"text": "Evidence-constrained reasoning reduces unsupported conclusions.", "evidence_ids": ["E1"]}, + "modules": [{"id": "parser", "name": "Document Parser", "evidence_ids": ["E2"]}], + "relations": [{"source": "parser", "target": "answer", "type": "data_flow", "evidence_ids": ["E2"]}], + "outputs": [{"id": "answer", "name": "Evidence-linked Answer", "evidence_ids": ["E2"]}], + "training_flow": ["Training loss"], + "must_show": ["Training loss", "Document Parser"], + "forbidden_inventions": ["External memory bank"], + "uncertainties": ["The internal decoder architecture is not specified"], + } + } + evidence = [ + {"id": "E1", "page": 1, "section_hint": "Abstract", "text": "The method reduces unsupported conclusions.", "confidence": 1.0, "source": "pymupdf"}, + {"id": "E2", "page": 3, "section_hint": "Method", "text": "The Document Parser creates page-aware passages used for the final answer.", "confidence": 0.99, "source": "pymupdf"}, + {"id": "E3", "page": 4, "section_hint": "Training", "text": "Training loss is optimized over synthetic tasks.", "confidence": 0.98, "source": "pymupdf"}, + ] + + contract = compile_review_ground_truth(plan, evidence) + + parser = next(item for item in contract["priorities"] if item["label"] == "Document Parser") + self.assertEqual(parser["evidence"][0]["page"], 3) + self.assertEqual(parser["evidence"][0]["section"], "Method") + self.assertIn("page-aware passages", parser["evidence"][0]["text"]) + self.assertTrue(contract["strict_evidence_required"]) + self.assertEqual(contract["forbidden"][0]["label"], "External memory bank") + self.assertIn("not specified", contract["unknown"][0]["label"]) + training = next(item for item in contract["priorities"] if item["label"] == "Training loss") + self.assertEqual(training["evidence_link_method"], "deterministic_text_match") + self.assertEqual(training["evidence"][0]["evidence_id"], "E3") + self.assertEqual(sum(item["label"] == "Training loss" for item in contract["priorities"]), 1) + self.assertEqual(sum(item["label"] == "Document Parser" for item in contract["priorities"]), 1) + self.assertTrue(training["required"], "must_show must keep an otherwise optional training item required") + + def test_training_and_inference_context_are_optional_unless_explicitly_visible(self): + plan = { + "figure_specification": { + "training_flow": [ + {"id": "train_context", "step": "Optimize auxiliary loss", "evidence_ids": ["E1"]}, + {"id": "train_visible", "step": "Supervised fine-tuning", "required": True, "evidence_ids": ["E1"]}, + ], + "inference_flow": [{"id": "infer_context", "step": "Run classifier", "evidence_ids": ["E1"]}], + } + } + evidence = [{"id": "E1", "page": 2, "section_hint": "Method", "text": "The model uses an auxiliary loss, supervised fine-tuning, and a classifier.", "confidence": 1.0, "source": "pymupdf"}] + + contract = compile_review_ground_truth(plan, evidence) + priorities = {item["id"]: item for item in contract["priorities"]} + + self.assertFalse(priorities["train_context"]["required"]) + self.assertTrue(priorities["train_visible"]["required"]) + self.assertFalse(priorities["infer_context"]["required"]) + + def test_repair_plan_prioritizes_paper_conflicts_and_carries_evidence_ids(self): + review = { + "preserve": ["correct left panel"], + "paper_alignment": { + "missing_priorities": [{"id": "GT_MODULE_parser", "label": "Document Parser", "reason": "missing", "evidence_ids": ["E2"]}], + "evidence_conflicts": [], + "unsupported_visual_claims": ["Unsupported memory bank"], + }, + "scientific": {"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "role_mismatches": [], "containment_mismatches": []}, + "topology": {"missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": []}, + "ocr": {"missing_labels": ["Document Parser"], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "unexpected_labels": ["Supported Detail"], "unsupported_unexpected_labels": []}, + "template": {"passed": True}, + "aesthetic": {"passed": True}, + "basic": {"visual_enrichment_passed": True}, + "repair": [], + "hard_errors": [], + } + + repair = build_repair_plan(review, "candidate_01", 1) + + self.assertEqual(repair["items"][0]["issue_type"], "paper_missing_priority") + self.assertEqual(repair["items"][0]["severity"], "critical") + self.assertEqual(repair["items"][0]["evidence_ids"], ["E2"]) + self.assertEqual(repair["items"][0]["preserve"], ["correct left panel"]) + ocr_item = next(item for item in repair["items"] if item["issue_type"] == "ocr_missing_labels") + self.assertEqual(ocr_item["owner"], "overlay_compiler") + self.assertEqual(repair["items"][0]["owner"], "image2_visual_substrate") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_semantic_blueprint.py b/tests/unit/test_semantic_blueprint.py new file mode 100644 index 0000000..efd4fc4 --- /dev/null +++ b/tests/unit/test_semantic_blueprint.py @@ -0,0 +1,319 @@ +import unittest + +from rfs.paper_to_image.semantic_blueprint import compile_semantic_blueprint + + +class SemanticBlueprintTests(unittest.TestCase): + def test_isolated_required_nodes_are_not_rendered_as_peer_graph_nodes(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Encoder", "Output", "Local Detail"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [ + {"id": "encoder", "name": "Encoder"}, + {"id": "detail", "name": "Local Detail"}, + ], + "outputs": [{"id": "output", "name": "Output"}], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow"}, + {"source": "encoder", "target": "output", "type": "data_flow"}, + ], + } + + report = compile_semantic_blueprint(specification) + + self.assertTrue(report["applied"]) + self.assertEqual(report["isolated_node_ids"], ["detail"]) + self.assertNotIn("detail", {item["id"] for item in report["nodes"]}) + + def test_linear_contract_uses_graph_ranks_and_keeps_parallel_outputs_in_one_layer(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Embed", "Encoder", "Output A", "Output B"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "embed", "name": "Embed"}, {"id": "encoder", "name": "Encoder"}], + "outputs": [{"id": "a", "name": "Output A"}, {"id": "b", "name": "Output B"}], + "relations": [ + {"source": "input", "target": "embed", "type": "data_flow"}, + {"source": "embed", "target": "encoder", "type": "data_flow"}, + {"source": "encoder", "target": "a", "type": "data_flow"}, + {"source": "encoder", "target": "b", "type": "data_flow"}, + ], + } + + report = compile_semantic_blueprint(specification) + nodes = {item["id"]: item for item in report["nodes"]} + + self.assertTrue(report["applied"]) + self.assertEqual(nodes["input"]["rank"], 0) + self.assertLess(nodes["embed"]["rank"], nodes["encoder"]["rank"]) + self.assertEqual(nodes["a"]["rank"], nodes["b"]["rank"]) + self.assertNotEqual(nodes["a"]["bbox_percent"]["y"], nodes["b"]["bbox_percent"]["y"]) + output_edges = [item for item in report["connectors"] if item["source"] == "encoder"] + self.assertNotEqual(output_edges[0]["path_percent"][0][1], output_edges[1]["path_percent"][0][1]) + + def test_multimodal_inputs_share_one_layer_and_converge(self): + specification = { + "topology": "multimodal", + "required_labels": ["Image", "Text", "Audio", "Encoders", "Joint Space", "Alignment"], + "inputs": [{"id": "image", "name": "Image"}, {"id": "text", "name": "Text"}, {"id": "audio", "name": "Audio"}], + "modules": [{"id": "encoders", "name": "Encoders"}, {"id": "joint", "name": "Joint Space", "role": "shared representation"}], + "outputs": [{"id": "alignment", "name": "Alignment"}], + "relations": [ + {"source": "image", "target": "encoders", "type": "encoding"}, + {"source": "text", "target": "encoders", "type": "encoding"}, + {"source": "audio", "target": "encoders", "type": "encoding"}, + {"source": "encoders", "target": "joint", "type": "alignment"}, + {"source": "joint", "target": "alignment", "type": "enables"}, + ], + } + + report = compile_semantic_blueprint(specification) + nodes = {item["id"]: item for item in report["nodes"]} + + self.assertTrue(report["applied"]) + self.assertEqual({nodes[node_id]["rank"] for node_id in ("image", "text", "audio")}, {0}) + self.assertEqual(len({nodes[node_id]["bbox_percent"]["x"] for node_id in ("image", "text", "audio")}), 1) + self.assertGreater(nodes["joint"]["rank"], nodes["encoders"]["rank"]) + input_edges = [item for item in report["connectors"] if item["target"] == "encoders"] + self.assertEqual(len({item["path_percent"][-1][1] for item in input_edges}), 3) + self.assertEqual(len({item["path_percent"][1][0] for item in input_edges}), 3) + + def test_feedback_loop_is_routed_outside_the_forward_graph(self): + specification = { + "topology": "feedback", + "required_labels": ["Sample", "Measure", "Update", "Estimate", "repeat"], + "inputs": [{"id": "sample", "name": "Sample"}], + "modules": [{"id": "measure", "name": "Measure"}, {"id": "update", "name": "Update"}], + "outputs": [{"id": "estimate", "name": "Estimate"}], + "relations": [ + {"source": "sample", "target": "measure", "type": "data_flow"}, + {"source": "measure", "target": "update", "type": "data_flow"}, + {"source": "update", "target": "estimate", "type": "data_flow"}, + {"source": "estimate", "target": "measure", "type": "feedback_loop", "label": "repeat"}, + ], + } + + report = compile_semantic_blueprint(specification) + loop = next(item for item in report["connectors"] if item["type"] == "feedback_loop") + + self.assertTrue(report["applied"]) + self.assertEqual(loop["route_style"], "outer_feedback") + self.assertIn(loop["path_percent"][1][1], {0.05, 0.95}) + self.assertEqual(loop["label"], "repeat") + + def test_source_only_conditioning_nodes_are_placed_immediately_before_their_target(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Patches", "Projection", "Class Token", "Position Embedding", "Encoder"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [ + {"id": "patches", "name": "Patches"}, + {"id": "projection", "name": "Projection"}, + {"id": "class", "name": "Class Token", "role": "conditioning"}, + {"id": "position", "name": "Position Embedding", "role": "conditioning"}, + {"id": "encoder", "name": "Encoder"}, + ], + "relations": [ + {"source": "input", "target": "patches", "type": "data_flow"}, + {"source": "patches", "target": "projection", "type": "data_flow"}, + {"source": "projection", "target": "encoder", "type": "data_flow"}, + {"source": "class", "target": "encoder", "type": "conditioning"}, + {"source": "position", "target": "encoder", "type": "conditioning"}, + ], + } + + report = compile_semantic_blueprint(specification) + nodes = {item["id"]: item for item in report["nodes"]} + + self.assertEqual(nodes["class"]["rank"], nodes["encoder"]["rank"] - 1) + self.assertEqual(nodes["position"]["rank"], nodes["encoder"]["rank"] - 1) + self.assertEqual(nodes["projection"]["rank"], nodes["class"]["rank"]) + + def test_required_label_whitelist_excludes_out_of_scope_entities(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Core", "Output"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "core", "name": "Core"}, {"id": "detail", "name": "Out-of-scope Detail"}], + "outputs": [{"id": "output", "name": "Output"}], + "innovations": [{"id": "core_innovation", "name": "Core"}], + "relations": [ + {"source": "input", "target": "core", "type": "data_flow"}, + {"source": "core", "target": "output", "type": "data_flow"}, + {"source": "core", "target": "detail", "type": "data_flow"}, + ], + } + + report = compile_semantic_blueprint(specification) + + self.assertEqual({item["label"] for item in report["nodes"]}, {"Input", "Core", "Output"}) + self.assertFalse(any(item["target"] == "detail" for item in report["connectors"])) + + def test_skip_layer_relation_uses_bypass_lane_instead_of_crossing_middle_node(self): + specification = { + "topology": "branch", + "required_labels": ["Input", "Middle", "Merge"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "middle", "name": "Middle"}, {"id": "merge", "name": "Merge"}], + "relations": [ + {"source": "input", "target": "middle", "type": "data_flow"}, + {"source": "middle", "target": "merge", "type": "data_flow"}, + {"source": "input", "target": "merge", "type": "conditioning"}, + ], + } + + report = compile_semantic_blueprint(specification) + bypass = next(item for item in report["connectors"] if item["source"] == "input" and item["target"] == "merge") + + self.assertEqual(bypass["route_style"], "bypass_orthogonal") + self.assertGreaterEqual(len(bypass["path_percent"]), 4) + self.assertEqual(bypass["routing_metrics"]["node_obstacle_intersections"], 0) + + def test_global_router_separates_nearby_long_vertical_lanes(self): + specification = { + "topology": "linear", + "required_labels": ["Primary Input", "Auxiliary Input", "Stage One", "Stage Two", "Merger", "Output"], + "inputs": [ + {"id": "primary", "name": "Primary Input"}, + {"id": "auxiliary", "name": "Auxiliary Input"}, + ], + "modules": [ + {"id": "stage_one", "name": "Stage One"}, + {"id": "stage_two", "name": "Stage Two"}, + {"id": "merger", "name": "Merger"}, + ], + "outputs": [{"id": "output", "name": "Output"}], + "relations": [ + {"source": "primary", "target": "stage_one", "type": "data_flow"}, + {"source": "stage_one", "target": "stage_two", "type": "data_flow"}, + {"source": "stage_two", "target": "merger", "type": "data_flow"}, + {"source": "auxiliary", "target": "merger", "type": "data_flow"}, + {"source": "merger", "target": "output", "type": "prediction"}, + ], + } + + report = compile_semantic_blueprint(specification) + primary = next(item for item in report["connectors"] if item["source"] == "primary") + auxiliary = next(item for item in report["connectors"] if item["source"] == "auxiliary") + + self.assertTrue(report["route_quality"]["passed"]) + self.assertEqual(report["route_quality"]["shared_segment_overlap"], 0.0) + self.assertNotEqual(primary["lane_assignment"], auxiliary["lane_assignment"]) + self.assertIn("outer_", auxiliary["lane_assignment"]) + + def test_fan_in_relations_use_distinct_target_ports(self): + specification = { + "topology": "multimodal", + "required_labels": ["Input A", "Input B", "Input C", "Fusion", "Output"], + "inputs": [ + {"id": "a", "name": "Input A"}, + {"id": "b", "name": "Input B"}, + {"id": "c", "name": "Input C"}, + ], + "modules": [{"id": "fusion", "name": "Fusion"}], + "outputs": [{"id": "output", "name": "Output"}], + "relations": [ + {"source": "a", "target": "fusion", "type": "data_flow"}, + {"source": "b", "target": "fusion", "type": "data_flow"}, + {"source": "c", "target": "fusion", "type": "data_flow"}, + {"source": "fusion", "target": "output", "type": "prediction"}, + ], + } + + report = compile_semantic_blueprint(specification) + incoming = [item for item in report["connectors"] if item["target"] == "fusion"] + + self.assertEqual(len({tuple(item["path_percent"][-1]) for item in incoming}), 3) + self.assertEqual(report["route_quality"]["node_obstacle_intersections"], 0) + self.assertEqual(report["route_quality"]["crossing_count"], 0) + + def test_contract_larger_than_blueprint_limit_is_not_forced_into_one_canvas(self): + modules = [{"id": f"node_{index}", "name": f"Node {index}"} for index in range(18)] + specification = { + "topology": "linear", + "required_labels": [item["name"] for item in modules], + "modules": modules, + "relations": [{"source": f"node_{index}", "target": f"node_{index + 1}", "type": "data_flow"} for index in range(17)], + } + + report = compile_semantic_blueprint(specification) + + self.assertFalse(report["applied"]) + self.assertEqual(report["reason"], "too_many_nodes") + + def test_two_panel_dense_overview_uses_panel_local_layout(self): + specification = { + "topology": "dense_multiframe", + "required_labels": ["Input", "Model", "Prediction"], + "inputs": [{"id": "input", "name": "Input", "evidence_ids": ["E_A"]}], + "modules": [{"id": "model", "name": "Model", "evidence_ids": ["E_B"]}], + "outputs": [{"id": "output", "name": "Prediction", "evidence_ids": ["E_A"]}], + "relations": [ + {"source": "input", "target": "model", "type": "data_flow"}, + {"source": "model", "target": "output", "type": "prediction"}, + ], + "overview_panels": [ + {"id": "panel_a", "panel_label": "a", "label": "Training and usage", "evidence_ids": ["E_A"], "entity_ids": ["input", "output"]}, + {"id": "panel_b", "panel_label": "b", "label": "Architecture", "evidence_ids": ["E_B"], "entity_ids": ["model"]}, + ], + } + + report = compile_semantic_blueprint(specification) + nodes = {item["id"]: item for item in report["nodes"]} + + self.assertEqual(len(report["panels"]), 2) + self.assertEqual(nodes["input"]["panel_id"], "panel_a") + self.assertEqual(nodes["model"]["panel_id"], "panel_b") + forward = next(item for item in report["connectors"] if item["source"] == "input") + self.assertNotEqual(forward["route_style"], "outer_feedback") + + def test_panel_instance_graph_preserves_repeated_labels_and_local_connectors(self): + specification = { + "topology": "dense_multiframe", + "panel_graphs": [ + { + "id": "panel_a", + "panel_label": "a", + "label": "Training and usage", + "nodes": [ + {"instance_id": "a_data", "name": "Dataset", "role": "input", "evidence_ids": ["E1"]}, + {"instance_id": "a_model_train", "name": "Model", "role": "module", "evidence_ids": ["E2"]}, + {"instance_id": "a_prediction", "name": "Prediction", "role": "output", "evidence_ids": ["E3"]}, + {"instance_id": "a_model_use", "name": "Model", "role": "module", "evidence_ids": ["E4"]}, + ], + "relations": [ + {"source": "a_data", "target": "a_model_train", "type": "data_flow"}, + {"source": "a_model_train", "target": "a_prediction", "type": "prediction"}, + ], + }, + { + "id": "panel_b", + "panel_label": "b", + "label": "Architecture", + "nodes": [ + {"instance_id": "b_model", "name": "Model", "role": "group", "evidence_ids": ["E5"]}, + {"instance_id": "b_attention", "name": "Attention", "role": "module", "evidence_ids": ["E6"]}, + {"instance_id": "b_output", "name": "Output", "role": "output", "evidence_ids": ["E7"]}, + ], + "relations": [ + {"source": "b_attention", "target": "b_output", "type": "prediction"}, + ], + }, + ], + } + + report = compile_semantic_blueprint(specification) + + self.assertTrue(report["applied"]) + self.assertEqual(report["source"], "panel_graphs") + self.assertEqual(report["node_count"], 6) + self.assertEqual(sum(item["label"] == "Model" for item in report["nodes"]), 2) + self.assertEqual(report["panels"][1]["groups"][0]["label"], "Model") + self.assertTrue(all(item["panel_id"] == "panel_a" for item in report["connectors"][:2])) + self.assertEqual(report["connectors"][2]["panel_id"], "panel_b") + self.assertIn("a_model_use", report["isolated_node_ids"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_semantic_contract.py b/tests/unit/test_semantic_contract.py new file mode 100644 index 0000000..ff1ed16 --- /dev/null +++ b/tests/unit/test_semantic_contract.py @@ -0,0 +1,94 @@ +import tempfile +import unittest +from pathlib import Path + +from rfs.semantic_contract import apply_paper_semantic_contract + + +class SemanticContractTests(unittest.TestCase): + def test_card_ids_and_ocr_geometry_drive_semantic_mapping_and_hidden_innovation_is_skipped(self): + program = { + "slots": [ + {"id": "slot_micro_left", "bbox_percent": {"x": 0.08, "y": 0.3, "w": 0.08, "h": 0.1}}, + {"id": "slot_micro_middle", "bbox_percent": {"x": 0.43, "y": 0.3, "w": 0.08, "h": 0.1}}, + {"id": "slot_micro_right", "bbox_percent": {"x": 0.78, "y": 0.3, "w": 0.08, "h": 0.1}}, + ], + "cards": [ + {"id": "card_exact_input", "bbox_percent": {"x": 0.05, "y": 0.2, "w": 0.2, "h": 0.35}}, + {"id": "card_paper_method", "bbox_percent": {"x": 0.4, "y": 0.2, "w": 0.2, "h": 0.35}}, + {"id": "card_exact_output", "bbox_percent": {"x": 0.75, "y": 0.2, "w": 0.2, "h": 0.35}}, + ], + "panels": [], + "arrows": [ + {"id": "cross_card_guess", "source_id": "slot_micro_left", "target_id": "slot_micro_middle", "path_percent": [[0.16, 0.35], [0.43, 0.35]]}, + {"id": "local_method_detail", "source_id": "slot_micro_middle", "target_id": "slot_micro_middle", "path_percent": [[0.44, 0.32], [0.5, 0.38]]}, + ], + "text_program": {"items": [ + {"id": "ocr_input", "text": "Exact Input", "bbox_percent": {"x": 0.08, "y": 0.22, "w": 0.1, "h": 0.04}}, + {"id": "ocr_method", "text": "Paper Method", "bbox_percent": {"x": 0.44, "y": 0.22, "w": 0.1, "h": 0.04}}, + {"id": "ocr_output", "text": "Exact Output", "bbox_percent": {"x": 0.79, "y": 0.22, "w": 0.1, "h": 0.04}}, + ]}, + } + contract = {"figure_specification": { + "required_labels": ["Exact Input", "Paper Method", "Exact Output"], + "inputs": [{"id": "input", "name": "Exact Input", "evidence_ids": ["E1"]}], + "modules": [{"id": "method", "name": "Paper Method", "evidence_ids": ["E2"]}], + "outputs": [{"id": "output", "name": "Exact Output", "evidence_ids": ["E3"]}], + "innovations": [{"id": "hidden", "name": "Background Innovation", "evidence_ids": ["E4"]}], + "relations": [], + }} + with tempfile.TemporaryDirectory() as tmp: + updated, report = apply_paper_semantic_contract(program, contract, Path(tmp)) + updated, second_report = apply_paper_semantic_contract(updated, contract, Path(tmp)) + + mapped = {item["entity_id"]: item["object_id"] for item in report["mappings"]} + self.assertEqual(mapped, { + "input": "card_exact_input", + "method": "card_paper_method", + "output": "card_exact_output", + }) + self.assertEqual(report["status"], "pass") + self.assertEqual(report["low_confidence_mappings"], []) + self.assertIn("local_method_detail", {item["id"] for item in updated["arrows"]}) + self.assertNotIn("cross_card_guess", {item["id"] for item in updated["arrows"]}) + self.assertEqual(second_report["status"], "pass") + self.assertEqual({item["entity_id"]: item["object_id"] for item in second_report["mappings"]}, mapped) + self.assertEqual(len([item for item in updated["text_program"]["items"] if item.get("semantic_entity_id")]), 3) + self.assertNotIn("Background Innovation", {item["text"] for item in updated["text_program"]["items"]}) + + def test_exact_paper_labels_and_relations_override_image_interpretation(self): + program = { + "slots": [ + {"id": "left", "bbox_percent": {"x": 0.05, "y": 0.25, "w": 0.2, "h": 0.3}}, + {"id": "middle", "bbox_percent": {"x": 0.4, "y": 0.25, "w": 0.2, "h": 0.3}}, + {"id": "right", "bbox_percent": {"x": 0.75, "y": 0.25, "w": 0.2, "h": 0.3}}, + ], + "cards": [], + "panels": [], + "arrows": [{"id": "image_guess", "path_percent": [[0.1, 0.1], [0.9, 0.9]]}], + "text_program": {"items": [{"id": "ocr_wrong", "text": "lnput", "bbox_percent": {"x": 0.08, "y": 0.3, "w": 0.1, "h": 0.04}}]}, + } + contract = {"figure_specification": { + "inputs": [{"id": "input", "name": "Exact Input", "evidence_ids": ["E1"]}], + "modules": [{"id": "method", "name": "Paper Method", "evidence_ids": ["E2"]}], + "outputs": [{"id": "output", "name": "Exact Output", "evidence_ids": ["E3"]}], + "innovations": [], + "relations": [ + {"source": "input", "target": "method", "type": "data_flow", "evidence_ids": ["E4"]}, + {"source": "method", "target": "output", "type": "data_flow", "evidence_ids": ["E5"]}, + ], + }} + with tempfile.TemporaryDirectory() as tmp: + updated, report = apply_paper_semantic_contract(program, contract, Path(tmp)) + + labels = {item["text"] for item in updated["text_program"]["items"]} + self.assertEqual(labels, {"Exact Input", "Paper Method", "Exact Output"}) + self.assertEqual(report["mapped_relation_count"], 2) + for arrow in updated["arrows"]: + for start, end in zip(arrow["path_percent"], arrow["path_percent"][1:]): + self.assertTrue(start[0] == end[0] or start[1] == end[1], arrow["path_percent"]) + self.assertTrue((Path(tmp) / "semantic_binding_report.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_semantic_ppt.py b/tests/unit/test_semantic_ppt.py new file mode 100644 index 0000000..2e68ee9 --- /dev/null +++ b/tests/unit/test_semantic_ppt.py @@ -0,0 +1,144 @@ +import json +import tempfile +import unittest +import zipfile +from pathlib import Path + +from PIL import Image + +from rfs.paper_to_image.editable_ppt import build_semantic_figure_program, build_visual_substrate_figure_program, compile_semantic_ppt, compile_visual_substrate_ppt + + +class SemanticPptTests(unittest.TestCase): + def test_compiles_contract_to_native_editable_shapes_text_and_connectors(self): + specification = { + "topology": "linear", + "required_labels": ["Input Image", "Patch Encoder", "Transformer", "Prediction"], + "inputs": [{"id": "image", "name": "Input Image"}], + "modules": [ + {"id": "patch", "name": "Patch Encoder"}, + {"id": "transformer", "name": "Transformer"}, + ], + "outputs": [{"id": "prediction", "name": "Prediction"}], + "relations": [ + {"source": "image", "target": "patch", "type": "data_flow"}, + {"source": "patch", "target": "transformer", "type": "feature_flow"}, + {"source": "transformer", "target": "prediction", "type": "data_flow"}, + ], + } + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + report = compile_semantic_ppt(specification, root, aspect_ratio="16:9") + + self.assertTrue(report["ok"]) + self.assertEqual(report["node_count"], 4) + self.assertEqual(report["connector_count"], 3) + self.assertEqual(report["editable_layers"], ["native_shapes", "native_text", "native_connectors"]) + with zipfile.ZipFile(report["pptx"]) as archive: + slide_xml = archive.read("ppt/slides/slide1.xml").decode("utf-8") + for label in specification["required_labels"]: + self.assertIn(label, slide_xml) + self.assertGreaterEqual(slide_xml.count(""), 3) + self.assertGreaterEqual(slide_xml.count("RFS Semantic Node"), 4) + self.assertGreaterEqual(slide_xml.count("RFS Connector"), 3) + + quality = json.loads((root / "composition_quality_report.json").read_text(encoding="utf-8")) + self.assertEqual(len(quality["semantic_nodes"]), 4) + self.assertTrue(all(item["editable_in"] == "pptx" for item in quality["semantic_nodes"])) + + def test_feedback_contract_uses_external_dashed_editable_loop(self): + specification = { + "topology": "feedback", + "required_labels": ["Sample", "Measure", "Update", "Estimate"], + "inputs": [{"id": "sample", "name": "Sample"}], + "modules": [{"id": "measure", "name": "Measure"}, {"id": "update", "name": "Update"}], + "outputs": [{"id": "estimate", "name": "Estimate"}], + "relations": [ + {"source": "sample", "target": "measure", "type": "data_flow"}, + {"source": "measure", "target": "update", "type": "data_flow"}, + {"source": "update", "target": "estimate", "type": "data_flow"}, + {"source": "estimate", "target": "measure", "type": "feedback_loop", "label": "revise"}, + ], + } + + program = build_semantic_figure_program(specification, aspect_ratio="3:2") + loop = next(item for item in program["arrows"] if item["type"] == "feedback_loop") + self.assertEqual(loop["line_pattern"], "dash") + self.assertEqual(loop["route_style"], "outer_feedback") + self.assertEqual(program["labels"][0]["text"], "revise") + + def test_long_words_and_relation_labels_are_compiled_for_readable_rendering(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Encoder", "contextualized embeddings"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "encoder", "name": "Encoder"}], + "outputs": [{"id": "output", "name": "contextualized embeddings"}], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow", "label": "serialize"}, + {"source": "encoder", "target": "output", "type": "data_flow"}, + ], + } + + program = build_semantic_figure_program(specification) + output = next(item for item in program["semantic_nodes"] if item["id"] == "output") + relation_label = program["labels"][0] + + self.assertLessEqual(output["font_size_pt"], 10.5) + self.assertLess(relation_label["bbox_percent"]["w"], 0.13) + self.assertLess(relation_label["bbox_percent"]["y"], 0.5) + + def test_image2_substrate_is_below_native_editable_labels_and_connectors(self): + specification = { + "topology": "linear", + "required_labels": ["Input Image", "Patch Encoder", "Prediction"], + "inputs": [{"id": "image", "name": "Input Image"}], + "modules": [{"id": "patch", "name": "Patch Encoder"}], + "outputs": [{"id": "prediction", "name": "Prediction"}], + "relations": [ + {"source": "image", "target": "patch", "type": "data_flow"}, + {"source": "patch", "target": "prediction", "type": "prediction"}, + ], + } + overlay = { + "labels": [ + {"target_id": "image", "text": "Input Image"}, + {"target_id": "patch", "text": "Patch Encoder"}, + {"target_id": "prediction", "text": "Prediction"}, + ], + "connectors": [ + {"source": "image", "target": "patch", "type": "data_flow"}, + {"source": "patch", "target": "prediction", "type": "prediction"}, + ], + } + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + substrate = root / "candidate_raw.png" + Image.new("RGB", (1600, 900), "#F7F8F5").save(substrate) + + program = build_visual_substrate_figure_program(specification, substrate, aspect_ratio="16:9", overlay_spec=overlay) + self.assertTrue(program["visual_substrate_mode"]) + self.assertTrue(all(node["fill_transparency"] == 1.0 for node in program["semantic_nodes"])) + self.assertTrue(all(node.get("label_bbox_percent") for node in program["semantic_nodes"])) + + report = compile_visual_substrate_ppt(specification, substrate, root, aspect_ratio="16:9", overlay_spec=overlay) + + self.assertTrue(report["ok"]) + self.assertFalse(report["scientific_text_baked_into_visual_layer"]) + self.assertFalse(report["connectors_baked_into_visual_layer"]) + with zipfile.ZipFile(report["pptx"]) as archive: + slide_xml = archive.read("ppt/slides/slide1.xml").decode("utf-8") + self.assertGreaterEqual(slide_xml.count(""), 1) + self.assertEqual(slide_xml.count("RFS Semantic Label"), 3) + self.assertGreaterEqual(slide_xml.count(""), 2) + for label in specification["required_labels"]: + self.assertIn(label, slide_xml) + + quality = json.loads((root / "composition_quality_report.json").read_text(encoding="utf-8")) + self.assertGreaterEqual(quality["background_visual"]["canvas_area_fill_percent"], 99.0) + self.assertTrue(all(item["label_rendered_as"] == "separate_editable_header_shape" for item in quality["semantic_nodes"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_text_grouping.py b/tests/unit/test_text_grouping.py new file mode 100644 index 0000000..e3536f5 --- /dev/null +++ b/tests/unit/test_text_grouping.py @@ -0,0 +1,125 @@ +import unittest + +from rfs.text_grouping import group_text_regions, group_text_regions_heuristic + + +def _region(text: str, text_id: str, x: float, y: float, w: float, h: float, target_id: str = "card_a", role: str = "body_label") -> dict: + return { + "id": text_id, + "text": text, + "raw_text": text, + "role": role, + "ocr_role_guess": role, + "target_id": target_id, + "bbox_percent": {"x": x, "y": y, "w": w, "h": h}, + "center_percent": {"x": x + w / 2, "y": y + h / 2}, + "width_percent": w, + "height_percent": h, + "estimated_font_ratio": h * 0.9, + "raw_estimated_font_ratio": h * 0.9, + "font_size_pt": 8.0, + "raw_font_size_pt": 8.0, + "font_family_guess": "Arial", + "confidence": 0.95, + "color_hex": "#222222", + "source": "reference_ocr_text_region", + "editable_in": "pptx", + } + + +class TextGroupingTests(unittest.TestCase): + def test_multiline_body_text_is_merged_to_one_paragraph_region(self): + raw = [ + _region("first line of body", "ocr_1", 0.10, 0.20, 0.24, 0.030), + _region("second line continues", "ocr_2", 0.101, 0.234, 0.25, 0.031), + _region("third line continues", "ocr_3", 0.100, 0.269, 0.22, 0.030), + ] + + grouped, plan, report = group_text_regions_heuristic(raw) + + self.assertEqual(report["paragraph_group_count"], 1) + self.assertEqual(len(grouped), 1) + self.assertEqual(grouped[0]["ocr_member_ids"], ["ocr_1", "ocr_2", "ocr_3"]) + self.assertEqual(grouped[0]["text"], "first line of body\nsecond line continues\nthird line continues") + self.assertLess(grouped[0]["raw_estimated_font_ratio"], grouped[0]["height_percent"]) + self.assertEqual(plan["groups"][0]["group_id"], grouped[0]["id"]) + + def test_panel_titles_are_not_heuristically_paragraph_merged(self): + raw = [ + _region("Graph-based Textual", "ocr_title_1", 0.20, 0.04, 0.20, 0.030, target_id="panel_a", role="panel_title"), + _region("Knowledge Grounding", "ocr_title_2", 0.20, 0.075, 0.22, 0.030, target_id="panel_a", role="panel_title"), + ] + + grouped, _plan, report = group_text_regions_heuristic(raw) + + self.assertEqual(report["paragraph_group_count"], 0) + self.assertEqual(len(grouped), 2) + self.assertTrue(all(item["ocr_member_count"] == 1 for item in grouped)) + + def test_text_from_different_targets_does_not_merge(self): + raw = [ + _region("card A line one", "ocr_a1", 0.10, 0.20, 0.20, 0.030, target_id="card_a"), + _region("card B line one", "ocr_b1", 0.10, 0.234, 0.20, 0.030, target_id="card_b"), + ] + + grouped, _plan, report = group_text_regions_heuristic(raw) + + self.assertEqual(report["paragraph_group_count"], 0) + self.assertEqual(len(grouped), 2) + self.assertEqual({item["target_id"] for item in grouped}, {"card_a", "card_b"}) + + def test_vlm_grouping_uses_raw_ocr_union_not_vlm_bbox(self): + raw = [ + _region("line one", "ocr_1", 0.10, 0.20, 0.20, 0.030), + _region("line two", "ocr_2", 0.101, 0.234, 0.22, 0.030), + _region("noise", "ocr_noise", 0.70, 0.80, 0.04, 0.012, role="free_text"), + ] + + def fake_adapter(_reference, _raw_regions, _heuristic_regions, _program, _model): + return { + "summary": "fake grouping", + "groups": [{ + "group_id": "body_para", + "ocr_member_ids": ["ocr_1", "ocr_2"], + "role": "annotation", + "align": "left", + "bbox_percent": {"x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0}, + "confidence": 0.93, + "reason": "two visible body lines", + }], + "ignored_ocr_ids": ["ocr_noise"], + } + + grouped, plan, report = group_text_regions(raw, mode="vlm", adapter=fake_adapter, reference_path="reference.png", program={}) + + self.assertEqual(report["status"], "pass") + self.assertEqual(len(grouped), 1) + self.assertEqual(grouped[0]["id"], "body_para") + self.assertEqual(grouped[0]["role"], "annotation") + self.assertEqual(grouped[0]["align"], "left") + self.assertLess(grouped[0]["bbox_percent"]["x"], 0.105) + self.assertGreater(grouped[0]["bbox_percent"]["x"], 0.09) + self.assertLess(grouped[0]["bbox_percent"]["w"], 0.23) + self.assertEqual(plan["ignored_ocr_ids"], ["ocr_noise"]) + + def test_hybrid_grouping_falls_back_to_heuristic_when_vlm_fails(self): + raw = [ + _region("first line", "ocr_1", 0.10, 0.20, 0.20, 0.030), + _region("second line", "ocr_2", 0.10, 0.234, 0.20, 0.030), + ] + + def failing_adapter(*_args): + raise RuntimeError("grouping unavailable") + + grouped, plan, report = group_text_regions(raw, mode="hybrid", adapter=failing_adapter, reference_path="reference.png", program={}) + + self.assertEqual(report["status"], "fallback_to_heuristic") + self.assertEqual(report["effective_mode"], "heuristic") + self.assertEqual(len(grouped), 1) + self.assertEqual(grouped[0]["ocr_member_ids"], ["ocr_1", "ocr_2"]) + self.assertIn("grouping unavailable", report["warnings"][0]) + self.assertEqual(plan["effective_mode"], "heuristic") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_text_layer_ocr.py b/tests/unit/test_text_layer_ocr.py similarity index 79% rename from tests/test_text_layer_ocr.py rename to tests/unit/test_text_layer_ocr.py index 1d74238..b15007e 100644 --- a/tests/test_text_layer_ocr.py +++ b/tests/unit/test_text_layer_ocr.py @@ -8,6 +8,7 @@ from rfs.ppt_compiler import compile_ppt from rfs.text_layer import build_text_layer +from rfs.text_layer_ownership import apply_text_layer_ownership def _base_program() -> dict: @@ -40,6 +41,24 @@ def _style() -> dict: class TextLayerOcrTests(unittest.TestCase): + def test_text_ownership_keeps_critical_labels_editable(self): + program = { + "slots": [{"id": "slot_a", "bbox_percent": {"x": 0.1, "y": 0.1, "w": 0.5, "h": 0.5}}] + } + regions = [ + {"id": "title", "text": "Method", "role": "panel_title", "target_id": "slot_a", "font_size_pt": 12, "bbox_percent": {"x": 0.2, "y": 0.2, "w": 0.2, "h": 0.1}}, + {"id": "inferred_title", "text": "Architecture", "role": "panel_title", "target_id": "slot_a", "bbox_percent": {"x": 0.2, "y": 0.1, "w": 0.2, "h": 0.05}}, + {"id": "decorative", "text": "x", "role": "free_text", "target_id": "slot_a", "font_size_pt": 3, "bbox_percent": {"x": 0.3, "y": 0.3, "w": 0.05, "h": 0.05}}, + ] + + planned, _plan, report = apply_text_layer_ownership(regions, program) + + by_id = {item["id"]: item for item in planned} + self.assertEqual(by_id["title"]["layer_ownership"], "editable_text_layer") + self.assertEqual(by_id["inferred_title"]["layer_ownership"], "editable_text_layer") + self.assertEqual(by_id["decorative"]["layer_ownership"], "decorative_asset_text") + self.assertEqual(report["editable_text_count"], 2) + def test_fake_ocr_creates_reference_text_geometry_without_duplicate_panel_title(self): with tempfile.TemporaryDirectory() as tmp: out = Path(tmp) diff --git a/tests/unit/test_vlm_client.py b/tests/unit/test_vlm_client.py new file mode 100644 index 0000000..beef1b0 --- /dev/null +++ b/tests/unit/test_vlm_client.py @@ -0,0 +1,47 @@ +import unittest +from unittest.mock import patch + +import requests + +from rfs.vlm_client import call_vlm_json + + +class _Response: + def raise_for_status(self): + return None + + def json(self): + return {"choices": [{"message": {"content": '{"ok": true}'}}]} + + +class VlmClientTests(unittest.TestCase): + @patch("rfs.vlm_client.time.sleep", return_value=None) + @patch("rfs.vlm_client.requests.post", side_effect=[requests.exceptions.SSLError("EOF during TLS"), _Response()]) + def test_retry_metadata_records_recovery_without_secrets(self, _post, _sleep): + metadata = {} + with patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-token"}, clear=False): + result = call_vlm_json("Return JSON", [], model="test-model", timeout=10, retries=1, call_metadata=metadata) + + self.assertTrue(result["ok"]) + self.assertEqual(metadata["attempts"], 2) + self.assertEqual(metadata["retries_used"], 1) + self.assertTrue(metadata["success"]) + self.assertEqual(metadata["failure_categories"], ["tls"]) + self.assertNotIn("secret-token", str(metadata)) + + @patch("rfs.vlm_client.requests.post") + def test_expired_deadline_skips_provider_call(self, post): + metadata = {} + with patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-token"}, clear=False): + with self.assertRaisesRegex(TimeoutError, "deadline budget exhausted"): + call_vlm_json("Return JSON", [], model="test-model", timeout=30, retries=2, call_metadata=metadata, deadline_at=0.0) + + post.assert_not_called() + self.assertEqual(metadata["attempts"], 0) + self.assertEqual(metadata["retries_used"], 0) + self.assertEqual(metadata["failure_categories"], ["timeout"]) + self.assertTrue(metadata["deadline_reached"]) + + +if __name__ == "__main__": + unittest.main()