From fbf06ade3a8725ed56af5e3cb4e9f2735d75ae92 Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Sun, 4 Oct 2026 17:05:18 -0500 Subject: [PATCH 1/4] fix(text): keep table values in their columns A "$" merged into its number dropped the symbol's cell, so every value after it sat one column left of the rows without one; a value spanning the symbol and number columns (colspan 2) landed apart from one beside a "$". A self-closing spacer swallowed the cell after it. Merges now keep each cell's position, a symbol column joins the number's where no row fills both, and a self-closing cell reads as empty. --- tests/test_text_tables.py | 70 +++++++++++++++++++++++++++++++++++++++ xbrlkit/text/tables.py | 28 +++++++++++++--- 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/tests/test_text_tables.py b/tests/test_text_tables.py index addbe75..b44ea0f 100644 --- a/tests/test_text_tables.py +++ b/tests/test_text_tables.py @@ -77,6 +77,12 @@ def test_th_elements(self): def test_empty_table(self): assert _parse_rows("
") == [] + def test_self_closing_cell_is_empty(self): + """Apple's filer writes spacers as ````; read as an open + tag one swallowed the next cell, and the row came up a column short.""" + html = 'Europe111,032' + assert _parse_rows(html) == [[("Europe", 1), ("", 3), ("111,032", 1), ("", 1)]] + @pytest.mark.unit class TestMergeCurrencyCells: @@ -108,6 +114,70 @@ def test_no_merge_needed(self): result = _merge_currency_cells(rows) assert result == [["A", "B", "C"]] + def test_dollar_rows_stay_in_their_columns(self): + """A "$" merge used to drop the symbol's cell, so every value after it + moved one column left of the rows without one.""" + rows = [ + [ + ("Income", 1), + ("$", 1), + ("11,670", 1), + ("", 1), + ("$", 1), + ("(22,146", 1), + (")", 1), + ], + [ + ("Add back", 1), + ("", 1), + ("29,751", 1), + ("", 1), + ("", 1), + ("28,467", 1), + ("", 1), + ], + ] + result = _merge_currency_cells(rows) + assert result == [ + ["Income", "$11,670", "$(22,146)"], + ["Add back", "29,751", "28,467"], + ] + + def test_values_spanning_the_symbol_column_align(self): + """Workiva's layout: a value with no "$" spans the symbol and number + columns (colspan 2), as the period header spans all three, so both start + in the symbol's column while a "$" value sits one to the right.""" + rows = [ + [("", 1), ("2026", 3), ("", 1), ("2025", 3)], + [ + ("Income", 1), + ("$", 1), + ("11,670", 1), + ("", 1), + ("", 1), + ("$", 1), + ("8,233", 1), + ("", 1), + ], + [("Add back", 1), ("29,751", 2), ("", 1), ("", 1), ("28,467", 2), ("", 1)], + ] + result = _merge_currency_cells(rows) + assert result == [ + ["", "2026", "2025"], + ["Income", "$11,670", "$8,233"], + ["Add back", "29,751", "28,467"], + ] + + def test_symbol_column_holding_values_is_kept(self): + """A column that holds a value in the same row as its neighbour is a + column in its own right, not a symbol's, and is not joined.""" + rows = [ + [("A", 1), ("$", 1), ("1", 1)], + [("B", 1), ("2", 1), ("3", 1)], + ] + result = _merge_currency_cells(rows) + assert result == [["A", "", "$1"], ["B", "2", "3"]] + @pytest.mark.unit class TestIsLayoutTable: diff --git a/xbrlkit/text/tables.py b/xbrlkit/text/tables.py index 6f95bfa..25feee0 100644 --- a/xbrlkit/text/tables.py +++ b/xbrlkit/text/tables.py @@ -171,11 +171,13 @@ def _parse_rows(table_html: str) -> list[list[tuple[str, int]]]: tr_content = tr_match.group(1) cells: list[tuple[str, int]] = [] + # A self-closing ```` is an empty cell; read as an open + # tag it swallowed the cell after it up to that one's ````. for cell_match in re.finditer( - r"<(t[dh])([^>]*)>(.*?)", tr_content, re.DOTALL | re.IGNORECASE + r"<(t[dh])\b([^>]*?)(?:/>|>(.*?))", tr_content, re.DOTALL | re.IGNORECASE ): attrs = cell_match.group(2) - inner = cell_match.group(3) + inner = cell_match.group(3) or "" text = _cell_text(inner) colspan = _get_colspan(attrs) cells.append((text, colspan)) @@ -192,8 +194,15 @@ def _merge_currency_cells(rows: list[list[tuple[str, int]]]) -> list[list[str]]: SEC filings often put "$" in its own , separate from the number. This merges ["$", "11,866.1"] into ["$11,866.1"] and expands colspan. Also drops columns that are empty across all rows (SEC spacer cells). + + A merge keeps every cell's position, so a row with a "$" stays as wide as + one without. The column a "$" stood in is then joined to the number's + column wherever no row fills both: a value with no "$" often spans the two + (colspan 2) and lands in the symbol's column, as do the period headers + above, while a "$" row's value lands one to the right. """ merged: list[list[str]] = [] + symbol_cols: set[int] = set() for row in rows: # Expand colspan first @@ -214,10 +223,11 @@ def _merge_currency_cells(rows: list[list[tuple[str, int]]]) -> list[list[str]]: while j < len(expanded) and not expanded[j].strip(): j += 1 if j < len(expanded): - # Add empty cells between as-is, then the merged cell - for k in range(i + 1, j): + # The symbol's cell and any between stay, empty; then the merged cell + for _ in range(i, j): final.append("") final.append(cell + expanded[j]) + symbol_cols.update(range(i, j)) i = j + 1 else: final.append(cell) @@ -227,6 +237,7 @@ def _merge_currency_cells(rows: list[list[tuple[str, int]]]) -> list[list[str]]: for k in range(len(final) - 1, -1, -1): if final[k].strip(): final[k] = final[k] + cell + final.append("") break else: final.append(cell) @@ -245,6 +256,15 @@ def _merge_currency_cells(rows: list[list[tuple[str, int]]]) -> list[list[str]]: while len(row) < max_cols: row.append("") + for col in sorted(symbol_cols, reverse=True): + if col + 1 < max_cols and not any( + row[col].strip() and row[col + 1].strip() for row in merged + ): + for row in merged: + row[col] = row[col] or row[col + 1] + del row[col + 1] + max_cols -= 1 + keep_cols = [ col for col in range(max_cols) From daf1f38b7c79136b9e0e5368d85906b72e12da14 Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Sun, 4 Oct 2026 17:06:31 -0500 Subject: [PATCH 2/4] fix(serve): read a document's form from its cover, not a citation A document opened on its own took the first "Form 10-K" anywhere in its first 400,000 characters as its form, so an earnings release whose safe harbor cites the annual report loaded as a 10-K. A cover names its form in its first few hundred characters; only those are read now. --- tests/test_serve.py | 22 +++++++++++++++++++++- xbrlkit/serve/session.py | 7 ++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/test_serve.py b/tests/test_serve.py index 77eee05..92dd755 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -35,7 +35,12 @@ ) from xbrlkit.serve import FilingSession, LoadedFiling, SourceError, build_text from xbrlkit.serve import tools -from xbrlkit.serve.session import _find_load_target, _locate, _restore_sec_identity +from xbrlkit.serve.session import ( + _find_load_target, + _form_on_the_cover, + _locate, + _restore_sec_identity, +) US_GAAP = "http://fasb.org/us-gaap/2024-01-31" IS_ROLE = "http://acme.example/role/StatementOfIncome" @@ -742,6 +747,21 @@ def test_locate_matches_across_renderings() -> None: assert _locate(text, "NOTE 19. Commitments") is None # too short to trust +def test_form_on_the_cover_reads_only_the_cover() -> None: + cover = ( + "

UNITED STATES SECURITIES AND EXCHANGE COMMISSION

" + "

Washington, D.C. 20549

FORM 8-K

CURRENT REPORT

" + ) + assert _form_on_the_cover(cover) == "8-K" + # A press release cites the annual report in its safe harbor; it is not one. + release = ( + "

Workiva Announces Second Quarter 2026 Financial Results

" + + "

Revenue grew.

" * 300 + + "

See our annual report on Form 10-K for risk factors.

" + ) + assert _form_on_the_cover(release) is None + + def test_locate_skips_a_contents_row_that_carries_the_whole_heading() -> None: heading = ( "Item 7. Management's Discussion and Analysis of Financial Condition and " diff --git a/xbrlkit/serve/session.py b/xbrlkit/serve/session.py index dc803c3..b318585 100644 --- a/xbrlkit/serve/session.py +++ b/xbrlkit/serve/session.py @@ -1962,11 +1962,16 @@ def first(names: tuple[str, ...]) -> str | None: _COVER_FORM_RE = re.compile( r"\bFORM\s+(10-K|10-Q|20-F|40-F|8-K|S-1|S-3|DEF\s*14A)\b", re.IGNORECASE ) +# A cover names its form within a few hundred characters. Past that, "Form +# 10-K" is a citation — a press release's safe harbor pointing at the annual +# report — and the document is not one. +_COVER_CHARS = 2_000 def _form_on_the_cover(html: str) -> str | None: """The form a document names on its cover, from the top of the document.""" - m = _COVER_FORM_RE.search(_html_to_text(html[:400_000])) + text = _html_to_text(html[:400_000])[:_COVER_CHARS] + m = _COVER_FORM_RE.search(text) return re.sub(r"\s+", " ", m.group(1)).upper() if m else None From 3161c37e7affad3dc02ede11b284902d629863a4 Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Sun, 4 Oct 2026 17:11:35 -0500 Subject: [PATCH 3/4] feat(serve): read an 8-K with its EX-99 exhibits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An 8-K loaded from EDGAR was its cover page alone, so search_text found nothing the release says and told the caller to reword. Its EX-99 exhibits are now read into the same text, each a section under its type with its offset — the layout the RoboSystems sec server reads an 8-K in, so the two answer alike. join_texts is the assembly, exported for a host that builds the text itself. The items note and next steps point at the text when the exhibits are in it; an 8-K whose exhibits cannot be reached loads alone. --- tests/test_serve.py | 125 ++++++++++++++++++++++++++++++++++++ tests/test_serve_surface.py | 4 ++ xbrlkit/edgar/items.py | 26 +++++--- xbrlkit/serve/README.md | 11 +++- xbrlkit/serve/__init__.py | 12 +++- xbrlkit/serve/server.py | 13 ++-- xbrlkit/serve/session.py | 91 +++++++++++++++++++++++++- xbrlkit/serve/tools.py | 13 +++- 8 files changed, 274 insertions(+), 21 deletions(-) diff --git a/tests/test_serve.py b/tests/test_serve.py index 92dd755..4f235c3 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -17,6 +17,7 @@ import textwrap from datetime import date from pathlib import Path +from typing import Any import pytest @@ -2421,6 +2422,9 @@ def test_an_earnings_8k_is_identified_and_points_at_the_exhibit() -> None: assert items_note(["9.01"]) is not None assert items_note(["5.07"]) is not None assert items_note([]) is None + # With the exhibits read into the text, the text tools are the way in. + joined = items_note(["2.02", "9.01"], exhibits_in_text=True) + assert joined is not None and "search_text" in joined and "documents" not in joined def test_filing_meta_carries_the_items_off_an_edgar_ref() -> None: @@ -2477,6 +2481,127 @@ def test_an_8k_describes_its_items_and_leads_with_the_exhibit() -> None: assert "documents" in out["next"][0] and "Item 2.02" in out["next"][0] +def test_join_texts_sets_each_document_under_its_heading() -> None: + from xbrlkit.serve import TextSection, join_texts + + inner = TextSection(id="item_2_02", label="Item 2.02", kind="item", chars=5, offset=3) + text, sections = join_texts( + [ + ("form_8k", "Form 8-K", "the cover", [inner]), + ("ex_10_1", "EX-10.1", "", []), + ("ex_99_1", "EX-99.1", "the release", []), + ] + ) + assert text == "## Form 8-K\nthe cover\n\n## EX-99.1\nthe release\n\n" + by_id = {s.id: s for s in sections} + assert set(by_id) == { + "form_8k", + "item_2_02", + "ex_99_1", + } # an empty body is no section + release = by_id["ex_99_1"] + assert text[release.offset : release.offset + release.chars] == "the release" + assert text[release.offset - release.heading_chars :].startswith("## EX-99.1\n") + # A document's own sections move with it. + assert by_id["item_2_02"].offset == by_id["form_8k"].offset + 3 + + +_EIGHT_K = ( + "

UNITED STATES SECURITIES AND EXCHANGE COMMISSION

" + "

FORM 8-K

Item 2.02 Results of Operations and Financial Condition. " + "The press release is furnished as Exhibit 99.1.

" +) + + +class _EightKExhibits: + """An 8-K's index and exhibits, answered from memory.""" + + def __init__(self, fail: bool = False) -> None: + self.fail = fail + + def other_documents(self, lf: object) -> list[Any]: + from xbrlkit.edgar.filing_index import FilingDocument + + if self.fail: + raise SourceError("the index could not be fetched") + return [ + FilingDocument(seq=2, type="EX-10.1", document="credit.htm"), + FilingDocument(seq=3, type="EX-99.1", document="release.htm"), + FilingDocument(seq=4, type="GRAPHIC", document="logo.jpg"), + ] + + def read_other_document(self, lf: object, name: str) -> Any: + from xbrlkit.serve.session import ReadDocument + + assert name == "release.htm", "only the EX-99 is read into the text" + return ReadDocument("Free cash flow was $78 million for the quarter.", []) + + +def test_an_8k_is_read_with_its_release(tmp_path: Path, monkeypatch: Any) -> None: + """Read alone, an earnings 8-K is a cover page: a search for anything the + release says found nothing and told the caller to reword.""" + path = tmp_path / "acme-8k.htm" + path.write_text(_EIGHT_K) + session = FilingSession() + try: + lf = session.load(str(path)) + lf.model.filing.items = ["2.02", "9.01"] + assert lf.model.filing.form == "8-K" + stub = _EightKExhibits() + monkeypatch.setattr(session, "other_documents", stub.other_documents) + monkeypatch.setattr(session, "read_other_document", stub.read_other_document) + lf = session._with_exhibits(lf) + + assert lf.exhibits_in_text == ["EX-99.1"] + hits = tools.search_text(lf, "free cash flow") + assert hits["total"] == 1 and hits["hits"][0]["section"] == "EX-99.1" + assert "FORM 8-K" in tools.search_text(lf, "FORM 8-K")["hits"][0]["text"] + + out = tools.describe_filing(lf) + items = {s["id"]: s for s in out["sections"]["items"]} + assert {"form_8k", "ex_99_1"} <= set(items) + page = tools.read_text(lf, offset=items["ex_99_1"]["offset"]) + assert page["text"].startswith("Free cash flow") + assert "part of this text" in out["filing"]["items_note"] + assert out["next"][0].startswith("read_text from the EX-99.1") + finally: + session.close() + + +def test_an_8k_whose_exhibits_cannot_be_reached_loads_alone( + tmp_path: Path, monkeypatch: Any +) -> None: + path = tmp_path / "acme-8k.htm" + path.write_text(_EIGHT_K) + session = FilingSession() + try: + lf = session.load(str(path)) + before = lf.text + monkeypatch.setattr( + session, "other_documents", _EightKExhibits(fail=True).other_documents + ) + lf = session._with_exhibits(lf) + assert lf.text == before and lf.exhibits_in_text == [] + finally: + session.close() + + +def test_only_an_8k_reaches_for_its_exhibits(tmp_path: Path, monkeypatch: Any) -> None: + path = tmp_path / "form10-k.htm" + path.write_text("

FORM 10-K

Annual report.

") + session = FilingSession() + try: + lf = session.load(str(path)) + + def refuse(lf: object) -> list[Any]: + raise AssertionError("a 10-K's exhibits are not read into its text") + + monkeypatch.setattr(session, "other_documents", refuse) + assert session._with_exhibits(lf).exhibits_in_text == [] + finally: + session.close() + + def test_a_filing_with_no_items_says_nothing_about_them(loaded: LoadedFiling) -> None: out = tools.describe_filing(loaded) assert "items" not in out["filing"] and "items_note" not in out["filing"] diff --git a/tests/test_serve_surface.py b/tests/test_serve_surface.py index 14d88e2..853ff3d 100644 --- a/tests/test_serve_surface.py +++ b/tests/test_serve_surface.py @@ -23,6 +23,7 @@ # has to be. The six marked (RoboSystems) are the ones in use today — # ``robosystems/operations/roboledger/views/information_blocks.py``. DECLARED: dict[str, str] = { + "FORM_SECTION_ID": "str", "FilingSession": "class", "LoadedFiling": "class", # RoboSystems "SourceError": "class", @@ -34,6 +35,7 @@ "build_text": "callable", "disclosures": "callable", # RoboSystems "information_block": "callable", # RoboSystems + "join_texts": "callable", "serve": "callable", } @@ -53,6 +55,8 @@ def test_every_declared_name_resolves() -> None: assert isinstance(obj, type), f"{name} should be a class" elif kind == "int": assert isinstance(obj, int), f"{name} should be an int" + elif kind == "str": + assert isinstance(obj, str), f"{name} should be a str" else: assert callable(obj), f"{name} should be callable" diff --git a/xbrlkit/edgar/items.py b/xbrlkit/edgar/items.py index 528659d..9e7fa50 100644 --- a/xbrlkit/edgar/items.py +++ b/xbrlkit/edgar/items.py @@ -108,36 +108,44 @@ def is_earnings_release(codes: list[str] | None) -> bool: return EARNINGS_ITEM in (codes or []) -def items_note(codes: list[str] | None) -> str | None: +def items_note(codes: list[str] | None, exhibits_in_text: bool = False) -> str | None: """What a reader should do next, given what the filing is coded as. An 8-K's tagged content is its cover page; the substance is the exhibit. The note says so, and says it most strongly where the exhibit is the earnings release — the numbers a company leads with, which no XBRL anywhere carries. + ``exhibits_in_text`` when the exhibits were read into the filing's text, so + the text tools reach them and ``documents`` is not the way in. """ codes = codes or [] if not codes: return None + if exhibits_in_text: + reach = ( + "The exhibits are part of this text, each under `sections.items` with " + "its `offset`: read_text from one, or search_text across the 8-K and " + "all of them." + ) + else: + reach = "Call documents, then read_document on the exhibit." if is_earnings_release(codes): return ( "Item 2.02 — this is an earnings release. The results are in the " "attached exhibit (usually EX-99.1), not in this filing's XBRL, which " - "is the cover page only. Call documents, then read_document on the " - "EX-99.1. The exhibit carries what the company leads with — non-GAAP " - "measures, adjusted EBITDA, segment detail and guidance — which no XBRL " - "holds, and it arrives weeks before the 10-Q that restates part of it." + f"is the cover page only. {reach} The exhibit carries what the company " + "leads with — non-GAAP measures, adjusted EBITDA, segment detail and " + "guidance — which no XBRL holds, and it arrives weeks before the 10-Q " + "that restates part of it." ) if FD_ITEM in codes: return ( "Item 7.01 — Regulation FD disclosure. The substance is normally an " - "exhibit (a presentation, a script, a release): call documents, then " - "read_document." + f"exhibit (a presentation, a script, a release). {reach}" ) if EXHIBITS_ITEM in codes: return ( "This 8-K has exhibits (Item 9.01) and an 8-K's tagged content is its " - "cover page, so the substance is in them: call documents, then " - "read_document." + f"cover page, so the substance is in them. {reach}" ) return ( "An 8-K's tagged content is its cover page; anything more is in the " diff --git a/xbrlkit/serve/README.md b/xbrlkit/serve/README.md index 27de4ee..6f2d753 100644 --- a/xbrlkit/serve/README.md +++ b/xbrlkit/serve/README.md @@ -137,7 +137,16 @@ So an earnings 8-K is identifiable **before** anything is fetched: attached exhibit (usually EX-99.1), not in this filing's XBRL …" ``` -and `next` leads with `documents` rather than the fact tools. +and `next` leads with the release rather than the fact tools. + +**An 8-K loaded from EDGAR is read with its `EX-99` exhibits.** The filing's +own text comes first under `## Form 8-K`, then each exhibit under its type +(`## EX-99.1`), every one a section in `sections.items` with its offset — so +`search_text` finds what the release says and `read_text` pages it, with no +second document to know about. It costs the index fetch `documents` makes and +one fetch per exhibit, both kept for `read_document`; when they cannot be +reached the 8-K loads alone. Other exhibits — a credit agreement, a +certification — stay in `documents`. **Why this matters more than a routing convenience.** The exhibit carries what the company leads with — adjusted EBITDA, non-GAAP margin, segment detail, diff --git a/xbrlkit/serve/__init__.py b/xbrlkit/serve/__init__.py index ed1242d..e5dd454 100644 --- a/xbrlkit/serve/__init__.py +++ b/xbrlkit/serve/__init__.py @@ -34,7 +34,15 @@ from pathlib import Path from typing import Any -from .session import FilingSession, LoadedFiling, SourceError, TextSection, build_text +from .session import ( + FORM_SECTION_ID, + FilingSession, + LoadedFiling, + SourceError, + TextSection, + build_text, + join_texts, +) from .tools import ( MAX_BLOCK_MEMBERS_CAP, MAX_BLOCK_ROWS, @@ -44,6 +52,7 @@ ) __all__ = ( + "FORM_SECTION_ID", "MAX_BLOCK_MEMBERS_CAP", "MAX_BLOCK_ROWS", "FilingSession", @@ -55,6 +64,7 @@ "build_text", "disclosures", "information_block", + "join_texts", "serve", ) diff --git a/xbrlkit/serve/server.py b/xbrlkit/serve/server.py index 581cd72..4441b52 100644 --- a/xbrlkit/serve/server.py +++ b/xbrlkit/serve/server.py @@ -120,6 +120,10 @@ non-GAAP measures, adjusted EBITDA, segment detail, guidance — which no XBRL \ anywhere holds, and it lands weeks before the 10-Q that restates part of it \ in GAAP. Treat it as a primary source, not as a supplement to the 10-Q. +- An 8-K loaded from EDGAR is read with its EX-99 exhibits: each is a section \ +of the same text, under `sections.items` with its offset, so search_text \ +finds the release and read_text pages it. An exhibit loaded on its own, by its \ +URL, reads as a document with no filing around it. RECORDS (XML filings) - records returns the form's own tables — a Form 4's transactions and \ @@ -692,11 +696,10 @@ def information_block( description=( "What else was filed with this filing — the exhibits, and any second " "document the content actually lives in (an 8-K's EX-99.1 press " - "release, a 13F's INFORMATION TABLE of holdings). On an 8-K this is the " - "tool that answers the question: its tagged content is the cover page, " - "so the substance is always here — and on an Item 2.02 the EX-99.1 is " - "the earnings release, carrying the non-GAAP measures and guidance no " - "XBRL holds. Costs one small fetch the first time and nothing after. " + "release, a 13F's INFORMATION TABLE of holdings). An 8-K's EX-99 " + "exhibits are already in its text, for search_text and read_text; this " + "lists them with the rest. Costs one small fetch the first time and " + "nothing after. " "Every document carries its URL and says whether read_document can read " "it: a PDF or an image is listed with its address so a caller that can " "open one may fetch it directly. The XBRL package and the SEC's own " diff --git a/xbrlkit/serve/session.py b/xbrlkit/serve/session.py index b318585..b1858c4 100644 --- a/xbrlkit/serve/session.py +++ b/xbrlkit/serve/session.py @@ -19,7 +19,7 @@ import threading from concurrent.futures import ThreadPoolExecutor from collections import Counter -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import date from pathlib import Path from collections.abc import Mapping @@ -52,6 +52,13 @@ SectionKind = Literal["item", "text_block", "records"] +# An 8-K's own text is its cover and its items; what it discloses — the +# release, the slides, the script — is an EX-99 filed beside it, so the two +# are read as one text. +_CURRENT_REPORT_FORMS = frozenset({"8-K", "8-K/A"}) +_CURRENT_REPORT_EXHIBIT = "EX-99" +FORM_SECTION_ID = "form_8k" + _INLINE_SUFFIXES = {".htm", ".html", ".xhtml"} _PLAIN_SUFFIXES = {".txt", ".md"} # A report serialized as JSON — read into the model here, never by Arelle. @@ -162,6 +169,9 @@ class LoadedFiling: # and each one read on first read. Nothing is fetched until something asks. other_documents: list[FilingDocument] | None = None read_documents: dict[str, "ReadDocument"] = field(default_factory=dict) + # The exhibits read into ``text`` after the filing's own document, by type + # (``EX-99.1``): an 8-K's release, searched and paged with the form. + exhibits_in_text: list[str] = field(default_factory=list) # The document the filing was loaded from, when it already is one of the # serializations this server hands out (a holon, a TAVI), and which one. # ``view_filing`` serves that document as it is rather than a re-projection @@ -737,6 +747,39 @@ def read_other_document(self, lf: LoadedFiling, name: str) -> ReadDocument: lf.read_documents[match.document] = read return read + def _with_exhibits(self, lf: LoadedFiling) -> LoadedFiling: + """An 8-K read with its EX-99 exhibits, one text the text tools search. + + Read alone, an 8-K is a cover page and a line saying a release was + furnished: a search for anything the release says finds nothing, and the + caller is left to know the exhibit is a second document. Costs the index + fetch ``documents`` makes and one fetch per exhibit, both kept for + ``read_document``. A filing whose exhibits cannot be reached loads as its + 8-K alone, as it did before. + """ + form = (lf.model.filing.form or "").upper() + if form not in _CURRENT_REPORT_FORMS or not lf.has_document: + return lf + try: + exhibits = [ + d + for d in self.other_documents(lf) + if d.type.upper().startswith(_CURRENT_REPORT_EXHIBIT) and d.is_readable + ] + read = [(d, self.read_other_document(lf, d.document)) for d in exhibits] + except (SourceError, requests.RequestException, OSError) as exc: + logger.warning("%s loads without its exhibits: %s", lf.accession, exc) + return lf + if not read: + return lf + parts = [(FORM_SECTION_ID, f"Form {form}", lf.text, lf.sections)] + parts += [ + (_exhibit_section_id(d.type), d.type.upper(), r.text, r.sections) for d, r in read + ] + lf.text, lf.sections = join_texts(parts) + lf.exhibits_in_text = [d.type.upper() for d, r in read if r.text] + return lf + def _edgar_coordinates(self, lf: LoadedFiling) -> tuple[str, str]: """The CIK and accession needed to reach back to EDGAR for this filing.""" cik, accession = lf.model.filing.cik, lf.accession @@ -819,7 +862,7 @@ def _load_edgar(self, cik: str, accession: str, source: str) -> LoadedFiling: model = self._parse(target, accession, filing=filing, entity=None) loaded = self._finish(accession, source, model, target, package_dir, document) _enrich_filer(loaded, self._filer_metadata(cik, client=client)) - return loaded + return self._with_exhibits(loaded) def _load_document_only( self, client: Any, cik: str, accession: str, ref: Any, source: str @@ -857,7 +900,7 @@ def _load_document_only( # No XBRL, so no cover page to read: the submissions header is the only # account of the filer this filing has. _enrich_filer(loaded, self._filer_metadata(cik, client=client)) - return loaded + return self._with_exhibits(loaded) def _load_from_submission( self, client: Any, cik: str, accession: str, ref: Any, source: str @@ -1354,6 +1397,48 @@ def _read_document(doc: Path | None, model: XbrlModel) -> ReadDocument | None: return None +def join_texts( + parts: list[tuple[str, str, str, list[TextSection]]], +) -> tuple[str, list[TextSection]]: + """Several documents read as one text, in the order given. + + ``parts`` is ``(id, label, text, sections)`` for each document. Each is set + under a ``## label`` heading and becomes a section whose offset is its + body's start, with the heading counted in ``heading_chars``; its own + sections move with it. + """ + chunks: list[str] = [] + sections: list[TextSection] = [] + offset = 0 + for section_id, label, body, inner in parts: + if not body: + continue + header = f"## {label}\n" + start = offset + len(header) + sections.append( + TextSection( + id=section_id, + label=label, + kind="item", + chars=len(body), + offset=start, + heading_chars=len(header), + ) + ) + sections.extend( + replace(s, offset=None if s.offset is None else s.offset + start) for s in inner + ) + chunk = header + body + "\n\n" + chunks.append(chunk) + offset += len(chunk) + return "".join(chunks), sections + + +def _exhibit_section_id(exhibit_type: str) -> str: + """``ex_99_1`` for ``EX-99.1``.""" + return re.sub(r"[^a-z0-9]+", "_", exhibit_type.lower()).strip("_") + + def build_text(model: XbrlModel, html: str | None) -> tuple[str, list[TextSection]]: """The filing as plain text plus its section map. diff --git a/xbrlkit/serve/tools.py b/xbrlkit/serve/tools.py index eba6713..042e142 100644 --- a/xbrlkit/serve/tools.py +++ b/xbrlkit/serve/tools.py @@ -663,7 +663,10 @@ def describe_filing( "extension_namespace": filing.extension_namespace, "taxonomies": len(filing.taxonomy_namespaces), **( - {"items": describe_items(filing.items), "items_note": items_note(filing.items)} + { + "items": describe_items(filing.items), + "items_note": items_note(filing.items, bool(lf.exhibits_in_text)), + } if filing.items else {} ), @@ -774,7 +777,13 @@ def _next_steps(lf: LoadedFiling) -> list[str]: "calculation for what sums to a total, where the taxonomy has calculation arcs", ] exhibit_first: list[str] = [] - if is_earnings_release(lf.model.filing.items): + if lf.exhibits_in_text: + exhibit_first = [ + f"read_text from the {lf.exhibits_in_text[0]} `offset` in `sections.items` " + "— the exhibits are part of this text", + "search_text for a figure or a phrase across the 8-K and its exhibits", + ] + elif is_earnings_release(lf.model.filing.items): exhibit_first = [ "documents, then read_document on the EX-99.1 — Item 2.02 means the " "results are in the attached release, not in this filing's XBRL", From 82dc5eb818ac44826980d64c2411aa2a2269b383 Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Sun, 4 Oct 2026 17:26:43 -0500 Subject: [PATCH 4/4] fix(serve): send an 8-K's reader to its exhibits only where they are read Under pure, or reading the tagged text alone, the exhibits are not in the text the tools search and sections.items is not shown, yet the note and next steps sent the caller there; they now say documents in those modes. One exhibit out of reach no longer drops the rest, and two exhibits typed plain EX-99 each keep a section id of their own. --- tests/test_serve.py | 57 ++++++++++++++++++++++++++++++++++++++++ xbrlkit/serve/server.py | 15 ++++++----- xbrlkit/serve/session.py | 22 ++++++++++++---- xbrlkit/serve/tools.py | 14 +++++++--- 4 files changed, 93 insertions(+), 15 deletions(-) diff --git a/tests/test_serve.py b/tests/test_serve.py index 4f235c3..7ec7d40 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -2586,6 +2586,63 @@ def test_an_8k_whose_exhibits_cannot_be_reached_loads_alone( session.close() +def test_without_the_whole_text_an_8k_still_points_at_documents( + tmp_path: Path, monkeypatch: Any +) -> None: + """Under pure, or reading the tagged text alone, the exhibits are not in the + text the tools search, so the note cannot send the caller there.""" + path = tmp_path / "acme-8k.htm" + path.write_text(_EIGHT_K) + session = FilingSession() + try: + lf = session.load(str(path)) + lf.model.filing.items = ["2.02", "9.01"] + stub = _EightKExhibits() + monkeypatch.setattr(session, "other_documents", stub.other_documents) + monkeypatch.setattr(session, "read_other_document", stub.read_other_document) + lf = session._with_exhibits(lf) + for pure, whole in ((True, False), (True, True), (False, False)): + out = tools.describe_filing(lf, pure=pure, whole=whole) + assert "documents" in out["filing"]["items_note"], (pure, whole) + assert "part of this text" not in out["filing"]["items_note"], (pure, whole) + assert not out["next"][0].startswith("read_text from the EX-99.1") + finally: + session.close() + + +def test_an_exhibit_out_of_reach_leaves_the_others( + tmp_path: Path, monkeypatch: Any +) -> None: + from xbrlkit.edgar.filing_index import FilingDocument + from xbrlkit.serve.session import ReadDocument + + path = tmp_path / "acme-8k.htm" + path.write_text(_EIGHT_K) + session = FilingSession() + try: + lf = session.load(str(path)) + documents = [ + FilingDocument(seq=2, type="EX-99", document="slides.htm"), + FilingDocument(seq=3, type="EX-99", document="script.htm"), + FilingDocument(seq=4, type="EX-99", document="gone.htm"), + ] + + def read(lf: object, name: str) -> ReadDocument: + if name == "gone.htm": + raise SourceError("404") + return ReadDocument(f"the {name.split('.')[0]}", []) + + monkeypatch.setattr(session, "other_documents", lambda lf: documents) + monkeypatch.setattr(session, "read_other_document", read) + lf = session._with_exhibits(lf) + ids = [s.id for s in lf.sections if s.kind == "item" and s.id.startswith("ex_")] + # Two exhibits typed plain EX-99 each keep a section of their own. + assert ids == ["ex_99", "ex_99_2"] + assert "the slides" in lf.text and "the script" in lf.text + finally: + session.close() + + def test_only_an_8k_reaches_for_its_exhibits(tmp_path: Path, monkeypatch: Any) -> None: path = tmp_path / "form10-k.htm" path.write_text("

FORM 10-K

Annual report.

") diff --git a/xbrlkit/serve/server.py b/xbrlkit/serve/server.py index 4441b52..8aa0767 100644 --- a/xbrlkit/serve/server.py +++ b/xbrlkit/serve/server.py @@ -120,10 +120,12 @@ non-GAAP measures, adjusted EBITDA, segment detail, guidance — which no XBRL \ anywhere holds, and it lands weeks before the 10-Q that restates part of it \ in GAAP. Treat it as a primary source, not as a supplement to the 10-Q. -- An 8-K loaded from EDGAR is read with its EX-99 exhibits: each is a section \ -of the same text, under `sections.items` with its offset, so search_text \ -finds the release and read_text pages it. An exhibit loaded on its own, by its \ -URL, reads as a document with no filing around it. +- An 8-K loaded from EDGAR is read with its EX-99 exhibits: while the text \ +tools read the whole document, each is a section of the same text, under \ +`sections.items` with its offset, so search_text finds the release and \ +read_text pages it. describe_filing's items_note says which way in applies; \ +reading the tagged text alone, it is documents and read_document. An exhibit \ +loaded on its own, by its URL, reads as a document with no filing around it. RECORDS (XML filings) - records returns the form's own tables — a Form 4's transactions and \ @@ -697,8 +699,9 @@ def information_block( "What else was filed with this filing — the exhibits, and any second " "document the content actually lives in (an 8-K's EX-99.1 press " "release, a 13F's INFORMATION TABLE of holdings). An 8-K's EX-99 " - "exhibits are already in its text, for search_text and read_text; this " - "lists them with the rest. Costs one small fetch the first time and " + "exhibits are also in its text while the text tools read whole " + "documents; this lists them with the rest. Costs one small fetch the " + "first time and " "nothing after. " "Every document carries its URL and says whether read_document can read " "it: a PDF or an image is listed with its address so a caller that can " diff --git a/xbrlkit/serve/session.py b/xbrlkit/serve/session.py index b1858c4..24fa244 100644 --- a/xbrlkit/serve/session.py +++ b/xbrlkit/serve/session.py @@ -766,16 +766,28 @@ def _with_exhibits(self, lf: LoadedFiling) -> LoadedFiling: for d in self.other_documents(lf) if d.type.upper().startswith(_CURRENT_REPORT_EXHIBIT) and d.is_readable ] - read = [(d, self.read_other_document(lf, d.document)) for d in exhibits] except (SourceError, requests.RequestException, OSError) as exc: logger.warning("%s loads without its exhibits: %s", lf.accession, exc) return lf - if not read: + read: list[tuple[FilingDocument, ReadDocument]] = [] + for d in exhibits: + try: + read.append((d, self.read_other_document(lf, d.document))) + except (SourceError, requests.RequestException, OSError) as exc: + # One exhibit out of reach leaves the others in the text. + logger.warning("%s loads without %s: %s", lf.accession, d.document, exc) + if not any(r.text for _, r in read): return lf parts = [(FORM_SECTION_ID, f"Form {form}", lf.text, lf.sections)] - parts += [ - (_exhibit_section_id(d.type), d.type.upper(), r.text, r.sections) for d, r in read - ] + taken = {FORM_SECTION_ID} + for d, r in read: + # Two exhibits typed plain "EX-99" would otherwise share one id. + section_id = base = _exhibit_section_id(d.type) + n = 2 + while section_id in taken: + section_id, n = f"{base}_{n}", n + 1 + taken.add(section_id) + parts.append((section_id, d.type.upper(), r.text, r.sections)) lf.text, lf.sections = join_texts(parts) lf.exhibits_in_text = [d.type.upper() for d, r in read if r.text] return lf diff --git a/xbrlkit/serve/tools.py b/xbrlkit/serve/tools.py index 042e142..0624e57 100644 --- a/xbrlkit/serve/tools.py +++ b/xbrlkit/serve/tools.py @@ -602,6 +602,12 @@ def describe_filing( ) items = [] if pure else [s for s in sections if s.kind == "item"] + # The exhibits are reachable through the text tools only when they read the + # whole text and `sections.items` names where each one starts; otherwise + # `documents` is still the way in. + exhibits_readable = bool( + lf.exhibits_in_text and whole and lf.has_document and not pure + ) blocks = sorted( (s for s in sections if s.kind == "text_block"), key=lambda s: -s.chars )[:DESCRIBE_TEXT_BLOCKS] @@ -665,7 +671,7 @@ def describe_filing( **( { "items": describe_items(filing.items), - "items_note": items_note(filing.items, bool(lf.exhibits_in_text)), + "items_note": items_note(filing.items, exhibits_readable), } if filing.items else {} @@ -722,7 +728,7 @@ def describe_filing( ], "note": "offsets index into the plain text that search_text and read_text read", }, - "next": _next_steps(lf), + "next": _next_steps(lf, exhibits_readable), } @@ -759,7 +765,7 @@ def _describe_taxonomy(taxonomy: TaxonomyEntry) -> dict[str, Any]: } -def _next_steps(lf: LoadedFiling) -> list[str]: +def _next_steps(lf: LoadedFiling, exhibits_readable: bool = False) -> list[str]: """What to call next, given what this filing actually is. An 8-K leads with its exhibits whatever else is true of it: its tagged @@ -777,7 +783,7 @@ def _next_steps(lf: LoadedFiling) -> list[str]: "calculation for what sums to a total, where the taxonomy has calculation arcs", ] exhibit_first: list[str] = [] - if lf.exhibits_in_text: + if exhibits_readable: exhibit_first = [ f"read_text from the {lf.exhibits_in_text[0]} `offset` in `sections.items` " "— the exhibits are part of this text",