diff --git a/tests/test_serve.py b/tests/test_serve.py index 77eee05..7ec7d40 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -17,6 +17,7 @@ import textwrap from datetime import date from pathlib import Path +from typing import Any import pytest @@ -35,7 +36,12 @@ ) from xbrlkit.serve import FilingSession, LoadedFiling, SourceError, build_text from xbrlkit.serve import tools -from xbrlkit.serve.session import _find_load_target, _locate, _restore_sec_identity +from xbrlkit.serve.session import ( + _find_load_target, + _form_on_the_cover, + _locate, + _restore_sec_identity, +) US_GAAP = "http://fasb.org/us-gaap/2024-01-31" IS_ROLE = "http://acme.example/role/StatementOfIncome" @@ -742,6 +748,21 @@ def test_locate_matches_across_renderings() -> None: assert _locate(text, "NOTE 19. Commitments") is None # too short to trust +def test_form_on_the_cover_reads_only_the_cover() -> None: + cover = ( + "

UNITED STATES SECURITIES AND EXCHANGE COMMISSION

" + "

Washington, D.C. 20549

FORM 8-K

CURRENT REPORT

" + ) + assert _form_on_the_cover(cover) == "8-K" + # A press release cites the annual report in its safe harbor; it is not one. + release = ( + "

Workiva Announces Second Quarter 2026 Financial Results

" + + "

Revenue grew.

" * 300 + + "

See our annual report on Form 10-K for risk factors.

" + ) + assert _form_on_the_cover(release) is None + + def test_locate_skips_a_contents_row_that_carries_the_whole_heading() -> None: heading = ( "Item 7. Management's Discussion and Analysis of Financial Condition and " @@ -2401,6 +2422,9 @@ def test_an_earnings_8k_is_identified_and_points_at_the_exhibit() -> None: assert items_note(["9.01"]) is not None assert items_note(["5.07"]) is not None assert items_note([]) is None + # With the exhibits read into the text, the text tools are the way in. + joined = items_note(["2.02", "9.01"], exhibits_in_text=True) + assert joined is not None and "search_text" in joined and "documents" not in joined def test_filing_meta_carries_the_items_off_an_edgar_ref() -> None: @@ -2457,6 +2481,184 @@ def test_an_8k_describes_its_items_and_leads_with_the_exhibit() -> None: assert "documents" in out["next"][0] and "Item 2.02" in out["next"][0] +def test_join_texts_sets_each_document_under_its_heading() -> None: + from xbrlkit.serve import TextSection, join_texts + + inner = TextSection(id="item_2_02", label="Item 2.02", kind="item", chars=5, offset=3) + text, sections = join_texts( + [ + ("form_8k", "Form 8-K", "the cover", [inner]), + ("ex_10_1", "EX-10.1", "", []), + ("ex_99_1", "EX-99.1", "the release", []), + ] + ) + assert text == "## Form 8-K\nthe cover\n\n## EX-99.1\nthe release\n\n" + by_id = {s.id: s for s in sections} + assert set(by_id) == { + "form_8k", + "item_2_02", + "ex_99_1", + } # an empty body is no section + release = by_id["ex_99_1"] + assert text[release.offset : release.offset + release.chars] == "the release" + assert text[release.offset - release.heading_chars :].startswith("## EX-99.1\n") + # A document's own sections move with it. + assert by_id["item_2_02"].offset == by_id["form_8k"].offset + 3 + + +_EIGHT_K = ( + "

UNITED STATES SECURITIES AND EXCHANGE COMMISSION

" + "

FORM 8-K

Item 2.02 Results of Operations and Financial Condition. " + "The press release is furnished as Exhibit 99.1.

" +) + + +class _EightKExhibits: + """An 8-K's index and exhibits, answered from memory.""" + + def __init__(self, fail: bool = False) -> None: + self.fail = fail + + def other_documents(self, lf: object) -> list[Any]: + from xbrlkit.edgar.filing_index import FilingDocument + + if self.fail: + raise SourceError("the index could not be fetched") + return [ + FilingDocument(seq=2, type="EX-10.1", document="credit.htm"), + FilingDocument(seq=3, type="EX-99.1", document="release.htm"), + FilingDocument(seq=4, type="GRAPHIC", document="logo.jpg"), + ] + + def read_other_document(self, lf: object, name: str) -> Any: + from xbrlkit.serve.session import ReadDocument + + assert name == "release.htm", "only the EX-99 is read into the text" + return ReadDocument("Free cash flow was $78 million for the quarter.", []) + + +def test_an_8k_is_read_with_its_release(tmp_path: Path, monkeypatch: Any) -> None: + """Read alone, an earnings 8-K is a cover page: a search for anything the + release says found nothing and told the caller to reword.""" + path = tmp_path / "acme-8k.htm" + path.write_text(_EIGHT_K) + session = FilingSession() + try: + lf = session.load(str(path)) + lf.model.filing.items = ["2.02", "9.01"] + assert lf.model.filing.form == "8-K" + stub = _EightKExhibits() + monkeypatch.setattr(session, "other_documents", stub.other_documents) + monkeypatch.setattr(session, "read_other_document", stub.read_other_document) + lf = session._with_exhibits(lf) + + assert lf.exhibits_in_text == ["EX-99.1"] + hits = tools.search_text(lf, "free cash flow") + assert hits["total"] == 1 and hits["hits"][0]["section"] == "EX-99.1" + assert "FORM 8-K" in tools.search_text(lf, "FORM 8-K")["hits"][0]["text"] + + out = tools.describe_filing(lf) + items = {s["id"]: s for s in out["sections"]["items"]} + assert {"form_8k", "ex_99_1"} <= set(items) + page = tools.read_text(lf, offset=items["ex_99_1"]["offset"]) + assert page["text"].startswith("Free cash flow") + assert "part of this text" in out["filing"]["items_note"] + assert out["next"][0].startswith("read_text from the EX-99.1") + finally: + session.close() + + +def test_an_8k_whose_exhibits_cannot_be_reached_loads_alone( + tmp_path: Path, monkeypatch: Any +) -> None: + path = tmp_path / "acme-8k.htm" + path.write_text(_EIGHT_K) + session = FilingSession() + try: + lf = session.load(str(path)) + before = lf.text + monkeypatch.setattr( + session, "other_documents", _EightKExhibits(fail=True).other_documents + ) + lf = session._with_exhibits(lf) + assert lf.text == before and lf.exhibits_in_text == [] + finally: + session.close() + + +def test_without_the_whole_text_an_8k_still_points_at_documents( + tmp_path: Path, monkeypatch: Any +) -> None: + """Under pure, or reading the tagged text alone, the exhibits are not in the + text the tools search, so the note cannot send the caller there.""" + path = tmp_path / "acme-8k.htm" + path.write_text(_EIGHT_K) + session = FilingSession() + try: + lf = session.load(str(path)) + lf.model.filing.items = ["2.02", "9.01"] + stub = _EightKExhibits() + monkeypatch.setattr(session, "other_documents", stub.other_documents) + monkeypatch.setattr(session, "read_other_document", stub.read_other_document) + lf = session._with_exhibits(lf) + for pure, whole in ((True, False), (True, True), (False, False)): + out = tools.describe_filing(lf, pure=pure, whole=whole) + assert "documents" in out["filing"]["items_note"], (pure, whole) + assert "part of this text" not in out["filing"]["items_note"], (pure, whole) + assert not out["next"][0].startswith("read_text from the EX-99.1") + finally: + session.close() + + +def test_an_exhibit_out_of_reach_leaves_the_others( + tmp_path: Path, monkeypatch: Any +) -> None: + from xbrlkit.edgar.filing_index import FilingDocument + from xbrlkit.serve.session import ReadDocument + + path = tmp_path / "acme-8k.htm" + path.write_text(_EIGHT_K) + session = FilingSession() + try: + lf = session.load(str(path)) + documents = [ + FilingDocument(seq=2, type="EX-99", document="slides.htm"), + FilingDocument(seq=3, type="EX-99", document="script.htm"), + FilingDocument(seq=4, type="EX-99", document="gone.htm"), + ] + + def read(lf: object, name: str) -> ReadDocument: + if name == "gone.htm": + raise SourceError("404") + return ReadDocument(f"the {name.split('.')[0]}", []) + + monkeypatch.setattr(session, "other_documents", lambda lf: documents) + monkeypatch.setattr(session, "read_other_document", read) + lf = session._with_exhibits(lf) + ids = [s.id for s in lf.sections if s.kind == "item" and s.id.startswith("ex_")] + # Two exhibits typed plain EX-99 each keep a section of their own. + assert ids == ["ex_99", "ex_99_2"] + assert "the slides" in lf.text and "the script" in lf.text + finally: + session.close() + + +def test_only_an_8k_reaches_for_its_exhibits(tmp_path: Path, monkeypatch: Any) -> None: + path = tmp_path / "form10-k.htm" + path.write_text("

FORM 10-K

Annual report.

") + session = FilingSession() + try: + lf = session.load(str(path)) + + def refuse(lf: object) -> list[Any]: + raise AssertionError("a 10-K's exhibits are not read into its text") + + monkeypatch.setattr(session, "other_documents", refuse) + assert session._with_exhibits(lf).exhibits_in_text == [] + finally: + session.close() + + def test_a_filing_with_no_items_says_nothing_about_them(loaded: LoadedFiling) -> None: out = tools.describe_filing(loaded) assert "items" not in out["filing"] and "items_note" not in out["filing"] diff --git a/tests/test_serve_surface.py b/tests/test_serve_surface.py index 14d88e2..853ff3d 100644 --- a/tests/test_serve_surface.py +++ b/tests/test_serve_surface.py @@ -23,6 +23,7 @@ # has to be. The six marked (RoboSystems) are the ones in use today — # ``robosystems/operations/roboledger/views/information_blocks.py``. DECLARED: dict[str, str] = { + "FORM_SECTION_ID": "str", "FilingSession": "class", "LoadedFiling": "class", # RoboSystems "SourceError": "class", @@ -34,6 +35,7 @@ "build_text": "callable", "disclosures": "callable", # RoboSystems "information_block": "callable", # RoboSystems + "join_texts": "callable", "serve": "callable", } @@ -53,6 +55,8 @@ def test_every_declared_name_resolves() -> None: assert isinstance(obj, type), f"{name} should be a class" elif kind == "int": assert isinstance(obj, int), f"{name} should be an int" + elif kind == "str": + assert isinstance(obj, str), f"{name} should be a str" else: assert callable(obj), f"{name} should be callable" diff --git a/tests/test_text_tables.py b/tests/test_text_tables.py index addbe75..b44ea0f 100644 --- a/tests/test_text_tables.py +++ b/tests/test_text_tables.py @@ -77,6 +77,12 @@ def test_th_elements(self): def test_empty_table(self): assert _parse_rows("
") == [] + def test_self_closing_cell_is_empty(self): + """Apple's filer writes spacers as ````; read as an open + tag one swallowed the next cell, and the row came up a column short.""" + html = 'Europe111,032' + assert _parse_rows(html) == [[("Europe", 1), ("", 3), ("111,032", 1), ("", 1)]] + @pytest.mark.unit class TestMergeCurrencyCells: @@ -108,6 +114,70 @@ def test_no_merge_needed(self): result = _merge_currency_cells(rows) assert result == [["A", "B", "C"]] + def test_dollar_rows_stay_in_their_columns(self): + """A "$" merge used to drop the symbol's cell, so every value after it + moved one column left of the rows without one.""" + rows = [ + [ + ("Income", 1), + ("$", 1), + ("11,670", 1), + ("", 1), + ("$", 1), + ("(22,146", 1), + (")", 1), + ], + [ + ("Add back", 1), + ("", 1), + ("29,751", 1), + ("", 1), + ("", 1), + ("28,467", 1), + ("", 1), + ], + ] + result = _merge_currency_cells(rows) + assert result == [ + ["Income", "$11,670", "$(22,146)"], + ["Add back", "29,751", "28,467"], + ] + + def test_values_spanning_the_symbol_column_align(self): + """Workiva's layout: a value with no "$" spans the symbol and number + columns (colspan 2), as the period header spans all three, so both start + in the symbol's column while a "$" value sits one to the right.""" + rows = [ + [("", 1), ("2026", 3), ("", 1), ("2025", 3)], + [ + ("Income", 1), + ("$", 1), + ("11,670", 1), + ("", 1), + ("", 1), + ("$", 1), + ("8,233", 1), + ("", 1), + ], + [("Add back", 1), ("29,751", 2), ("", 1), ("", 1), ("28,467", 2), ("", 1)], + ] + result = _merge_currency_cells(rows) + assert result == [ + ["", "2026", "2025"], + ["Income", "$11,670", "$8,233"], + ["Add back", "29,751", "28,467"], + ] + + def test_symbol_column_holding_values_is_kept(self): + """A column that holds a value in the same row as its neighbour is a + column in its own right, not a symbol's, and is not joined.""" + rows = [ + [("A", 1), ("$", 1), ("1", 1)], + [("B", 1), ("2", 1), ("3", 1)], + ] + result = _merge_currency_cells(rows) + assert result == [["A", "", "$1"], ["B", "2", "3"]] + @pytest.mark.unit class TestIsLayoutTable: diff --git a/xbrlkit/edgar/items.py b/xbrlkit/edgar/items.py index 528659d..9e7fa50 100644 --- a/xbrlkit/edgar/items.py +++ b/xbrlkit/edgar/items.py @@ -108,36 +108,44 @@ def is_earnings_release(codes: list[str] | None) -> bool: return EARNINGS_ITEM in (codes or []) -def items_note(codes: list[str] | None) -> str | None: +def items_note(codes: list[str] | None, exhibits_in_text: bool = False) -> str | None: """What a reader should do next, given what the filing is coded as. An 8-K's tagged content is its cover page; the substance is the exhibit. The note says so, and says it most strongly where the exhibit is the earnings release — the numbers a company leads with, which no XBRL anywhere carries. + ``exhibits_in_text`` when the exhibits were read into the filing's text, so + the text tools reach them and ``documents`` is not the way in. """ codes = codes or [] if not codes: return None + if exhibits_in_text: + reach = ( + "The exhibits are part of this text, each under `sections.items` with " + "its `offset`: read_text from one, or search_text across the 8-K and " + "all of them." + ) + else: + reach = "Call documents, then read_document on the exhibit." if is_earnings_release(codes): return ( "Item 2.02 — this is an earnings release. The results are in the " "attached exhibit (usually EX-99.1), not in this filing's XBRL, which " - "is the cover page only. Call documents, then read_document on the " - "EX-99.1. The exhibit carries what the company leads with — non-GAAP " - "measures, adjusted EBITDA, segment detail and guidance — which no XBRL " - "holds, and it arrives weeks before the 10-Q that restates part of it." + f"is the cover page only. {reach} The exhibit carries what the company " + "leads with — non-GAAP measures, adjusted EBITDA, segment detail and " + "guidance — which no XBRL holds, and it arrives weeks before the 10-Q " + "that restates part of it." ) if FD_ITEM in codes: return ( "Item 7.01 — Regulation FD disclosure. The substance is normally an " - "exhibit (a presentation, a script, a release): call documents, then " - "read_document." + f"exhibit (a presentation, a script, a release). {reach}" ) if EXHIBITS_ITEM in codes: return ( "This 8-K has exhibits (Item 9.01) and an 8-K's tagged content is its " - "cover page, so the substance is in them: call documents, then " - "read_document." + f"cover page, so the substance is in them. {reach}" ) return ( "An 8-K's tagged content is its cover page; anything more is in the " diff --git a/xbrlkit/serve/README.md b/xbrlkit/serve/README.md index 27de4ee..6f2d753 100644 --- a/xbrlkit/serve/README.md +++ b/xbrlkit/serve/README.md @@ -137,7 +137,16 @@ So an earnings 8-K is identifiable **before** anything is fetched: attached exhibit (usually EX-99.1), not in this filing's XBRL …" ``` -and `next` leads with `documents` rather than the fact tools. +and `next` leads with the release rather than the fact tools. + +**An 8-K loaded from EDGAR is read with its `EX-99` exhibits.** The filing's +own text comes first under `## Form 8-K`, then each exhibit under its type +(`## EX-99.1`), every one a section in `sections.items` with its offset — so +`search_text` finds what the release says and `read_text` pages it, with no +second document to know about. It costs the index fetch `documents` makes and +one fetch per exhibit, both kept for `read_document`; when they cannot be +reached the 8-K loads alone. Other exhibits — a credit agreement, a +certification — stay in `documents`. **Why this matters more than a routing convenience.** The exhibit carries what the company leads with — adjusted EBITDA, non-GAAP margin, segment detail, diff --git a/xbrlkit/serve/__init__.py b/xbrlkit/serve/__init__.py index ed1242d..e5dd454 100644 --- a/xbrlkit/serve/__init__.py +++ b/xbrlkit/serve/__init__.py @@ -34,7 +34,15 @@ from pathlib import Path from typing import Any -from .session import FilingSession, LoadedFiling, SourceError, TextSection, build_text +from .session import ( + FORM_SECTION_ID, + FilingSession, + LoadedFiling, + SourceError, + TextSection, + build_text, + join_texts, +) from .tools import ( MAX_BLOCK_MEMBERS_CAP, MAX_BLOCK_ROWS, @@ -44,6 +52,7 @@ ) __all__ = ( + "FORM_SECTION_ID", "MAX_BLOCK_MEMBERS_CAP", "MAX_BLOCK_ROWS", "FilingSession", @@ -55,6 +64,7 @@ "build_text", "disclosures", "information_block", + "join_texts", "serve", ) diff --git a/xbrlkit/serve/server.py b/xbrlkit/serve/server.py index 581cd72..8aa0767 100644 --- a/xbrlkit/serve/server.py +++ b/xbrlkit/serve/server.py @@ -120,6 +120,12 @@ non-GAAP measures, adjusted EBITDA, segment detail, guidance — which no XBRL \ anywhere holds, and it lands weeks before the 10-Q that restates part of it \ in GAAP. Treat it as a primary source, not as a supplement to the 10-Q. +- An 8-K loaded from EDGAR is read with its EX-99 exhibits: while the text \ +tools read the whole document, each is a section of the same text, under \ +`sections.items` with its offset, so search_text finds the release and \ +read_text pages it. describe_filing's items_note says which way in applies; \ +reading the tagged text alone, it is documents and read_document. An exhibit \ +loaded on its own, by its URL, reads as a document with no filing around it. RECORDS (XML filings) - records returns the form's own tables — a Form 4's transactions and \ @@ -692,11 +698,11 @@ def information_block( description=( "What else was filed with this filing — the exhibits, and any second " "document the content actually lives in (an 8-K's EX-99.1 press " - "release, a 13F's INFORMATION TABLE of holdings). On an 8-K this is the " - "tool that answers the question: its tagged content is the cover page, " - "so the substance is always here — and on an Item 2.02 the EX-99.1 is " - "the earnings release, carrying the non-GAAP measures and guidance no " - "XBRL holds. Costs one small fetch the first time and nothing after. " + "release, a 13F's INFORMATION TABLE of holdings). An 8-K's EX-99 " + "exhibits are also in its text while the text tools read whole " + "documents; this lists them with the rest. Costs one small fetch the " + "first time and " + "nothing after. " "Every document carries its URL and says whether read_document can read " "it: a PDF or an image is listed with its address so a caller that can " "open one may fetch it directly. The XBRL package and the SEC's own " diff --git a/xbrlkit/serve/session.py b/xbrlkit/serve/session.py index dc803c3..24fa244 100644 --- a/xbrlkit/serve/session.py +++ b/xbrlkit/serve/session.py @@ -19,7 +19,7 @@ import threading from concurrent.futures import ThreadPoolExecutor from collections import Counter -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import date from pathlib import Path from collections.abc import Mapping @@ -52,6 +52,13 @@ SectionKind = Literal["item", "text_block", "records"] +# An 8-K's own text is its cover and its items; what it discloses — the +# release, the slides, the script — is an EX-99 filed beside it, so the two +# are read as one text. +_CURRENT_REPORT_FORMS = frozenset({"8-K", "8-K/A"}) +_CURRENT_REPORT_EXHIBIT = "EX-99" +FORM_SECTION_ID = "form_8k" + _INLINE_SUFFIXES = {".htm", ".html", ".xhtml"} _PLAIN_SUFFIXES = {".txt", ".md"} # A report serialized as JSON — read into the model here, never by Arelle. @@ -162,6 +169,9 @@ class LoadedFiling: # and each one read on first read. Nothing is fetched until something asks. other_documents: list[FilingDocument] | None = None read_documents: dict[str, "ReadDocument"] = field(default_factory=dict) + # The exhibits read into ``text`` after the filing's own document, by type + # (``EX-99.1``): an 8-K's release, searched and paged with the form. + exhibits_in_text: list[str] = field(default_factory=list) # The document the filing was loaded from, when it already is one of the # serializations this server hands out (a holon, a TAVI), and which one. # ``view_filing`` serves that document as it is rather than a re-projection @@ -737,6 +747,51 @@ def read_other_document(self, lf: LoadedFiling, name: str) -> ReadDocument: lf.read_documents[match.document] = read return read + def _with_exhibits(self, lf: LoadedFiling) -> LoadedFiling: + """An 8-K read with its EX-99 exhibits, one text the text tools search. + + Read alone, an 8-K is a cover page and a line saying a release was + furnished: a search for anything the release says finds nothing, and the + caller is left to know the exhibit is a second document. Costs the index + fetch ``documents`` makes and one fetch per exhibit, both kept for + ``read_document``. A filing whose exhibits cannot be reached loads as its + 8-K alone, as it did before. + """ + form = (lf.model.filing.form or "").upper() + if form not in _CURRENT_REPORT_FORMS or not lf.has_document: + return lf + try: + exhibits = [ + d + for d in self.other_documents(lf) + if d.type.upper().startswith(_CURRENT_REPORT_EXHIBIT) and d.is_readable + ] + except (SourceError, requests.RequestException, OSError) as exc: + logger.warning("%s loads without its exhibits: %s", lf.accession, exc) + return lf + read: list[tuple[FilingDocument, ReadDocument]] = [] + for d in exhibits: + try: + read.append((d, self.read_other_document(lf, d.document))) + except (SourceError, requests.RequestException, OSError) as exc: + # One exhibit out of reach leaves the others in the text. + logger.warning("%s loads without %s: %s", lf.accession, d.document, exc) + if not any(r.text for _, r in read): + return lf + parts = [(FORM_SECTION_ID, f"Form {form}", lf.text, lf.sections)] + taken = {FORM_SECTION_ID} + for d, r in read: + # Two exhibits typed plain "EX-99" would otherwise share one id. + section_id = base = _exhibit_section_id(d.type) + n = 2 + while section_id in taken: + section_id, n = f"{base}_{n}", n + 1 + taken.add(section_id) + parts.append((section_id, d.type.upper(), r.text, r.sections)) + lf.text, lf.sections = join_texts(parts) + lf.exhibits_in_text = [d.type.upper() for d, r in read if r.text] + return lf + def _edgar_coordinates(self, lf: LoadedFiling) -> tuple[str, str]: """The CIK and accession needed to reach back to EDGAR for this filing.""" cik, accession = lf.model.filing.cik, lf.accession @@ -819,7 +874,7 @@ def _load_edgar(self, cik: str, accession: str, source: str) -> LoadedFiling: model = self._parse(target, accession, filing=filing, entity=None) loaded = self._finish(accession, source, model, target, package_dir, document) _enrich_filer(loaded, self._filer_metadata(cik, client=client)) - return loaded + return self._with_exhibits(loaded) def _load_document_only( self, client: Any, cik: str, accession: str, ref: Any, source: str @@ -857,7 +912,7 @@ def _load_document_only( # No XBRL, so no cover page to read: the submissions header is the only # account of the filer this filing has. _enrich_filer(loaded, self._filer_metadata(cik, client=client)) - return loaded + return self._with_exhibits(loaded) def _load_from_submission( self, client: Any, cik: str, accession: str, ref: Any, source: str @@ -1354,6 +1409,48 @@ def _read_document(doc: Path | None, model: XbrlModel) -> ReadDocument | None: return None +def join_texts( + parts: list[tuple[str, str, str, list[TextSection]]], +) -> tuple[str, list[TextSection]]: + """Several documents read as one text, in the order given. + + ``parts`` is ``(id, label, text, sections)`` for each document. Each is set + under a ``## label`` heading and becomes a section whose offset is its + body's start, with the heading counted in ``heading_chars``; its own + sections move with it. + """ + chunks: list[str] = [] + sections: list[TextSection] = [] + offset = 0 + for section_id, label, body, inner in parts: + if not body: + continue + header = f"## {label}\n" + start = offset + len(header) + sections.append( + TextSection( + id=section_id, + label=label, + kind="item", + chars=len(body), + offset=start, + heading_chars=len(header), + ) + ) + sections.extend( + replace(s, offset=None if s.offset is None else s.offset + start) for s in inner + ) + chunk = header + body + "\n\n" + chunks.append(chunk) + offset += len(chunk) + return "".join(chunks), sections + + +def _exhibit_section_id(exhibit_type: str) -> str: + """``ex_99_1`` for ``EX-99.1``.""" + return re.sub(r"[^a-z0-9]+", "_", exhibit_type.lower()).strip("_") + + def build_text(model: XbrlModel, html: str | None) -> tuple[str, list[TextSection]]: """The filing as plain text plus its section map. @@ -1962,11 +2059,16 @@ def first(names: tuple[str, ...]) -> str | None: _COVER_FORM_RE = re.compile( r"\bFORM\s+(10-K|10-Q|20-F|40-F|8-K|S-1|S-3|DEF\s*14A)\b", re.IGNORECASE ) +# A cover names its form within a few hundred characters. Past that, "Form +# 10-K" is a citation — a press release's safe harbor pointing at the annual +# report — and the document is not one. +_COVER_CHARS = 2_000 def _form_on_the_cover(html: str) -> str | None: """The form a document names on its cover, from the top of the document.""" - m = _COVER_FORM_RE.search(_html_to_text(html[:400_000])) + text = _html_to_text(html[:400_000])[:_COVER_CHARS] + m = _COVER_FORM_RE.search(text) return re.sub(r"\s+", " ", m.group(1)).upper() if m else None diff --git a/xbrlkit/serve/tools.py b/xbrlkit/serve/tools.py index eba6713..0624e57 100644 --- a/xbrlkit/serve/tools.py +++ b/xbrlkit/serve/tools.py @@ -602,6 +602,12 @@ def describe_filing( ) items = [] if pure else [s for s in sections if s.kind == "item"] + # The exhibits are reachable through the text tools only when they read the + # whole text and `sections.items` names where each one starts; otherwise + # `documents` is still the way in. + exhibits_readable = bool( + lf.exhibits_in_text and whole and lf.has_document and not pure + ) blocks = sorted( (s for s in sections if s.kind == "text_block"), key=lambda s: -s.chars )[:DESCRIBE_TEXT_BLOCKS] @@ -663,7 +669,10 @@ def describe_filing( "extension_namespace": filing.extension_namespace, "taxonomies": len(filing.taxonomy_namespaces), **( - {"items": describe_items(filing.items), "items_note": items_note(filing.items)} + { + "items": describe_items(filing.items), + "items_note": items_note(filing.items, exhibits_readable), + } if filing.items else {} ), @@ -719,7 +728,7 @@ def describe_filing( ], "note": "offsets index into the plain text that search_text and read_text read", }, - "next": _next_steps(lf), + "next": _next_steps(lf, exhibits_readable), } @@ -756,7 +765,7 @@ def _describe_taxonomy(taxonomy: TaxonomyEntry) -> dict[str, Any]: } -def _next_steps(lf: LoadedFiling) -> list[str]: +def _next_steps(lf: LoadedFiling, exhibits_readable: bool = False) -> list[str]: """What to call next, given what this filing actually is. An 8-K leads with its exhibits whatever else is true of it: its tagged @@ -774,7 +783,13 @@ def _next_steps(lf: LoadedFiling) -> list[str]: "calculation for what sums to a total, where the taxonomy has calculation arcs", ] exhibit_first: list[str] = [] - if is_earnings_release(lf.model.filing.items): + if exhibits_readable: + exhibit_first = [ + f"read_text from the {lf.exhibits_in_text[0]} `offset` in `sections.items` " + "— the exhibits are part of this text", + "search_text for a figure or a phrase across the 8-K and its exhibits", + ] + elif is_earnings_release(lf.model.filing.items): exhibit_first = [ "documents, then read_document on the EX-99.1 — Item 2.02 means the " "results are in the attached release, not in this filing's XBRL", diff --git a/xbrlkit/text/tables.py b/xbrlkit/text/tables.py index 6f95bfa..25feee0 100644 --- a/xbrlkit/text/tables.py +++ b/xbrlkit/text/tables.py @@ -171,11 +171,13 @@ def _parse_rows(table_html: str) -> list[list[tuple[str, int]]]: tr_content = tr_match.group(1) cells: list[tuple[str, int]] = [] + # A self-closing ```` is an empty cell; read as an open + # tag it swallowed the cell after it up to that one's ````. for cell_match in re.finditer( - r"<(t[dh])([^>]*)>(.*?)", tr_content, re.DOTALL | re.IGNORECASE + r"<(t[dh])\b([^>]*?)(?:/>|>(.*?))", tr_content, re.DOTALL | re.IGNORECASE ): attrs = cell_match.group(2) - inner = cell_match.group(3) + inner = cell_match.group(3) or "" text = _cell_text(inner) colspan = _get_colspan(attrs) cells.append((text, colspan)) @@ -192,8 +194,15 @@ def _merge_currency_cells(rows: list[list[tuple[str, int]]]) -> list[list[str]]: SEC filings often put "$" in its own , separate from the number. This merges ["$", "11,866.1"] into ["$11,866.1"] and expands colspan. Also drops columns that are empty across all rows (SEC spacer cells). + + A merge keeps every cell's position, so a row with a "$" stays as wide as + one without. The column a "$" stood in is then joined to the number's + column wherever no row fills both: a value with no "$" often spans the two + (colspan 2) and lands in the symbol's column, as do the period headers + above, while a "$" row's value lands one to the right. """ merged: list[list[str]] = [] + symbol_cols: set[int] = set() for row in rows: # Expand colspan first @@ -214,10 +223,11 @@ def _merge_currency_cells(rows: list[list[tuple[str, int]]]) -> list[list[str]]: while j < len(expanded) and not expanded[j].strip(): j += 1 if j < len(expanded): - # Add empty cells between as-is, then the merged cell - for k in range(i + 1, j): + # The symbol's cell and any between stay, empty; then the merged cell + for _ in range(i, j): final.append("") final.append(cell + expanded[j]) + symbol_cols.update(range(i, j)) i = j + 1 else: final.append(cell) @@ -227,6 +237,7 @@ def _merge_currency_cells(rows: list[list[tuple[str, int]]]) -> list[list[str]]: for k in range(len(final) - 1, -1, -1): if final[k].strip(): final[k] = final[k] + cell + final.append("") break else: final.append(cell) @@ -245,6 +256,15 @@ def _merge_currency_cells(rows: list[list[tuple[str, int]]]) -> list[list[str]]: while len(row) < max_cols: row.append("") + for col in sorted(symbol_cols, reverse=True): + if col + 1 < max_cols and not any( + row[col].strip() and row[col + 1].strip() for row in merged + ): + for row in merged: + row[col] = row[col] or row[col + 1] + del row[col + 1] + max_cols -= 1 + keep_cols = [ col for col in range(max_cols)