From 73e0bd769571664f22dfcd5aa85665c1b2536987 Mon Sep 17 00:00:00 2001
From: DonislawDev
Date: Wed, 14 Oct 2026 21:44:58 +0200
Subject: [PATCH 1/2] html: links, an indented page, and the BOM label
links puts a list of that many links above the pictures and the blocks,
eighteen kinds in turn - relative, from the root, a query alone, absolute
with a query and a fragment, http, without a scheme, percent encoded, an
international domain, a port, mailto:, tel:, ftp:, a download, a new
window, nofollow and data: - each saying in its text what it is, every
address on a domain kept for examples. Up to a million: DOMPurify ran out
of an 8 GB heap over a million, which lxml read in 2.9 s.
formatting=indented puts two spaces in front of a line for every element it
sits in and leaves the text of pre alone. A guard indents the page of lines
with its own indenter and asks for the same bytes, and the structural check
counts every line's depth itself.
What closes the levels, the body and the document is written after the
last block, a chunk at a time, rather than inside it - a hundred thousand
indented levels are 10 GB. Same bytes for every layout: two golden cases of
nesting on one line were measured with the binary of main before the move.
The writing of a page moved to a type of its own, content, when settings
grew past the methods one type may carry.
The window spells the bom setting BOM, where it read Bom, and the mutation
coverage guard reads a guard's name in either quote.
Co-Authored-By: Claude Opus 5.5
---
CHANGELOG.md | 21 ++
README.md | 2 +-
internal/format/htmlfile/blocks.go | 25 +-
internal/format/htmlfile/catalogue.go | 28 +-
internal/format/htmlfile/characters.go | 29 +-
internal/format/htmlfile/content.go | 156 +++++++++++
internal/format/htmlfile/html.go | 44 +++-
internal/format/htmlfile/indent.go | 151 +++++++++++
internal/format/htmlfile/links.go | 89 +++++++
internal/format/htmlfile/markup.go | 28 +-
internal/format/htmlfile/page.go | 87 +++---
internal/format/htmlfile/pictures.go | 19 +-
internal/format/htmlfile/settings.go | 82 ++++--
internal/guard/flagnames_test.go | 6 +-
internal/guard/generatorbytes_test.go | 8 +
internal/guard/htmllayout_test.go | 247 ++++++++++++++++++
internal/guard/htmlsettings_test.go | 56 +++-
internal/guard/mutationcoverage_test.go | 6 +-
internal/guard/notelemetry_test.go | 23 +-
internal/guard/parity_test.go | 1 +
internal/guard/testdata/generator-golden.json | 8 +
internal/guard/testdata/screens/catalogue.png | Bin 2171637 -> 2171813 bytes
internal/guard/testdata/screens/catalogue.xml | 10 +-
internal/gui/text/labels.go | 4 +-
internal/gui/text/locale/registry/en.json | 18 +-
internal/gui/text/locale/registry/pl.json | 6 +-
internal/gui/text/locale/said/en.json | 17 +-
internal/gui/text/locale/said/pl.json | 5 +-
internal/oracle/strict.py | 101 ++++++-
web/public/ar/formats/index.html | 7 +-
web/public/cs/formaty/index.html | 7 +-
web/public/de/formate/index.html | 7 +-
web/public/es/formatos/index.html | 7 +-
web/public/formats/index.html | 7 +-
web/public/fr/formats/index.html | 7 +-
web/public/hi/formats/index.html | 7 +-
web/public/id/format/index.html | 7 +-
web/public/it/formati/index.html | 7 +-
web/public/ja/formats/index.html | 7 +-
web/public/ko/formats/index.html | 7 +-
web/public/nl/formaten/index.html | 7 +-
web/public/pl/formaty/index.html | 7 +-
web/public/pt-br/formatos/index.html | 7 +-
web/public/ro/formate/index.html | 7 +-
web/public/ru/formats/index.html | 7 +-
web/public/th/formats/index.html | 7 +-
web/public/tr/bicimler/index.html | 7 +-
web/public/uk/formats/index.html | 7 +-
web/public/vi/dinh-dang/index.html | 7 +-
web/public/zh-hans/formats/index.html | 7 +-
web/public/zh-hant/formats/index.html | 7 +-
51 files changed, 1257 insertions(+), 174 deletions(-)
create mode 100644 internal/format/htmlfile/content.go
create mode 100644 internal/format/htmlfile/indent.go
create mode 100644 internal/format/htmlfile/links.go
create mode 100644 internal/guard/htmllayout_test.go
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5e9ba811..46ec41e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -185,6 +185,24 @@ because it turns other people's test suites red.
BeautifulSoup on html5lib count the rules as text of the page. No rule
names anything to fetch. Files made with the earlier settings keep their
bytes.
+- **HTML: links, and an indented page.** `links` puts a list of that many
+ links above the pictures and the blocks. They go through 18 kinds in turn -
+ a link to the list itself, relative, to the folder above, from the root, a
+ query alone, absolute with a query and a fragment, `http`, without a
+ scheme, percent encoded, an international domain, a port, `mailto:`,
+ `tel:`, `ftp:`, a download, a new window, `nofollow` and `data:` - and the
+ text of each says which it is. Every address is on a domain kept for
+ examples. The Nu HTML Checker finds nothing wrong with any of them, and
+ jsdom, lxml, BeautifulSoup and Python's `urljoin` resolve them alike.
+ DOMPurify with its defaults takes out the `data:` address and the target
+ that opens a new window, and it ran out of 8 GB of memory over a million
+ links, which lxml read in 3 seconds. The manifest records `links`.
+ `formatting=indented` puts two spaces in front of a line for every element
+ it sits in and leaves the text of `pre` as it is - Chromium places every
+ element of it where it places them on the page of lines, and the Nu HTML
+ Checker finds nothing wrong with it. With `nesting` the spaces grow with
+ every level, so `nesting` 1000 makes a page of at least 1.9 MB. Files made
+ with the earlier settings keep their bytes.
- **A length of time as a setting.** `duration`, `change_interval` and `keyframe_interval` take
`10s`, `1m30s`, `1h`, `500ms` or `59.9s`, to the millisecond. A bare number
is refused rather than read as seconds, and so is a length that does not
@@ -299,6 +317,9 @@ because it turns other people's test suites red.
### Fixed
+- **The window called the byte order mark setting "Bom".** It reads "BOM" now,
+ on `txt`, `md`, `xml` and `html` alike.
+
- **Ctrl+C waited for a file that holds thousands of files to be planned.**
A zip, a tar.gz, a mail message or a page with ten thousand small pictures
inside took over a minute to plan, and a run stopped in that minute went
diff --git a/README.md b/README.md
index 7b31e827..9bec5ad7 100644
--- a/README.md
+++ b/README.md
@@ -634,7 +634,7 @@ recipe. `tfg formats ` prints the allowed range or list for each:
| `geojson` | `geometry`, `formatting`, `precision`, `altitude`, `vertices`, `holes`, `winding`, `antimeridian`, `unlocated`, `ids`, `bbox` |
| `eml` | `body`, `text_encoding`, `charset`, `headers`, `recipients`, `line_endings`, `attachments`, `attachment_format`, `attachment_size`, `filename_style`, `message_depth`, `inline`, `inline_format`, `inline_size` |
| `svg` | `width`, `height` |
-| `html` | `structure`, `nesting`, `characters`, `elements`, `bom`, `formatting`, `styles`, `inline`, `inline_format`, `inline_size` |
+| `html` | `structure`, `nesting`, `characters`, `elements`, `links`, `bom`, `formatting`, `styles`, `inline`, `inline_format`, `inline_size` |
| `yaml`, `toml` | none in this build - the document is a fixed shape, and a size is the only thing to ask for |
```
diff --git a/internal/format/htmlfile/blocks.go b/internal/format/htmlfile/blocks.go
index 799e3cde..f12cf178 100644
--- a/internal/format/htmlfile/blocks.go
+++ b/internal/format/htmlfile/blocks.go
@@ -22,21 +22,23 @@ import (
// or a list cut in the middle still renders, and it says something other than
// it meant.
type blocks struct {
- // tail is what the closing record ends with, which is the only thing the
- // shape of the file changes down here.
- tail string
// m is how the blocks are spelled. The words drawn are the same in every
// spelling, so one seed fills a page of one line and a page of many with
// the same blocks.
m *markup
+ // lead is the indentation of a block and inner the one of a list's items
+ // and a table's rows, both empty unless the page is indented.
+ lead, inner string
}
-// Shortest is the smallest closing record: an empty paragraph plus whatever
-// this shape closes after it.
-func (b blocks) Shortest() int64 { return int64(len(b.m[paraOpen]) + len(b.tail)) }
+// Shortest is the smallest closing record: an empty paragraph.
+func (b blocks) Shortest() int64 {
+ return int64(len(b.lead) + len(b.m[paraOpen]) + len(b.m[paraClose]))
+}
func (b blocks) Append(dst []byte, rng *rand.Rand) []byte {
m := b.m
+ dst = append(dst, b.lead...)
switch rng.IntN(5) {
case 0:
dst = append(dst, m[h2Open]...)
@@ -45,14 +47,19 @@ func (b blocks) Append(dst []byte, rng *rand.Rand) []byte {
case 1:
dst = append(dst, m[ulOpen]...)
for i, n := 0, 2+rng.IntN(3); i < n; i++ {
+ dst = append(dst, b.inner...)
dst = append(dst, m[liOpen]...)
dst = appendPhrase(dst, rng, 3+rng.IntN(4))
dst = append(dst, m[liClose]...)
}
+ dst = append(dst, b.lead...)
return append(dst, m[ulClose]...)
case 2:
dst = append(dst, m[tableOpen]...)
+ dst = append(dst, b.inner...)
+ dst = append(dst, m[tableHead]...)
for i, n := 0, 2+rng.IntN(3); i < n; i++ {
+ dst = append(dst, b.inner...)
dst = append(dst, m[rowOpen]...)
dst = strconv.AppendInt(dst, int64(i+1), 10)
dst = append(dst, m[cellSep]...)
@@ -61,6 +68,7 @@ func (b blocks) Append(dst []byte, rng *rand.Rand) []byte {
dst = append(dst, words[rng.IntN(len(words))]...)
dst = append(dst, m[rowClose]...)
}
+ dst = append(dst, b.lead...)
return append(dst, m[tableClose]...)
case 3:
// A real page carries entities, and an ampersand left raw is the
@@ -84,10 +92,11 @@ func (blocks) Discard() {}
func (b blocks) AppendExact(dst []byte, rng *rand.Rand, n int64) []byte {
start := len(dst)
+ dst = append(dst, b.lead...)
dst = append(dst, b.m[paraOpen]...)
- used := int64(len(dst)-start) + int64(len(b.tail))
+ used := int64(len(dst)-start) + int64(len(b.m[paraClose]))
dst = appendFiller(dst, n-used)
- return append(dst, b.tail...)
+ return append(dst, b.m[paraClose]...)
}
func appendPhrase(dst []byte, rng *rand.Rand, n int) []byte {
diff --git a/internal/format/htmlfile/catalogue.go b/internal/format/htmlfile/catalogue.go
index b8c24f7e..acc37e0e 100644
--- a/internal/format/htmlfile/catalogue.go
+++ b/internal/format/htmlfile/catalogue.go
@@ -150,10 +150,28 @@ func (s settings) catalogueHeadFor() string {
return s.spelling()[headCatalogue]
}
-// catalogueBlocks is what the catalogue adds to the body.
-func (s settings) catalogueBlocks() string {
- if !s.every {
- return ""
+// appendCatalogue appends what the catalogue adds to the body, at the depth of
+// its content.
+func (c content) appendCatalogue(dst []byte, lead string) []byte {
+ s := c.s
+ switch {
+ case !s.every:
+ return dst
+ case s.indented():
+ return appendLaidOut(dst, s.laid().catalogue, lead)
+ }
+ return append(dst, s.spelling()[bodyCatalogue]...)
+}
+
+// catalogueBytes is what appendCatalogue appends, worked out without building
+// it.
+func (c content) catalogueBytes() int64 {
+ s := c.s
+ switch {
+ case !s.every:
+ return 0
+ case s.indented():
+ return laidOutBytes(s.laid().catalogue, s.leadBytes())
}
- return s.spelling()[bodyCatalogue]
+ return int64(len(s.spelling()[bodyCatalogue]))
}
diff --git a/internal/format/htmlfile/characters.go b/internal/format/htmlfile/characters.go
index 67ce624d..baa1f0bd 100644
--- a/internal/format/htmlfile/characters.go
+++ b/internal/format/htmlfile/characters.go
@@ -29,11 +29,28 @@ const (
`and entities & < > " © € € 😀.
` + "\n"
)
-// internationalBlocks is what the setting adds, nothing for ascii, in the
-// spelling of the page.
-func (s settings) internationalBlocks() string {
- if !s.international {
- return ""
+// appendInternational appends what the setting adds, nothing for ascii, in the
+// spelling of the page and at the depth of its content.
+func (c content) appendInternational(dst []byte, lead string) []byte {
+ s := c.s
+ switch {
+ case !s.international:
+ return dst
+ case s.indented():
+ return appendLaidOut(dst, s.laid().scripts, lead)
}
- return s.spelling()[scripts]
+ return append(dst, s.spelling()[scripts]...)
+}
+
+// internationalBytes is what appendInternational appends, worked out without
+// building it.
+func (c content) internationalBytes() int64 {
+ s := c.s
+ switch {
+ case !s.international:
+ return 0
+ case s.indented():
+ return laidOutBytes(s.laid().scripts, s.leadBytes())
+ }
+ return int64(len(s.spelling()[scripts]))
}
diff --git a/internal/format/htmlfile/content.go b/internal/format/htmlfile/content.go
new file mode 100644
index 00000000..3c3b5988
--- /dev/null
+++ b/internal/format/htmlfile/content.go
@@ -0,0 +1,156 @@
+package htmlfile
+
+import (
+ "context"
+ "io"
+
+ "github.com/donislawdev/TestingFilesGenerator/internal/core"
+)
+
+// content is what a page writes after its label - the levels of nesting, the
+// paragraphs in other scripts, the catalogue, the links, the blocks and what
+// closes them - each one written from the settings and counted without being
+// built. Split from settings on 2026-10-14, when indentation and links took
+// that type past the methods one type may carry (typeshape_test.go): settings
+// says what was asked for, and this writes it.
+type content struct{ s settings }
+
+// content is the writing of a page of these settings.
+func (s settings) content() content { return content{s} }
+
+// chunks builds what a page repeats and writes it each time it passes
+// writeChunk, asking between two writes whether the run was stopped. After the
+// first error it builds nothing more, and close says what the error was.
+type chunks struct {
+ ctx context.Context
+ w io.Writer
+ buf []byte
+ err error
+}
+
+func newChunks(ctx context.Context, w io.Writer) *chunks {
+ return &chunks{ctx: ctx, w: w, buf: make([]byte, 0, writeChunk+1024)}
+}
+
+// add appends two pieces, one after the other - most often an indentation and
+// what stands at it.
+func (c *chunks) add(first, second string) {
+ if c.err != nil {
+ return
+ }
+ c.buf = append(append(c.buf, first...), second...)
+ if len(c.buf) >= writeChunk {
+ c.err = flush(c.ctx, c.w, c.buf)
+ c.buf = c.buf[:0]
+ }
+}
+
+// close writes what is left, or says why nothing more was written.
+func (c *chunks) close() error {
+ if c.err != nil {
+ return c.err
+ }
+ return core.WriteAll(c.w, c.buf)
+}
+
+// flush writes what was built so far, after asking whether the run was
+// interrupted.
+func flush(ctx context.Context, w io.Writer, buf []byte) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ return core.WriteAll(w, buf)
+}
+
+// levelIndent is the indentation of the level k levels in, cut from the
+// deepest one rather than built for every level, nothing unless the page is
+// indented.
+func (c content) levelIndent(in indents, k int) string {
+ return in.lead[:min(len(in.lead), int(indentationBytes(c.s.bodyDepth()+k)))]
+}
+
+// levelsBytes is what the levels take opened, or closed - each one at the depth
+// it opens at, which an indented page writes in front of it. Indented, a level
+// k deep has 2(b+k) spaces, so the sum is worked out rather than added up.
+func (c content) levelsBytes(piece string) int64 {
+ s := c.s
+ n := int64(s.nesting)
+ total := n * int64(len(piece))
+ if s.indented() {
+ b := int64(s.bodyDepth())
+ total += int64(len(indentUnit)) * (n*b + n*(n-1)/2)
+ }
+ return total
+}
+
+// writeOpening writes what follows the label: the levels of nesting, then the
+// paragraphs in other scripts and the catalogue. The levels are written a
+// chunk at a time - a hundred thousand of them indented are 10 GB.
+func (c content) writeOpening(ctx context.Context, w io.Writer, in indents) error {
+ s, m := c.s, c.s.spelling()
+ out := newChunks(ctx, w)
+ for k := 0; k < s.nesting && out.err == nil; k++ {
+ out.add(c.levelIndent(in, k), m[divOpen])
+ }
+ if out.err == nil {
+ out.buf = c.appendInternational(out.buf, in.lead)
+ out.buf = c.appendCatalogue(out.buf, in.lead)
+ }
+ return out.close()
+}
+
+// openingBytes is what writeOpening writes, worked out without building it.
+func (c content) openingBytes() int64 {
+ return c.levelsBytes(c.s.spelling()[divOpen]) + c.internationalBytes() + c.catalogueBytes()
+}
+
+// writeClosing writes what closes the levels from the innermost out, then the
+// body and the document. It follows the last block rather than ending it, so a
+// page closing a hundred thousand indented levels never builds them as one
+// record.
+func (c content) writeClosing(ctx context.Context, w io.Writer, in indents) error {
+ s, m := c.s, c.s.spelling()
+ out := newChunks(ctx, w)
+ for k := s.nesting - 1; k >= 0 && out.err == nil; k-- {
+ out.add(c.levelIndent(in, k), m[divClose])
+ }
+ out.add("", c.documentClose())
+ return out.close()
+}
+
+// closingBytes is what writeClosing writes, worked out without building it.
+func (c content) closingBytes() int64 {
+ return c.levelsBytes(c.s.spelling()[divClose]) + int64(len(c.documentClose()))
+}
+
+// documentClose closes the body and the document of a page, and nothing in a
+// fragment, which has neither.
+func (c content) documentClose() string {
+ s := c.s
+ if s.shape != structureDocument {
+ return ""
+ }
+ closing := s.spelling()[bodyClose]
+ if s.indented() {
+ return joined(layOut(closing, s.bodyDepth()))
+ }
+ return closing
+}
+
+// blocks is the body builder for these settings.
+//
+// What closes the levels, the body and the document follows the last record
+// (writeClosing) and is counted out of the bytes the records fill, so a change
+// to it has to reach both. Until 2026-10-14 it sat in the last record, and the
+// golden hashes - a fragment and a page nested on one line among them - hold
+// the bytes to what they were then.
+func (c content) blocks(in indents) blocks {
+ return blocks{m: c.s.spelling(), lead: in.lead, inner: in.inner}
+}
+
+// shortestRecord is the smallest closing record, worked out without building
+// the indentation: an empty paragraph at the depth of the content.
+func (c content) shortestRecord() int64 {
+ m := c.s.spelling()
+ return c.s.leadBytes() + int64(len(m[paraOpen])+len(m[paraClose]))
+}
diff --git a/internal/format/htmlfile/html.go b/internal/format/htmlfile/html.go
index b3a0d4cb..6ffd7aa9 100644
--- a/internal/format/htmlfile/html.go
+++ b/internal/format/htmlfile/html.go
@@ -56,9 +56,9 @@ func init() {
// than a comment.
Label: format.LabelVisible,
Oracle: "python-html",
- // Links come later, if at all (docs/HTML-2-2026-10-14.md section 3,
- // B4). Declaring only what is here is what makes a recipe asking for
- // them fail loudly instead of quietly producing something else.
+ // Declaring only what is here is what makes a recipe asking for
+ // something else fail loudly instead of quietly producing a page
+ // without it.
Properties: properties(),
GeneratorVersion: generatorVersion,
Generator: generator{},
@@ -88,7 +88,7 @@ func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan
return format.Plan{}, err
}
- pictures, pictureBytes, err := planPictures(ctx, r, s.pictures, s.spelling())
+ pictures, pictureBytes, err := planPictures(ctx, r, s)
if err != nil {
return format.Plan{}, err
}
@@ -155,9 +155,10 @@ func belowMinimum(s settings, requested, min int64) error {
// the reason names, whatever else was asked.
reason = core.Says("html.MinimumWithPictures", "the file holds whole blocks and every picture in base64, which is about a third larger than the picture itself, and together they need that much")
case s.adds():
- reason = core.Says("html.MinimumWithSettings", "the file holds whole blocks and what %s, %s, %s, %s and %s add to them, and together they need that much",
+ reason = core.Says("html.MinimumWithSettings", "the file holds whole blocks and what %s, %s, %s, %s, %s, %s and %s add to them, and together they need that much",
core.A("Nesting", core.LabelTerm(settingNesting)), core.A("Characters", core.LabelTerm(settingCharacters)),
- core.A("Elements", core.LabelTerm(settingElements)), core.A("BOM", core.LabelTerm(textenc.SettingBOM)),
+ core.A("Elements", core.LabelTerm(settingElements)), core.A("Links", core.LabelTerm(settingLinks)),
+ core.A("BOM", core.LabelTerm(textenc.SettingBOM)), core.A("Formatting", core.LabelTerm(settingFormatting)),
core.A("Styles", core.LabelTerm(settingStyles)))
case s.shape == structureFragment:
reason = core.Says("html.MinimumFragment", "a fragment holds whole blocks, and one of them needs that much")
@@ -182,6 +183,10 @@ func labelledHead(s settings, r format.Request, min int64) (head, heading string
label := core.Label("html", r.Bytes, r.Seed)
m := s.spelling()
heading = m[labelOpen] + label + m[labelClose]
+ if s.indented() {
+ // The label sits in the body, outside the levels.
+ heading = indentation(s.bodyDepth()) + heading
+ }
plain := s.prologue()
head, extra := plain, int64(0)
@@ -211,16 +216,27 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error {
// The label sits outside the levels of nesting, so a reader that drops
// elements past its depth - libxml2 does, measured on 2026-10-14 - still
// shows which file it read.
- head := m.head + m.labelLine + m.settings.opening()
+ c, in := m.settings.content(), m.settings.indents()
+ head := m.head + m.labelLine
if err := core.WriteAll(w, []byte(head)); err != nil {
return err
}
- if err := writePictures(ctx, w, m.pictures); err != nil {
+ if err := c.writeOpening(ctx, w, in); err != nil {
+ return err
+ }
+ if err := c.writeLinks(ctx, w, in); err != nil {
+ return err
+ }
+ if err := writePictures(ctx, w, m.pictures, in.lead); err != nil {
return err
}
rng := core.NewRand(m.seed)
- return core.FillRecords(ctx, w, rng, p.Bytes-int64(len(head))-m.pictureBytes, m.settings.blocks())
+ filled := p.Bytes - int64(len(head)) - c.openingBytes() - c.linksBytes() - m.pictureBytes - c.closingBytes()
+ if err := core.FillRecords(ctx, w, rng, filled, c.blocks(in)); err != nil {
+ return err
+ }
+ return c.writeClosing(ctx, w, in)
}
// minimumBytes is the smallest whole page, which is what the registry declares.
@@ -228,9 +244,11 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error {
// do.
func minimumBytes() int64 { return minimumFor(settings{shape: structureDocument}) }
-// minimumFor is the prologue, what the settings open the body with and one
-// whole block, computed from the pieces Write uses rather than written down so
-// it cannot drift away from the template.
+// minimumFor is the prologue, what the settings open the body with, one whole
+// block and what closes the levels and the document, computed from the pieces
+// Write uses rather than written down so it cannot drift away from the
+// template.
func minimumFor(s settings) int64 {
- return int64(len(s.prologue())) + s.openingBytes() + int64(len(s.spelling()[paraOpen])) + s.tailBytes()
+ c := s.content()
+ return int64(len(s.prologue())) + c.openingBytes() + c.linksBytes() + c.shortestRecord() + c.closingBytes()
}
diff --git a/internal/format/htmlfile/indent.go b/internal/format/htmlfile/indent.go
new file mode 100644
index 00000000..6ed4a00a
--- /dev/null
+++ b/internal/format/htmlfile/indent.go
@@ -0,0 +1,151 @@
+package htmlfile
+
+import (
+ "regexp"
+ "strings"
+)
+
+// An indented page: every line of a page of lines, indented two spaces for
+// each element open where it starts - the width JSON indents with, and the
+// children of html indented as Prettier lays them out. Nothing else about the
+// lines changes, so an indented page is a page of lines with spaces in front
+// of them (docs/HTML-5-2026-10-14.md section 2.2). Measured on 2026-10-14,
+// Chromium places every element of the two the same.
+//
+// The pieces written once - the head, the paragraphs in other scripts and the
+// catalogue - are indented here by the elements they open. The pieces a page
+// repeats - the levels of nesting, the blocks, the links and the pictures -
+// sit at a depth only the settings know, and take their indentation from it
+// as they are written, so a hundred thousand levels are never built in memory.
+
+// indentUnit is one level of indentation.
+const indentUnit = " "
+
+// indentation is the indentation of a line depth levels deep.
+func indentation(depth int) string { return strings.Repeat(indentUnit, depth) }
+
+// indentationBytes is its length, worked out without building it.
+func indentationBytes(depth int) int64 { return int64(depth * len(indentUnit)) }
+
+// line is one line of a piece laid out by its elements, without its line
+// break. A line inside pre is text rather than layout, and takes no
+// indentation at all - not its own and not the depth it is written at.
+type line struct {
+ text string
+ pre bool
+}
+
+// voidElements have no end tag, so they open nothing.
+var voidElements = map[string]bool{
+ "area": true, "base": true, "br": true, "col": true, "embed": true, "hr": true, "img": true,
+ "input": true, "link": true, "meta": true, "source": true, "track": true, "wbr": true,
+}
+
+// tag finds a start or an end tag: the slash, the name, and what follows the
+// name up to the bracket. The doctype starts with "!" and is not one.
+var tag = regexp.MustCompile(`<(/?)([a-zA-Z][a-zA-Z0-9-]*)([^>]*)>`)
+
+// layOut indents every line of text by the depth of the elements open where it
+// starts, beginning at depth: a line that starts by closing one stands at the
+// depth of its start tag. text ends with a line break, the way every piece of
+// a page of lines does.
+func layOut(text string, depth int) []line {
+ var out []line
+ if text == "" {
+ // Nothing is no lines, not one empty one - a fragment with no sheet
+ // has no prologue, and would otherwise open with a blank line.
+ return out
+ }
+ inPre := false
+ for _, raw := range strings.Split(strings.TrimSuffix(text, "\n"), "\n") {
+ out = append(out, lineAt(raw, depth, inPre))
+ depth, inPre = after(raw, depth, inPre)
+ }
+ return out
+}
+
+// lineAt is one line written depth levels deep, or as it is inside pre.
+func lineAt(raw string, depth int, inPre bool) line {
+ if inPre {
+ return line{text: raw, pre: true}
+ }
+ if strings.HasPrefix(raw, "") {
+ depth--
+ }
+ return line{text: indentation(max(depth, 0)) + raw}
+}
+
+// after is the depth once the tags of a line are read, and whether pre is still
+// open.
+func after(raw string, depth int, inPre bool) (int, bool) {
+ for _, m := range tag.FindAllStringSubmatch(raw, -1) {
+ closing, name := m[1] == "/", strings.ToLower(m[2])
+ if voidElements[name] || strings.HasSuffix(m[3], "/") {
+ continue
+ }
+ if name == "pre" {
+ inPre = !closing
+ }
+ if closing {
+ depth--
+ } else {
+ depth++
+ }
+ }
+ return depth, inPre
+}
+
+// laidOutPieces are the pieces an indented page writes once inside its levels:
+// the paragraphs in other scripts and the catalogue, laid out by their own
+// elements at depth zero and given the depth of the content as they are
+// written.
+type laidOutPieces struct{ scripts, catalogue []line }
+
+// laidOut are made when the program starts, one for each way of carrying
+// styles, out of the pieces of a page of lines - an indented page is spelled as
+// one.
+var laidOut = func() map[string]laidOutPieces {
+ out := map[string]laidOutPieces{}
+ for _, styles := range []string{stylesNone, stylesEmbedded, stylesInline} {
+ m := spellings[spellingKey{false, styles}]
+ out[styles] = laidOutPieces{scripts: layOut(m[scripts], 0), catalogue: layOut(m[bodyCatalogue], 0)}
+ }
+ return out
+}()
+
+// laid is the laid out pieces of these settings.
+func (s settings) laid() laidOutPieces { return laidOut[s.styleName()] }
+
+// joined is lines laid out at depth zero, as one string.
+func joined(lines []line) string {
+ var b strings.Builder
+ for _, l := range lines {
+ b.WriteString(l.text + "\n")
+ }
+ return b.String()
+}
+
+// laidOutBytes is what lines take written at a depth whose indentation is lead
+// bytes long.
+func laidOutBytes(lines []line, lead int64) int64 {
+ var n int64
+ for _, l := range lines {
+ n += int64(len(l.text)) + 1
+ if !l.pre {
+ n += lead
+ }
+ }
+ return n
+}
+
+// appendLaidOut appends lines written at the depth whose indentation is lead.
+func appendLaidOut(dst []byte, lines []line, lead string) []byte {
+ for _, l := range lines {
+ if !l.pre {
+ dst = append(dst, lead...)
+ }
+ dst = append(dst, l.text...)
+ dst = append(dst, '\n')
+ }
+ return dst
+}
diff --git a/internal/format/htmlfile/links.go b/internal/format/htmlfile/links.go
new file mode 100644
index 00000000..f6c4d150
--- /dev/null
+++ b/internal/format/htmlfile/links.go
@@ -0,0 +1,89 @@
+package htmlfile
+
+import (
+ "context"
+ "io"
+)
+
+// The links a page carries when links is above 0: a list above the pictures
+// and the blocks, before the pictures because lxml loses the rest of a page
+// after a picture of 15 MB (docs/HTML-3-2026-10-14.md section 9).
+//
+// Eighteen kinds, in turn, each saying in its text what kind it is, so what a
+// program pulls out of the page reads without this file beside it. Every one of
+// them was read on 2026-10-14 by the Nu HTML Checker, which finds nothing wrong
+// with any, by jsdom, lxml, BeautifulSoup and Python's urljoin, which resolve
+// them alike, and by DOMPurify, which with its defaults takes out the data:
+// address and target (docs/HTML-5-2026-10-14.md section 1.1). The two the
+// checker calls errors - a user name in the address and a raw space - are left
+// out: they belong to damaged files and to the security set (decision D3).
+//
+// Every address is on a domain RFC 2606 keeps for examples, and the number is
+// in the range NANPA keeps for fiction, so a link followed by mistake reaches
+// nobody. All of them are ASCII, which keeps links and characters independent.
+var linkKinds = []string{
+ `link to this list`,
+ `relative link`,
+ `link to the folder above`,
+ `link from the root`,
+ `link with only a query`,
+ `absolute link with a query and a fragment`,
+ `http link`,
+ `link without a scheme`,
+ `percent encoded link`,
+ `international domain link`,
+ `link with a port`,
+ `mail link`,
+ `phone link`,
+ `ftp link`,
+ `download link`,
+ `link to a new window`,
+ `nofollow link`,
+ `data link`,
+}
+
+// linksDetail is what both surfaces say about the setting.
+const linksDetail = "How many links the page carries, in a list above the pictures and the blocks. " +
+ "They go through 18 kinds in turn and then repeat - a link to the list itself, relative, to the folder above, from the root, a query alone, " +
+ "absolute with a query and a fragment, http, without a scheme, percent encoded, an international domain, a port, " +
+ "mailto:, tel:, ftp:, a download, a new window, nofollow and data: - and the text of each says which it is. " +
+ "Every address is on a domain kept for examples. DOMPurify with its defaults takes out the data: address and the target that opens a new window, " +
+ "and it ran out of 8 GB of memory over a million links, which lxml read in 3 seconds. " +
+ "The catalogue of every element has six links of its own, which this does not count."
+
+// linksBytes is what the list takes, worked out without building it: the list
+// around its items, and the items, the kinds in turn.
+func (c content) linksBytes() int64 {
+ s := c.s
+ if s.links == 0 {
+ return 0
+ }
+ m, lead := s.spelling(), s.leadBytes()
+ n := int64(s.links)
+ var cycle, part int64
+ for i, k := range linkKinds {
+ cycle += int64(len(k))
+ if i < s.links%len(linkKinds) {
+ part += int64(len(k))
+ }
+ }
+ item := s.innerBytes() + int64(len(m[liOpen])+len(m[liClose]))
+ return 2*lead + int64(len(m[linksOpen])+len(m[linksClose])) + n*item + n/int64(len(linkKinds))*cycle + part
+}
+
+// writeLinks writes the list a chunk at a time, so a million links never sit in
+// memory and an interrupted run stops between two chunks.
+func (c content) writeLinks(ctx context.Context, w io.Writer, in indents) error {
+ s, m := c.s, c.s.spelling()
+ if s.links == 0 {
+ return nil
+ }
+ out := newChunks(ctx, w)
+ out.add(in.lead, m[linksOpen])
+ for i := 0; i < s.links && out.err == nil; i++ {
+ out.add(in.inner, m[liOpen])
+ out.add(linkKinds[i%len(linkKinds)], m[liClose])
+ }
+ out.add(in.lead, m[linksClose])
+ return out.close()
+}
diff --git a/internal/format/htmlfile/markup.go b/internal/format/htmlfile/markup.go
index d3baf7bc..edbfe166 100644
--- a/internal/format/htmlfile/markup.go
+++ b/internal/format/htmlfile/markup.go
@@ -46,13 +46,20 @@ const (
liOpen
liClose
ulClose
+ // The table opens on a line of its own and its head row on the next, so
+ // an indented page can put the row a level further in.
tableOpen
+ tableHead
rowOpen
cellSep
rowClose
tableClose
quoteOpen
quoteClose
+ // The list of links around its items, which are spelled as the items of
+ // any list.
+ linksOpen
+ linksClose
// The paragraphs in other scripts, and a picture's line around its name
// and its base64.
scripts
@@ -95,14 +102,20 @@ var plain = markup{
liOpen: "",
liClose: "\n",
ulClose: "\n",
- tableOpen: "\n| id | field | value |
\n",
+ tableOpen: "\n",
+ tableHead: "| id | field | value |
\n",
rowOpen: "| ",
cellSep: " | ",
rowClose: " |
\n",
// The table closes and the quote holds a paragraph.
- tableClose: "
\n",
- quoteOpen: "",
- quoteClose: "
\n",
+ tableClose: "
\n",
+ quoteOpen: "",
+ quoteClose: "
\n",
+ // The id is what the first link points at, and one no document property
+ // has: DOMPurify 3.4.17 takes out an id that would shadow one - links and
+ // forms both, measured on 2026-10-14 - and the link would point nowhere.
+ linksOpen: `` + "\n",
+ linksClose: "
\n",
scripts: international,
pictureLead: `![`,
pictureTail: `]()
` + "\n",
@@ -249,16 +262,17 @@ func (s settings) styleName() string {
return s.styles
}
-// spelling is how these settings spell a page.
+// spelling is how these settings spell a page. An indented page is spelled as
+// a page of lines and indented as it is written.
func (s settings) spelling() *markup {
- return spellings[spellingKey{s.minified, s.styleName()}]
+ return spellings[spellingKey{s.minified(), s.styleName()}]
}
// lineEnding is what the manifest says the lines of the page end with: none for
// a page on one line, which has no line to end, where lf would describe a file
// that is not there.
func (s settings) lineEnding() string {
- if s.minified {
+ if s.minified() {
return "none"
}
return "lf"
diff --git a/internal/format/htmlfile/page.go b/internal/format/htmlfile/page.go
index b0ebf906..e80dc9a2 100644
--- a/internal/format/htmlfile/page.go
+++ b/internal/format/htmlfile/page.go
@@ -1,12 +1,10 @@
package htmlfile
-import "strings"
-
// How a page is laid out from its settings, split out of html.go on 2026-10-14.
// The order is the one measured and decided in docs/HTML-2-2026-10-14.md
// section 12: the mark, the skeleton and the head, the label, the levels of
-// nesting, the paragraphs in other scripts, the catalogue, the blocks, and the
-// closing paragraph with everything it closes after it.
+// nesting, the paragraphs in other scripts, the catalogue, the links, the
+// pictures, the blocks, and what closes the levels, the body and the document.
// The markup itself - the skeleton, the label, the levels and the blocks - is
// spelled in markup.go, in every layout and every way of carrying styles.
@@ -19,6 +17,11 @@ const emptyTitle = ``
// bytes so no escape in this file can be turned into something else on the way.
var byteOrderMark = string([]byte{0xEF, 0xBB, 0xBF})
+// writeChunk is how much of what a page repeats is built before it is written,
+// and so how often an interrupted run is noticed - the size the blocks are
+// written in (core.FillRecords).
+const writeChunk = 32 * 1024
+
// prologue is everything in front of the label: the mark, and the skeleton down
// to the body with whatever the catalogue and the style sheet put in the head -
// or for a fragment the mark and the sheet. A fragment has no head, and a sheet
@@ -32,56 +35,66 @@ func (s settings) prologue() string {
if s.shape == structureDocument {
p = m[headOpen] + s.catalogueHeadFor() + m[sheet] + m[headClose]
}
+ if s.indented() {
+ p = joined(layOut(p, 0))
+ }
if s.bom {
p = byteOrderMark + p
}
return p
}
-// opening is what follows the label: the levels of nesting, then the paragraphs
-// in other scripts and the catalogue. Built when the file is written and never
-// kept in the plan - a hundred thousand levels are 600 KB, and the engine holds
-// the plan of every file at once.
-func (s settings) opening() string {
- return strings.Repeat(s.spelling()[divOpen], s.nesting) + s.internationalBlocks() + s.catalogueBlocks()
+// depth is how many elements the content sits in: the levels of nesting, and
+// for a page the html and body elements around them. The label sits at the
+// depth of the body, outside the levels.
+func (s settings) depth() int {
+ return s.bodyDepth() + s.nesting
}
-// openingBytes is the length of opening, worked out without building it.
-func (s settings) openingBytes() int64 {
- return int64(s.nesting*len(s.spelling()[divOpen]) + len(s.internationalBlocks()) + len(s.catalogueBlocks()))
+// bodyDepth is the depth of what the body holds - none for a fragment, which
+// has no body.
+func (s settings) bodyDepth() int {
+ if s.shape == structureDocument {
+ return 2
+ }
+ return 0
}
-// blocks is the body builder for these settings.
-//
-// The settings have to reach the BUILDER and not only the prologue, and that is
-// the part of this easy to miss: the bytes that close the levels, the body and
-// the document sit in the last RECORD rather than in a footer. A change that
-// swapped only the prologue would end a fragment with