diff --git a/app_import_test.go b/app_import_test.go index 25f15b3..c8cbd36 100644 --- a/app_import_test.go +++ b/app_import_test.go @@ -198,30 +198,35 @@ func TestImportLogFile_DoesNotRememberAFormatThatFailed(t *testing.T) { } // A declared format has to reach the importer, or the dialog is decoration. +// +// Detection reads a JSON line on its own now, so the contrast has to come from +// something it cannot know: a level under a field name of the application's +// own choosing. func TestPreviewLogFile_AppliesTheDeclaredFormat(t *testing.T) { app, _ := importApp(t) p := filepath.Join(t.TempDir(), "app.json.log") - line := `{"time":"2026-03-17T21:42:10Z","level":"error","msg":"connection refused"}` + "\n" + line := `{"time":"2026-03-17T21:42:10Z","prio":"error","msg":"connection refused"}` + "\n" if err := os.WriteFile(p, []byte(line), 0o600); err != nil { t.Fatal(err) } - // Detection reads nothing out of a JSON line, which is the gap the modes - // exist to close. auto, err := app.PreviewLogFile(p, models.ImportFormat{Mode: models.ImportAuto}) if err != nil { t.Fatal(err) } if auto.Result.LevelDetected != 0 { - t.Errorf("detection claims to read a level out of JSON: %d", auto.Result.LevelDetected) + t.Errorf("a level was read from a field nothing could have guessed: %d", + auto.Result.LevelDetected) } - declared, err := app.PreviewLogFile(p, models.ImportFormat{Mode: models.ImportJSON}) + declared, err := app.PreviewLogFile(p, models.ImportFormat{ + Mode: models.ImportJSON, JSONLevel: "prio", + }) if err != nil { t.Fatal(err) } if declared.Result.LevelDetected != 1 || declared.Messages[0].SeverityLabel != "Error" { - t.Errorf("declaring JSON changed nothing: level=%d severity=%q", + t.Errorf("naming the field changed nothing: level=%d severity=%q", declared.Result.LevelDetected, declared.Messages[0].SeverityLabel) } } diff --git a/docs/assets/img/import-dark-1200.webp b/docs/assets/img/import-dark-1200.webp index 9c06cdb..848a035 100644 Binary files a/docs/assets/img/import-dark-1200.webp and b/docs/assets/img/import-dark-1200.webp differ diff --git a/docs/assets/img/import-dark-2000.webp b/docs/assets/img/import-dark-2000.webp index 3d24afa..6d9e0e8 100644 Binary files a/docs/assets/img/import-dark-2000.webp and b/docs/assets/img/import-dark-2000.webp differ diff --git a/docs/assets/img/import-dark.png b/docs/assets/img/import-dark.png index 405c74a..8f91c97 100644 Binary files a/docs/assets/img/import-dark.png and b/docs/assets/img/import-dark.png differ diff --git a/docs/assets/img/import-format-dark-1200.webp b/docs/assets/img/import-format-dark-1200.webp index 61689f3..acfca6b 100644 Binary files a/docs/assets/img/import-format-dark-1200.webp and b/docs/assets/img/import-format-dark-1200.webp differ diff --git a/docs/assets/img/import-format-dark-2000.webp b/docs/assets/img/import-format-dark-2000.webp index 2cfda3a..e81c62d 100644 Binary files a/docs/assets/img/import-format-dark-2000.webp and b/docs/assets/img/import-format-dark-2000.webp differ diff --git a/docs/assets/img/import-format-dark.png b/docs/assets/img/import-format-dark.png index c7af2e9..9d45f1a 100644 Binary files a/docs/assets/img/import-format-dark.png and b/docs/assets/img/import-format-dark.png differ diff --git a/docs/assets/img/import-format-light-1200.webp b/docs/assets/img/import-format-light-1200.webp index 0aae3ff..e3bfdc3 100644 Binary files a/docs/assets/img/import-format-light-1200.webp and b/docs/assets/img/import-format-light-1200.webp differ diff --git a/docs/assets/img/import-format-light-2000.webp b/docs/assets/img/import-format-light-2000.webp index c6c43ec..1f42c6c 100644 Binary files a/docs/assets/img/import-format-light-2000.webp and b/docs/assets/img/import-format-light-2000.webp differ diff --git a/docs/assets/img/import-format-light.png b/docs/assets/img/import-format-light.png index 84fd796..f976ebd 100644 Binary files a/docs/assets/img/import-format-light.png and b/docs/assets/img/import-format-light.png differ diff --git a/docs/assets/img/import-light-1200.webp b/docs/assets/img/import-light-1200.webp index 02fd184..dbce8aa 100644 Binary files a/docs/assets/img/import-light-1200.webp and b/docs/assets/img/import-light-1200.webp differ diff --git a/docs/assets/img/import-light-2000.webp b/docs/assets/img/import-light-2000.webp index 50880bb..e3219b8 100644 Binary files a/docs/assets/img/import-light-2000.webp and b/docs/assets/img/import-light-2000.webp differ diff --git a/docs/assets/img/import-light.png b/docs/assets/img/import-light.png index b714451..3f8ed58 100644 Binary files a/docs/assets/img/import-light.png and b/docs/assets/img/import-light.png differ diff --git a/frontend/screenshots/fixtures.js b/frontend/screenshots/fixtures.js index 8089a63..33c6c84 100644 --- a/frontend/screenshots/fixtures.js +++ b/frontend/screenshots/fixtures.js @@ -484,6 +484,15 @@ export const IMPORT_PREVIEW = { // fixture has no stack trace in it. unmatched: IMPORT_SCRIPT.filter((e) => e.silent).length, joined: 0, + // Three lines carry a priority and eleven do not: no single shape holds + // the file, which is exactly what "mixed" is for. + byShape: { + syslog: IMPORT_SCRIPT.filter((e) => e.pri).length, + plain: IMPORT_SCRIPT.filter((e) => !e.pri && !e.silent).length, + none: IMPORT_SCRIPT.filter((e) => e.silent).length, + }, + detected: 'mixed', + detectedMode: '', stopped: false, bySeverity: IMPORT_SCRIPT.reduce((acc, e) => { const label = SEVERITY_LABELS[e.sev]; diff --git a/frontend/src/components/ImportDialog.svelte b/frontend/src/components/ImportDialog.svelte index 9249c6f..1c90694 100644 --- a/frontend/src/components/ImportDialog.svelte +++ b/frontend/src/components/ImportDialog.svelte @@ -36,7 +36,80 @@ const dispatch = createEventDispatcher<{ imported: ImportResult }>(); - const MODES: ImportMode[] = ['auto', 'syslog', 'json', 'access', 'logfmt', 'custom']; + /** + * The formats offered, by category. + * + * The list used to name the six parsing ENGINES, while the recogniser knew + * twenty formats — so a file could be reported as "Kubernetes klog" and + * then not be in the list at all. What a reader thinks in is a format's + * name, not an engine's, so the entries are names; several map to the same + * engine, and a couple carry field names with them. + * + * An entry's label is the same string the verdict uses, so the file cannot + * be called one thing above and another below. + */ + type FormatEntry = { + id: string; + group: string; + label: string; + hint: string; + mode: ImportMode; + preset?: Partial; + }; + + const FORMATS: FormatEntry[] = [ + { id: 'auto', group: '', label: 'import.format_auto', hint: 'import.hint_auto', mode: 'auto' }, + + { id: 'syslog', group: 'import.group_syslog', label: 'import.format_syslog', hint: 'import.hint_syslog', mode: 'syslog' }, + { id: 'bsd', group: 'import.group_syslog', label: 'import.shape_bsd', hint: 'import.hint_bsd', mode: 'bsd' }, + + { id: 'access', group: 'import.group_web', label: 'import.format_access', hint: 'import.hint_access', mode: 'access' }, + { id: 'apache', group: 'import.group_web', label: 'import.shape_apache', hint: 'import.hint_apache', mode: 'apache' }, + + { id: 'json', group: 'import.group_structured', label: 'import.format_json', hint: 'import.hint_json', mode: 'json' }, + { + id: 'clef', group: 'import.group_structured', label: 'import.format_clef', + hint: 'import.hint_clef', mode: 'json', + preset: { jsonTime: '@t', jsonLevel: '@l', jsonMessage: '@m' }, + }, + { id: 'logfmt', group: 'import.group_structured', label: 'import.format_logfmt', hint: 'import.hint_logfmt', mode: 'logfmt' }, + + { id: 'klog', group: 'import.group_platform', label: 'import.shape_klog', hint: 'import.hint_klog', mode: 'klog' }, + { id: 'logcat', group: 'import.group_platform', label: 'import.shape_logcat', hint: 'import.hint_logcat', mode: 'logcat' }, + { id: 'epoch', group: 'import.group_platform', label: 'import.shape_epoch', hint: 'import.hint_epoch', mode: 'epoch' }, + + { id: 'custom', group: 'import.group_custom', label: 'import.format_custom', hint: 'import.hint_custom', mode: 'custom' }, + ]; + + // Grouped for the markup, in the order above. + const GROUPED: { key: string; entries: FormatEntry[] }[] = FORMATS.reduce((acc, entry) => { + const last = acc[acc.length - 1]; + if (last && last.key === entry.group) last.entries.push(entry); + else acc.push({ key: entry.group, entries: [entry] }); + return acc; + }, [] as { key: string; entries: FormatEntry[] }[]); + + /** Which entry a format IS, so the list shows what is in force. */ + function entryFor(f: ImportFormat): FormatEntry { + const sameMode = FORMATS.filter(e => e.mode === f.mode); + const withPreset = sameMode.find(e => e.preset + && Object.entries(e.preset).every(([k, v]) => + (f as unknown as Record)[k] === v)); + return withPreset ?? sameMode.find(e => !e.preset) ?? FORMATS[0]; + } + + /** Choosing an entry gives exactly that entry's configuration. */ + function pickFormat(id: string) { + const entry = FORMATS.find(e => e.id === id) ?? FORMATS[0]; + adopted = false; + format = { + ...format, + mode: entry.mode, + jsonTime: '', jsonLevel: '', jsonMessage: '', jsonHost: '', jsonApp: '', + ...(entry.preset ?? {}), + }; + refresh(); + } let path = ''; let preview: ImportPreview | null = null; @@ -46,6 +119,27 @@ let formatError = ''; let format: ImportFormat = { mode: 'auto', joinContinuations: true, skipUnmatched: false }; + // Set when the format was chosen by the detector rather than by hand, so + // the panel can say which of the two happened. + let adopted = false; + + // The name of a shape the detector can report. The four that are also + // modes reuse the mode's own label, so the dropdown and the verdict cannot + // disagree about what "JSON" is called. + const SHAPE_LABEL: Record = { + syslog: 'import.format_syslog', + json: 'import.format_json', + access: 'import.format_access', + logfmt: 'import.format_logfmt', + bsd: 'import.shape_bsd', + klog: 'import.shape_klog', + logcat: 'import.shape_logcat', + apache: 'import.shape_apache', + epoch: 'import.shape_epoch', + plain: 'import.shape_plain', + mixed: 'import.shape_mixed', + }; + let debounce: ReturnType | undefined; // Which request is the current one. Two previews can be in flight on a // large file, and the one that finishes last is not necessarily the one @@ -61,6 +155,7 @@ busy = false; showFormat = false; formatError = ''; + adopted = false; } export function close() { @@ -79,6 +174,7 @@ format = await getImportFormat(); preview = await previewLogFile(path, format); formatError = ''; + await adoptDetected(); } catch (e: any) { toastError(e?.message || String(e)); path = ''; @@ -88,6 +184,27 @@ } } + /** + * Take the detector's word for it, when it has one. + * + * Only from 'auto', and only once: a format chosen by hand is a decision, + * and overruling it would be the application arguing with the operator. + * The sample is then read again through the chosen mode, so what is on + * screen is what that mode actually produces rather than what the chain + * produced on the way to naming it. + */ + async function adoptDetected() { + const mode = preview?.result?.detectedMode; + if (!mode || format.mode !== 'auto') return; + format = { ...format, mode }; + adopted = true; + try { + preview = await previewLogFile(path, format); + } catch (e: any) { + formatError = e?.message || String(e); + } + } + /** * Re-read the sample with the format as it now stands. * @@ -141,7 +258,8 @@ $: severities = preview ? Object.entries(preview.result.bySeverity).sort((a, b) => b[1] - a[1]) : []; - $: modeLabel = $_(`import.format_${format.mode}`); + $: current = entryFor(format); + $: modeLabel = $_(current.label); @@ -169,6 +287,9 @@
{preview.result.imported.toLocaleString()}{$_('import.linesSampled')}
{preview.result.syslog.toLocaleString()}{$_('import.syslogLines')}
{plain.toLocaleString()}{$_('import.plainLines')}
+ {#if preview.result.hostDetected > 0} +
{preview.result.hostDetected.toLocaleString()}{$_('import.hostDetected')}
+ {/if} {#if preview.result.unmatched > 0}
{preview.result.unmatched.toLocaleString()}{$_('import.unmatched')}
{/if} @@ -177,6 +298,14 @@ {/if} + {#if preview.result.detected} +

+ {$_('import.detected')} + {$_(SHAPE_LABEL[preview.result.detected] ?? 'import.shape_plain')}{#if adopted} + {$_('import.adopted')}{/if} +

+ {/if} + {#if plain > 0 && format.mode === 'auto'}

{$_('import.inferred', { @@ -223,13 +352,23 @@

-

{$_(`import.hint_${format.mode}`)}

+

{$_(current.hint)}

{#if format.mode === 'json' || format.mode === 'logfmt'}

{$_('import.fieldsHint')}

@@ -343,6 +482,8 @@ } .lead { margin: 0; font-size: 12px; color: var(--text-secondary); line-height: 1.5; } .note { margin: 0; font-size: 10px; color: var(--text-secondary); line-height: 1.5; } + .verdict { margin: 0; font-size: 12px; color: var(--text-primary); line-height: 1.5; } + .note-inline { font-size: 10px; color: var(--text-secondary); } .error { margin: 0; font-size: 11px; line-height: 1.5; color: var(--severity-error, #ff5555); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 437fe36..28a9e7d 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -272,6 +272,14 @@ export interface ImportResult { syslog: number; timeDetected: number; levelDetected: number; + // Lines whose host and application were read out of an RFC 3164 body. + hostDetected: number; + // What recognised each line, and what the file turned out to be. 'mixed' + // when no single shape holds a clear majority; detectedMode is the format + // to read it as, when the shape has one. + byShape: Record; + detected: string; + detectedMode: ImportMode | ''; // Lines that did not fit the declared format, and continuation lines folded // into the record above them. unmatched: number; @@ -287,7 +295,11 @@ export interface ImportPreview { // How a file should be read. 'auto' guesses and reports what it guessed; the // others are declared, which is what makes JSON lines, access logs and stack // traces readable — detection sees none of them. -export type ImportMode = 'auto' | 'syslog' | 'json' | 'access' | 'logfmt' | 'custom'; +export type ImportMode = + | 'auto' | 'syslog' | 'json' | 'access' | 'logfmt' | 'custom' + // Shapes automatic detection reads on its own, which can also be declared: + // a name the dialog reports has to be a name the reader can choose. + | 'bsd' | 'klog' | 'logcat' | 'apache' | 'epoch'; export interface ImportFormat { mode: ImportMode; // Field names for the json and logfmt modes. Empty means the usual diff --git a/frontend/src/lib/i18n/de.json b/frontend/src/lib/i18n/de.json index df8b9f0..d964a47 100644 --- a/frontend/src/lib/i18n/de.json +++ b/frontend/src/lib/i18n/de.json @@ -73,6 +73,16 @@ "linesSampled": "Zeilen als Stichprobe", "syslogLines": "mit Syslog-Priorität", "plainLines": "reiner Text", + "hostDetected": "mit Host und Anwendung", + "detected": "Erkannt:", + "adopted": "(automatisch ausgewählt)", + "shape_bsd": "Syslog ohne Priorität, wie in eine Datei geschrieben", + "shape_klog": "Kubernetes klog", + "shape_logcat": "Android logcat", + "shape_apache": "Apache-Fehlerlog", + "shape_epoch": "Epoch-Zeitstempel (Squid und ähnliche)", + "shape_plain": "Klartext mit Zeitstempel oder Level", + "shape_mixed": "Mehrere Formate in einer Datei", "inferred": "Von {total} Klartextzeilen wurde bei {time} ein Zeitstempel und bei {level} ein Level erkannt. Der Rest behält die Vorgabe.", "persist": "Auch in die Datenbank speichern", "persistHint": "Standardmäßig aus: Importierte Zeilen erscheinen dann im Verlauf neben empfangenem Verkehr.", @@ -93,6 +103,18 @@ "hint_access": "Die Formate Common und Combined. Der Zeitstempel ist der in Klammern, und der Statuscode ist der Schweregrad: 5xx ein Fehler, 4xx eine Warnung.", "hint_logfmt": "Zeilen aus Schlüssel=Wert-Paaren. Paare, die keine Felder sind, bleiben in der Nachricht.", "hint_custom": "Ein regulärer Ausdruck, auf jede Zeile angewendet.", + "group_syslog": "Syslog", + "group_web": "Webserver", + "group_structured": "Strukturiert", + "group_platform": "Plattformen", + "group_custom": "Eigenes", + "format_clef": "JSON, Serilog kompakt (@t, @l, @m)", + "hint_bsd": "Zeitstempel, Host und Tag, ohne Priorität — was Syslog-Daemons in eine Datei schreiben. Eine Zeile ohne diese Form gehört zur Zeile darüber.", + "hint_apache": "Apaches Fehlerlog: Kopf in Klammern mit dem Jahr zuletzt, Level in [Modul:Level], pid in eigener Klammer.", + "hint_klog": "Kubernetes-Komponenten. Der Schweregrad ist der erste Buchstabe, das Datum hat kein Jahr, und die Quelldatei steht dort, wo sonst ein Anwendungsname stünde.", + "hint_logcat": "Android, in beiden Formen: die threadtime-Zeilen, die adb in eine Datei schreibt, und die kurzen „E/Tag( 1234):“-Zeilen.", + "hint_epoch": "Ein Unix-Zeitstempel auf die Millisekunde am Zeilenanfang und sonst nichts, was nach Datum aussieht — so schreiben Squid und mehrere Proxys.", + "hint_clef": "Serilogs kompaktes Ereignisformat mit den Feldern @t, @l und @m.", "fieldsHint": "Ein leeres Feld probiert die üblichen Namen.", "pattern": "Muster", "patternHint": "Gelesen werden die Gruppen time, level, host, app und msg. Zum Beispiel: ^(?P