fix(import): read a priority-less syslog line, widen detection, and name the file

app_import_test.go

@@ -198,30 +198,35 @@ func TestImportLogFile_DoesNotRememberAFormatThatFailed(t *testing.T) {
 }
 
 // A declared format has to reach the importer, or the dialog is decoration.
+//
+// Detection reads a JSON line on its own now, so the contrast has to come from
+// something it cannot know: a level under a field name of the application's
+// own choosing.
 func TestPreviewLogFile_AppliesTheDeclaredFormat(t *testing.T) {
 	app, _ := importApp(t)
 	p := filepath.Join(t.TempDir(), "app.json.log")
-	line := `{"time":"2026-03-17T21:42:10Z","level":"error","msg":"connection refused"}` + "\n"
+	line := `{"time":"2026-03-17T21:42:10Z","prio":"error","msg":"connection refused"}` + "\n"
 	if err := os.WriteFile(p, []byte(line), 0o600); err != nil {
 		t.Fatal(err)
 	}
 
-	// Detection reads nothing out of a JSON line, which is the gap the modes
-	// exist to close.
 	auto, err := app.PreviewLogFile(p, models.ImportFormat{Mode: models.ImportAuto})
 	if err != nil {
 		t.Fatal(err)
 	}
 	if auto.Result.LevelDetected != 0 {
-		t.Errorf("detection claims to read a level out of JSON: %d", auto.Result.LevelDetected)
+		t.Errorf("a level was read from a field nothing could have guessed: %d",
+			auto.Result.LevelDetected)
 	}
 
-	declared, err := app.PreviewLogFile(p, models.ImportFormat{Mode: models.ImportJSON})
+	declared, err := app.PreviewLogFile(p, models.ImportFormat{
+		Mode: models.ImportJSON, JSONLevel: "prio",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
 	if declared.Result.LevelDetected != 1 || declared.Messages[0].SeverityLabel != "Error" {
-		t.Errorf("declaring JSON changed nothing: level=%d severity=%q",
+		t.Errorf("naming the field changed nothing: level=%d severity=%q",
 			declared.Result.LevelDetected, declared.Messages[0].SeverityLabel)
 	}
 }

frontend/screenshots/fixtures.js

@@ -484,6 +484,15 @@ export const IMPORT_PREVIEW = {
     // fixture has no stack trace in it.
     unmatched: IMPORT_SCRIPT.filter((e) => e.silent).length,
     joined: 0,
+    // Three lines carry a priority and eleven do not: no single shape holds
+    // the file, which is exactly what "mixed" is for.
+    byShape: {
+      syslog: IMPORT_SCRIPT.filter((e) => e.pri).length,
+      plain: IMPORT_SCRIPT.filter((e) => !e.pri && !e.silent).length,
+      none: IMPORT_SCRIPT.filter((e) => e.silent).length,
+    },
+    detected: 'mixed',
+    detectedMode: '',
     stopped: false,
     bySeverity: IMPORT_SCRIPT.reduce((acc, e) => {
       const label = SEVERITY_LABELS[e.sev];

frontend/src/components/ImportDialog.svelte

@@ -36,7 +36,80 @@
 
     const dispatch = createEventDispatcher<{ imported: ImportResult }>();
 
-    const MODES: ImportMode[] = ['auto', 'syslog', 'json', 'access', 'logfmt', 'custom'];
+    /**
+     * The formats offered, by category.
+     *
+     * The list used to name the six parsing ENGINES, while the recogniser knew
+     * twenty formats — so a file could be reported as "Kubernetes klog" and
+     * then not be in the list at all. What a reader thinks in is a format's
+     * name, not an engine's, so the entries are names; several map to the same
+     * engine, and a couple carry field names with them.
+     *
+     * An entry's label is the same string the verdict uses, so the file cannot
+     * be called one thing above and another below.
+     */
+    type FormatEntry = {
+        id: string;
+        group: string;
+        label: string;
+        hint: string;
+        mode: ImportMode;
+        preset?: Partial<ImportFormat>;
+    };
+
+    const FORMATS: FormatEntry[] = [
+        { id: 'auto', group: '', label: 'import.format_auto', hint: 'import.hint_auto', mode: 'auto' },
+
+        { id: 'syslog', group: 'import.group_syslog', label: 'import.format_syslog', hint: 'import.hint_syslog', mode: 'syslog' },
+        { id: 'bsd', group: 'import.group_syslog', label: 'import.shape_bsd', hint: 'import.hint_bsd', mode: 'bsd' },
+
+        { id: 'access', group: 'import.group_web', label: 'import.format_access', hint: 'import.hint_access', mode: 'access' },
+        { id: 'apache', group: 'import.group_web', label: 'import.shape_apache', hint: 'import.hint_apache', mode: 'apache' },
+
+        { id: 'json', group: 'import.group_structured', label: 'import.format_json', hint: 'import.hint_json', mode: 'json' },
+        {
+            id: 'clef', group: 'import.group_structured', label: 'import.format_clef',
+            hint: 'import.hint_clef', mode: 'json',
+            preset: { jsonTime: '@t', jsonLevel: '@l', jsonMessage: '@m' },
+        },
+        { id: 'logfmt', group: 'import.group_structured', label: 'import.format_logfmt', hint: 'import.hint_logfmt', mode: 'logfmt' },
+
+        { id: 'klog', group: 'import.group_platform', label: 'import.shape_klog', hint: 'import.hint_klog', mode: 'klog' },
+        { id: 'logcat', group: 'import.group_platform', label: 'import.shape_logcat', hint: 'import.hint_logcat', mode: 'logcat' },
+        { id: 'epoch', group: 'import.group_platform', label: 'import.shape_epoch', hint: 'import.hint_epoch', mode: 'epoch' },
+
+        { id: 'custom', group: 'import.group_custom', label: 'import.format_custom', hint: 'import.hint_custom', mode: 'custom' },
+    ];
+
+    // Grouped for the markup, in the order above.
+    const GROUPED: { key: string; entries: FormatEntry[] }[] = FORMATS.reduce((acc, entry) => {
+        const last = acc[acc.length - 1];
+        if (last && last.key === entry.group) last.entries.push(entry);
+        else acc.push({ key: entry.group, entries: [entry] });
+        return acc;
+    }, [] as { key: string; entries: FormatEntry[] }[]);
+
+    /** Which entry a format IS, so the list shows what is in force. */
+    function entryFor(f: ImportFormat): FormatEntry {
+        const sameMode = FORMATS.filter(e => e.mode === f.mode);
+        const withPreset = sameMode.find(e => e.preset
+            && Object.entries(e.preset).every(([k, v]) =>
+                (f as unknown as Record<string, unknown>)[k] === v));
+        return withPreset ?? sameMode.find(e => !e.preset) ?? FORMATS[0];
+    }
+
+    /** Choosing an entry gives exactly that entry's configuration. */
+    function pickFormat(id: string) {
+        const entry = FORMATS.find(e => e.id === id) ?? FORMATS[0];
+        adopted = false;
+        format = {
+            ...format,
+            mode: entry.mode,
+            jsonTime: '', jsonLevel: '', jsonMessage: '', jsonHost: '', jsonApp: '',
+            ...(entry.preset ?? {}),
+        };
+        refresh();
+    }
 
     let path = '';
     let preview: ImportPreview | null = null;
@@ -46,6 +119,27 @@
     let formatError = '';
     let format: ImportFormat = { mode: 'auto', joinContinuations: true, skipUnmatched: false };
 
+    // Set when the format was chosen by the detector rather than by hand, so
+    // the panel can say which of the two happened.
+    let adopted = false;
+
+    // The name of a shape the detector can report. The four that are also
+    // modes reuse the mode's own label, so the dropdown and the verdict cannot
+    // disagree about what "JSON" is called.
+    const SHAPE_LABEL: Record<string, string> = {
+        syslog: 'import.format_syslog',
+        json: 'import.format_json',
+        access: 'import.format_access',
+        logfmt: 'import.format_logfmt',
+        bsd: 'import.shape_bsd',
+        klog: 'import.shape_klog',
+        logcat: 'import.shape_logcat',
+        apache: 'import.shape_apache',
+        epoch: 'import.shape_epoch',
+        plain: 'import.shape_plain',
+        mixed: 'import.shape_mixed',
+    };
+
     let debounce: ReturnType<typeof setTimeout> | undefined;
     // Which request is the current one. Two previews can be in flight on a
     // large file, and the one that finishes last is not necessarily the one
@@ -61,6 +155,7 @@
         busy = false;
         showFormat = false;
         formatError = '';
+        adopted = false;
     }
 
     export function close() {
@@ -79,6 +174,7 @@
             format = await getImportFormat();
             preview = await previewLogFile(path, format);
             formatError = '';
+            await adoptDetected();
         } catch (e: any) {
             toastError(e?.message || String(e));
             path = '';
@@ -88,6 +184,27 @@
         }
     }
 
+    /**
+     * Take the detector's word for it, when it has one.
+     *
+     * Only from 'auto', and only once: a format chosen by hand is a decision,
+     * and overruling it would be the application arguing with the operator.
+     * The sample is then read again through the chosen mode, so what is on
+     * screen is what that mode actually produces rather than what the chain
+     * produced on the way to naming it.
+     */
+    async function adoptDetected() {
+        const mode = preview?.result?.detectedMode;
+        if (!mode || format.mode !== 'auto') return;
+        format = { ...format, mode };
+        adopted = true;
+        try {
+            preview = await previewLogFile(path, format);
+        } catch (e: any) {
+            formatError = e?.message || String(e);
+        }
+    }
+
     /**
      * Re-read the sample with the format as it now stands.
      *
@@ -141,7 +258,8 @@
     $: severities = preview
         ? Object.entries(preview.result.bySeverity).sort((a, b) => b[1] - a[1])
         : [];
-    $: modeLabel = $_(`import.format_${format.mode}`);
+    $: current = entryFor(format);
+    $: modeLabel = $_(current.label);
 </script>
 
 <svelte:window on:keydown={onKey} />
@@ -169,6 +287,9 @@
                         <div><b>{preview.result.imported.toLocaleString()}</b><span>{$_('import.linesSampled')}</span></div>
                         <div><b>{preview.result.syslog.toLocaleString()}</b><span>{$_('import.syslogLines')}</span></div>
                         <div><b>{plain.toLocaleString()}</b><span>{$_('import.plainLines')}</span></div>
+                        {#if preview.result.hostDetected > 0}
+                            <div><b>{preview.result.hostDetected.toLocaleString()}</b><span>{$_('import.hostDetected')}</span></div>
+                        {/if}
                         {#if preview.result.unmatched > 0}
                             <div class="warn"><b>{preview.result.unmatched.toLocaleString()}</b><span>{$_('import.unmatched')}</span></div>
                         {/if}
@@ -177,6 +298,14 @@
                         {/if}
                     </div>
 
+                    {#if preview.result.detected}
+                        <p class="verdict">
+                            <b>{$_('import.detected')}</b>
+                            {$_(SHAPE_LABEL[preview.result.detected] ?? 'import.shape_plain')}{#if adopted}
+                                <span class="note-inline">{$_('import.adopted')}</span>{/if}
+                        </p>
+                    {/if}
+
                     {#if plain > 0 && format.mode === 'auto'}
                         <p class="note">
                             {$_('import.inferred', {
@@ -223,13 +352,23 @@
                         <div class="format">
                             <label class="row">
                                 <span class="row-label">{$_('import.format')}</span>
-                                <select bind:value={format.mode} on:change={refresh}>
-                                    {#each MODES as mode}
-                                        <option value={mode}>{$_(`import.format_${mode}`)}</option>
+                                <select value={current.id} on:change={e => pickFormat(e.currentTarget.value)}>
+                                    {#each GROUPED as group}
+                                        {#if group.key === ''}
+                                            {#each group.entries as entry (entry.id)}
+                                                <option value={entry.id}>{$_(entry.label)}</option>
+                                            {/each}
+                                        {:else}
+                                            <optgroup label={$_(group.key)}>
+                                                {#each group.entries as entry (entry.id)}
+                                                    <option value={entry.id}>{$_(entry.label)}</option>
+                                                {/each}
+                                            </optgroup>
+                                        {/if}
                                     {/each}
                                 </select>
                             </label>
-                            <p class="note">{$_(`import.hint_${format.mode}`)}</p>
+                            <p class="note">{$_(current.hint)}</p>
 
                             {#if format.mode === 'json' || format.mode === 'logfmt'}
                                 <p class="note">{$_('import.fieldsHint')}</p>
@@ -343,6 +482,8 @@
     }
     .lead { margin: 0; font-size: 12px; color: var(--text-secondary); line-height: 1.5; }
     .note { margin: 0; font-size: 10px; color: var(--text-secondary); line-height: 1.5; }
+    .verdict { margin: 0; font-size: 12px; color: var(--text-primary); line-height: 1.5; }
+    .note-inline { font-size: 10px; color: var(--text-secondary); }
     .error {
         margin: 0; font-size: 11px; line-height: 1.5;
         color: var(--severity-error, #ff5555);

frontend/src/lib/api.ts

@@ -272,6 +272,14 @@ export interface ImportResult {
     syslog: number;
     timeDetected: number;
     levelDetected: number;
+    // Lines whose host and application were read out of an RFC 3164 body.
+    hostDetected: number;
+    // What recognised each line, and what the file turned out to be. 'mixed'
+    // when no single shape holds a clear majority; detectedMode is the format
+    // to read it as, when the shape has one.
+    byShape: Record<string, number>;
+    detected: string;
+    detectedMode: ImportMode | '';
     // Lines that did not fit the declared format, and continuation lines folded
     // into the record above them.
     unmatched: number;
@@ -287,7 +295,11 @@ export interface ImportPreview {
 // How a file should be read. 'auto' guesses and reports what it guessed; the
 // others are declared, which is what makes JSON lines, access logs and stack
 // traces readable — detection sees none of them.
-export type ImportMode = 'auto' | 'syslog' | 'json' | 'access' | 'logfmt' | 'custom';
+export type ImportMode =
+    | 'auto' | 'syslog' | 'json' | 'access' | 'logfmt' | 'custom'
+    // Shapes automatic detection reads on its own, which can also be declared:
+    // a name the dialog reports has to be a name the reader can choose.
+    | 'bsd' | 'klog' | 'logcat' | 'apache' | 'epoch';
 export interface ImportFormat {
     mode: ImportMode;
     // Field names for the json and logfmt modes. Empty means the usual

frontend/src/lib/i18n/de.json

@@ -73,6 +73,16 @@
     "linesSampled": "Zeilen als Stichprobe",
     "syslogLines": "mit Syslog-Priorität",
     "plainLines": "reiner Text",
+    "hostDetected": "mit Host und Anwendung",
+    "detected": "Erkannt:",
+    "adopted": "(automatisch ausgewählt)",
+    "shape_bsd": "Syslog ohne Priorität, wie in eine Datei geschrieben",
+    "shape_klog": "Kubernetes klog",
+    "shape_logcat": "Android logcat",
+    "shape_apache": "Apache-Fehlerlog",
+    "shape_epoch": "Epoch-Zeitstempel (Squid und ähnliche)",
+    "shape_plain": "Klartext mit Zeitstempel oder Level",
+    "shape_mixed": "Mehrere Formate in einer Datei",
     "inferred": "Von {total} Klartextzeilen wurde bei {time} ein Zeitstempel und bei {level} ein Level erkannt. Der Rest behält die Vorgabe.",
     "persist": "Auch in die Datenbank speichern",
     "persistHint": "Standardmäßig aus: Importierte Zeilen erscheinen dann im Verlauf neben empfangenem Verkehr.",
@@ -93,6 +103,18 @@
     "hint_access": "Die Formate Common und Combined. Der Zeitstempel ist der in Klammern, und der Statuscode ist der Schweregrad: 5xx ein Fehler, 4xx eine Warnung.",
     "hint_logfmt": "Zeilen aus Schlüssel=Wert-Paaren. Paare, die keine Felder sind, bleiben in der Nachricht.",
     "hint_custom": "Ein regulärer Ausdruck, auf jede Zeile angewendet.",
+    "group_syslog": "Syslog",
+    "group_web": "Webserver",
+    "group_structured": "Strukturiert",
+    "group_platform": "Plattformen",
+    "group_custom": "Eigenes",
+    "format_clef": "JSON, Serilog kompakt (@t, @l, @m)",
+    "hint_bsd": "Zeitstempel, Host und Tag, ohne Priorität — was Syslog-Daemons in eine Datei schreiben. Eine Zeile ohne diese Form gehört zur Zeile darüber.",
+    "hint_apache": "Apaches Fehlerlog: Kopf in Klammern mit dem Jahr zuletzt, Level in [Modul:Level], pid in eigener Klammer.",
+    "hint_klog": "Kubernetes-Komponenten. Der Schweregrad ist der erste Buchstabe, das Datum hat kein Jahr, und die Quelldatei steht dort, wo sonst ein Anwendungsname stünde.",
+    "hint_logcat": "Android, in beiden Formen: die threadtime-Zeilen, die adb in eine Datei schreibt, und die kurzen „E/Tag( 1234):“-Zeilen.",
+    "hint_epoch": "Ein Unix-Zeitstempel auf die Millisekunde am Zeilenanfang und sonst nichts, was nach Datum aussieht — so schreiben Squid und mehrere Proxys.",
+    "hint_clef": "Serilogs kompaktes Ereignisformat mit den Feldern @t, @l und @m.",
     "fieldsHint": "Ein leeres Feld probiert die üblichen Namen.",
     "pattern": "Muster",
     "patternHint": "Gelesen werden die Gruppen time, level, host, app und msg. Zum Beispiel: ^(?P<time>\\S+) (?P<level>\\w+) (?P<msg>.*)$",

frontend/src/lib/i18n/en.json

@@ -73,6 +73,16 @@
     "linesSampled": "lines sampled",
     "syslogLines": "with a syslog priority",
     "plainLines": "plain text",
+    "hostDetected": "with host and app",
+    "detected": "Detected:",
+    "adopted": "(selected automatically)",
+    "shape_bsd": "Syslog without a priority, as written to a file",
+    "shape_klog": "Kubernetes klog",
+    "shape_logcat": "Android logcat",
+    "shape_apache": "Apache error log",
+    "shape_epoch": "Epoch timestamps (Squid and similar)",
+    "shape_plain": "Plain text with a timestamp or a level",
+    "shape_mixed": "Several formats in one file",
     "inferred": "Out of {total} plain lines, a timestamp was recognised on {time} and a level on {level}. The rest keep the default.",
     "persist": "Also save to the database",
     "persistHint": "Off by default: imported lines would then appear in History next to received traffic.",
@@ -93,6 +103,18 @@
     "hint_access": "The Common and Combined formats. The timestamp is the one in brackets, and the status code is the severity: 5xx an error, 4xx a warning.",
     "hint_logfmt": "Lines of key=value pairs. Pairs that are not fields are kept in the message.",
     "hint_custom": "A regular expression run against each line.",
+    "group_syslog": "Syslog",
+    "group_web": "Web servers",
+    "group_structured": "Structured",
+    "group_platform": "Platforms",
+    "group_custom": "Your own",
+    "format_clef": "JSON, Serilog compact (@t, @l, @m)",
+    "hint_bsd": "A timestamp, a host and a tag, with no priority — what syslog daemons write to a file. A line without that shape belongs to the one above it.",
+    "hint_apache": "Apache's error log: a bracketed header with the year last, the level in [module:level], and the pid in its own bracket.",
+    "hint_klog": "Kubernetes components. The severity is the leading letter, the date carries no year, and the source file stands where an application name would.",
+    "hint_logcat": "Android, in either form: the threadtime lines adb writes to a file, and the brief “E/Tag( 1234):” lines it prints.",
+    "hint_epoch": "A Unix timestamp to the millisecond at the front of the line and nothing else a reader would recognise as a date, as Squid and several proxies write.",
+    "hint_clef": "Serilog's compact event format, where the fields are @t, @l and @m.",
     "fieldsHint": "Leave a box empty to try the usual names.",
     "pattern": "Pattern",
     "patternHint": "The groups read are time, level, host, app and msg. For example: ^(?P<time>\\S+) (?P<level>\\w+) (?P<msg>.*)$",