From b850966ce7e76b0633558b8a54f69ada9d9169fa Mon Sep 17 00:00:00 2001 From: Wasabules <39313803+Wasabules@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:18:44 +0200 Subject: [PATCH 1/4] fix(import): read the host and tag of a syslog line that has no priority (#50) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The priority exists only on the wire. A captured file routinely has none — rsyslog's default on-disk format is a timestamp, a host and a tag — and the importer read all of it as free text: the host and the tag stayed inside the message, and the Hostname and App columns came back empty. A file you cannot filter by host or application is most of the way back to a wall of text. It also produced the report that opened #50. Anonymous mode saw the hostname still sitting at the front of the message, rewrote it to a stand-in, and the result read as though the importer had prefixed every line with "example-01.invalid". So after a timestamp is recognised, "HOST TAG[PID]: MSG" is read out of what follows, and the tag comes off through the wire parser's own extractor rather than a second definition of what a tag looks like. The year and the zone stay the format panel's, not the wire parser's clock-relative guess. The first token is the only judgement call, and it is guarded: a severity word is never a hostname, and the tag must be a real "something:" token, so "21:42:40 WARN queue: depth 812" and "INFO worker pool started" keep their message intact. Six such lines are pinned in a test. In syslog mode a priority-less line was treated as the tail of the one above it, which folded a whole rsyslog file into a handful of messages. It is now read as what it is. --- internal/importer/bsdbody_test.go | 151 ++++++++++++++++++++++++++++++ internal/importer/format.go | 28 +++++- internal/importer/importer.go | 8 ++ internal/importer/plain.go | 47 ++++++++++ internal/importer/plain_test.go | 17 +++- internal/syslog/parser.go | 10 ++ 6 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 internal/importer/bsdbody_test.go diff --git a/internal/importer/bsdbody_test.go b/internal/importer/bsdbody_test.go new file mode 100644 index 0000000..3557943 --- /dev/null +++ b/internal/importer/bsdbody_test.go @@ -0,0 +1,151 @@ +package importer + +import ( + "strings" + "testing" + "time" + + "SyslogStudio/internal/models" +) + +// The file format rsyslog writes by default (#50). +// +// The priority exists only on the wire, so a captured file has a timestamp, a +// host and a tag and nothing that announces itself as syslog. Read as free +// text, the host and the tag stayed inside the message and the Hostname and +// App columns were empty — which is exactly what makes such a file impossible +// to filter, and what the reporter saw. + +func TestAuto_ReadsTheHostAndTagOfAPriorityLessSyslogLine(t *testing.T) { + // The reporter's own line, verbatim. + const content = "Sep 18 08:05:13 nbb-ad-01.vms.nbb.nod Microsoft-Windows-Security-Auditing[756]: An account was successfully logged on.\n" + + "Sep 18 08:05:32 nbb-ad-01.vms.nbb.nod Microsoft-Windows-GroupPolicy[1668]: Completed periodic policy processing\n" + + res, msgs := readFormat(t, content, models.ImportFormat{Mode: models.ImportAuto}) + + if res.HostDetected != 2 || res.TimeDetected != 2 { + t.Fatalf("host=%d time=%d, want 2 and 2", res.HostDetected, res.TimeDetected) + } + if msgs[0].Hostname != "nbb-ad-01.vms.nbb.nod" { + t.Errorf("Hostname = %q", msgs[0].Hostname) + } + if msgs[0].AppName != "Microsoft-Windows-Security-Auditing" || msgs[0].ProcID != "756" { + t.Errorf("app/pid = %q / %q", msgs[0].AppName, msgs[0].ProcID) + } + // The message must no longer carry the host and the tag: that text in front + // of it is what the reporter read as a prefix added by the importer. + if msgs[0].Message != "An account was successfully logged on." { + t.Errorf("Message = %q", msgs[0].Message) + } + if strings.Contains(msgs[0].Message, "nbb-ad-01") { + t.Error("the hostname is still inside the message") + } +} + +// rsyslog's other stock template, and the one syslog-ng writes: the same body +// behind an ISO 8601 timestamp. +func TestAuto_ReadsTheSameBodyBehindAnISOTimestamp(t *testing.T) { + const content = "2026-09-18T08:05:13.123456+02:00 web-1 sshd[4242]: Accepted publickey for deploy\n" + _, msgs := readFormat(t, content, models.ImportFormat{Mode: models.ImportAuto}) + + if msgs[0].Hostname != "web-1" || msgs[0].AppName != "sshd" || msgs[0].ProcID != "4242" { + t.Fatalf("got %q / %q / %q", msgs[0].Hostname, msgs[0].AppName, msgs[0].ProcID) + } + if msgs[0].Message != "Accepted publickey for deploy" { + t.Errorf("Message = %q", msgs[0].Message) + } +} + +// A tag with no pid is the more common half of RFC 3164. +func TestAuto_ReadsATagWithoutAProcessID(t *testing.T) { + const content = "Mar 17 21:44:00 db-2 postgres: checkpoint complete\n" + _, msgs := readFormat(t, content, models.ImportFormat{Mode: models.ImportAuto}) + + if msgs[0].Hostname != "db-2" || msgs[0].AppName != "postgres" { + t.Fatalf("got %q / %q", msgs[0].Hostname, msgs[0].AppName) + } + if msgs[0].ProcID != "" { + t.Errorf("ProcID = %q, want empty", msgs[0].ProcID) + } +} + +// The guard that keeps this from eating ordinary application logs. Each of +// these lines must come back with no hostname at all. +func TestAuto_DoesNotInventAHostOnAnApplicationLog(t *testing.T) { + lines := []struct{ name, line string }{ + {"no tag at all", "2026-03-17 21:42:01 INFO worker pool started with 16 threads"}, + {"a level then a colon", "2026-03-17 21:42:40 WARN queue: depth 812 above soft limit"}, + {"a level in brackets", "2026-03-17 21:42:10 [ERROR] connection refused to db-2"}, + {"a sentence with a colon", "2026-03-17 21:43:00 something happened here: it failed"}, + {"a java logger line", "2026-03-17 21:42:10,123 [http-nio-8080-exec-3] ERROR c.e.Service - boom"}, + {"a url in the message", "2026-03-17 21:43:05 fetching https://example.com/api failed"}, + } + for _, tt := range lines { + t.Run(tt.name, func(t *testing.T) { + _, msgs := readFormat(t, tt.line+"\n", models.ImportFormat{Mode: models.ImportAuto}) + if msgs[0].Hostname != "" { + t.Errorf("invented hostname %q from %q", msgs[0].Hostname, tt.line) + } + if msgs[0].AppName != "" { + t.Errorf("invented app %q from %q", msgs[0].AppName, tt.line) + } + }) + } +} + +// Nothing about this may go through the wire parser's year resolution, which +// answers to the clock rather than to the format panel. +func TestAuto_APriorityLessLineStillHonoursTheYearAndZone(t *testing.T) { + const content = "Nov 3 02:14:09 mail-1 postfix/smtpd[3121]: connect from unknown\n" + _, msgs := readFormat(t, content, models.ImportFormat{ + Mode: models.ImportAuto, Year: 2019, Timezone: "UTC", + }) + + got := msgs[0].Timestamp.UTC() + if got.Year() != 2019 { + t.Errorf("year = %d, want the one the panel was given", got.Year()) + } + if got.Format("01-02 15:04:05") != "11-03 02:14:09" { + t.Errorf("timestamp = %s", got.Format(time.RFC3339)) + } + if msgs[0].Hostname != "mail-1" || msgs[0].AppName != "postfix/smtpd" { + t.Errorf("host/app = %q / %q", msgs[0].Hostname, msgs[0].AppName) + } +} + +// In syslog mode a priority-less line used to be treated as the tail of the +// one above it, so a whole rsyslog file folded into a handful of messages. +func TestSyslogMode_ReadsAFileThatHasNoPrioritiesAtAll(t *testing.T) { + const content = "Sep 18 08:05:13 web-1 sshd[1]: first\n" + + "Sep 18 08:05:14 web-1 sshd[2]: second\n" + + "Sep 18 08:05:15 web-1 sshd[3]: third\n" + + res, msgs := readFormat(t, content, models.ImportFormat{ + Mode: models.ImportSyslog, JoinContinuations: true, + }) + + if res.Imported != 3 { + t.Fatalf("Imported = %d, want 3 — each line is a message of its own", res.Imported) + } + if res.Joined != 0 { + t.Errorf("Joined = %d, want 0", res.Joined) + } + if msgs[2].Message != "third" || msgs[2].ProcID != "3" { + t.Errorf("third message = %q (pid %q)", msgs[2].Message, msgs[2].ProcID) + } +} + +// A line with a priority keeps going through the wire parser untouched, so +// this change cannot have moved what a real capture reads as. +func TestSyslogMode_StillPrefersARealPriority(t *testing.T) { + const content = "<131>1 2026-03-17T21:42:10Z vpn-gw-01 ipsec 4242 - - tunnel torn down\n" + res, msgs := readFormat(t, content, models.ImportFormat{Mode: models.ImportSyslog}) + + if res.Syslog != 1 || res.HostDetected != 0 { + t.Fatalf("Syslog = %d, HostDetected = %d, want 1 and 0 — a priority is read, not guessed", + res.Syslog, res.HostDetected) + } + if msgs[0].SeverityLabel != "Error" || msgs[0].Hostname != "vpn-gw-01" { + t.Errorf("got %q / %q", msgs[0].SeverityLabel, msgs[0].Hostname) + } +} diff --git a/internal/importer/format.go b/internal/importer/format.go index af11ff1..abfc150 100644 --- a/internal/importer/format.go +++ b/internal/importer/format.go @@ -123,6 +123,7 @@ type record struct { syslog bool hasTime bool hasLevel bool + hasHost bool } func newParser(f models.ImportFormat) (*parser, error) { @@ -193,6 +194,12 @@ func (p *parser) parseAuto(line, file string) record { return record{msg: syslog.Parse([]byte(line), file, "file"), start: true, syslog: true} } + return p.parsePlain(line, file) +} + +// parsePlain reads a line that carries no priority: what it says about itself +// is read, and the rest is left alone. +func (p *parser) parsePlain(line, file string) record { d := Detect(line, p.year, p.loc) msg := base(d.Rest, line, file) r := record{msg: msg} @@ -205,6 +212,13 @@ func (p *parser) parseAuto(line, file string) record { r.msg.SeverityLabel = models.SeverityToLabel(d.Severity) r.hasLevel = true } + // An RFC 3164 body: the host, and then the tag, which the wire parser's own + // extractor takes off so a file and the wire agree on what a tag is. + if d.HasHost { + r.msg.Hostname = d.Host + syslog.ExtractTag(&r.msg) + r.hasHost = true + } // A line that said something about itself began a record. One that said // nothing did not — which is what lets a stack trace attach to the line // above it without a file of plain sentences collapsing into one message. @@ -223,11 +237,15 @@ func (p *parser) parseSyslog(line, file string) record { if isSyslogLine(line) { return record{msg: msg, start: true, syslog: true} } - // No priority: the parser's fallback stands, and the line did not start a - // record — in a syslog file, a line without a PRI is the tail of the one - // before it far more often than it is a message of its own. - msg.RawMessage = line - return record{msg: msg} + // No priority. The priority exists only on the wire, so a captured file + // routinely has none — rsyslog's default on-disk format is a timestamp, a + // host and a tag. Read it as such; a line that has none of that is the tail + // of the one before it. + r := p.parsePlain(line, file) + if !r.hasTime && !r.hasHost { + r.start = false + } + return r } // --- JSON -------------------------------------------------------------------- diff --git a/internal/importer/importer.go b/internal/importer/importer.go index e269b0f..9b48eb3 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -73,6 +73,11 @@ type Result struct { // rather than being told a number and left to trust it. TimeDetected int `json:"timeDetected"` LevelDetected int `json:"levelDetected"` + // HostDetected counts lines whose host and application were read out of an + // RFC 3164 body — the shape rsyslog writes to disk. Reported because those + // lines fill the Hostname and App columns, which is the difference between + // a file you can filter and a wall of text. + HostDetected int `json:"hostDetected"` // Unmatched counts lines that did not fit the declared format. Reported // rather than hidden: a format that matches nothing is a format chosen // wrongly, and the number says so before the import is confirmed. @@ -189,6 +194,9 @@ func Read(opts Options, emit func(models.SyslogMessage) bool) (Result, error) { if held.hasLevel { res.LevelDetected++ } + if held.hasHost { + res.HostDetected++ + } return emit(held.msg) } diff --git a/internal/importer/plain.go b/internal/importer/plain.go index 4d573fc..ab83c30 100644 --- a/internal/importer/plain.go +++ b/internal/importer/plain.go @@ -93,12 +93,51 @@ var timeLayouts = []string{ "Jan 2 15:04:05", } +// syslogBody matches what follows the timestamp in an RFC 3164 line: a +// hostname, then a tag that ends in a colon. +// +// This is the shape rsyslog writes to disk by default, and the one an operator +// is most likely to have in a file — the priority only exists on the wire, so a +// captured file has a timestamp, a host and a tag, and nothing that announces +// itself as syslog. Read as free text (#50), the host and the tag stay inside +// the message and the Hostname and App columns are empty, which is exactly what +// makes such a file useless to filter. +// +// The tag requirement is what keeps this from firing on an ordinary +// application log: "worker pool started with 16 threads" has no +// "something:" token in second position. +var syslogBody = regexp.MustCompile( + `^([A-Za-z0-9][A-Za-z0-9._:-]{0,253})[ \t]+([^\s:\[]{1,48}(?:\[[0-9]{1,10}\])?:)(?:[ \t]|$)`) + +// splitBSDBody reads "HOST TAG[PID]: MSG" out of what follows a timestamp. +// +// The only judgement call is the first token. A level word is never a hostname, +// and a line like "21:42:10 WARN queue: depth 812" would otherwise be filed +// under a host called WARN — so the severity vocabulary is excluded outright. +// Everything else the regular expression settles. +func splitBSDBody(rest string) (host, body string, ok bool) { + m := syslogBody.FindStringSubmatch(rest) + if m == nil { + return "", rest, false + } + if _, isLevel := severityWords[strings.ToUpper(m[1])]; isLevel { + return "", rest, false + } + // The body starts at the tag, which ExtractTag then takes off — one + // definition of what a tag is, shared with the wire parser. + return m[1], strings.TrimLeft(rest[len(m[1]):], " \t"), true +} + // Detection is what a plain line was willing to say about itself. type Detection struct { Timestamp time.Time HasTime bool Severity models.Severity HasLevel bool + // Host is the hostname an RFC 3164 body carries in front of its tag, when + // the line turned out to have that shape. + Host string + HasHost bool // Rest is the line with a recognised leading timestamp removed, which is // what belongs in the message column. The severity word is left in place: // it is part of what the line says, and deleting it would make the import @@ -175,6 +214,14 @@ func Detect(line string, year int, loc *time.Location) Detection { if t, rest, ok := detectTime(line, year, loc); ok { d.Timestamp, d.Rest, d.HasTime = t, rest, true + + // Only after a timestamp. "host tag: message" with nothing in front of + // it is far more often a sentence with a colon in it than a syslog + // line, and the cost of being wrong is a message filed under an + // invented host. + if host, body, ok := splitBSDBody(d.Rest); ok { + d.Host, d.Rest, d.HasHost = host, body, true + } } if sev, ok := detectSeverity(d.Rest); ok { d.Severity, d.HasLevel = sev, true diff --git a/internal/importer/plain_test.go b/internal/importer/plain_test.go index bf3c223..50754d2 100644 --- a/internal/importer/plain_test.go +++ b/internal/importer/plain_test.go @@ -21,7 +21,10 @@ func TestDetect_Timestamps(t *testing.T) { {"space separated", "2026-03-17 21:42:10 tunnel down", "2026-03-17T21:42:10Z", "tunnel down"}, {"bracketed", "[2026-03-17 21:42:10] tunnel down", "2026-03-17T21:42:10Z", "tunnel down"}, {"apache", `10.0.0.1 - - [17/Mar/2026:21:42:10 +0000] "GET / HTTP/1.1"`, "", ""}, - {"BSD, no year", "Mar 17 21:42:10 vpn-gw-01 ipsec: down", "2026-03-17T21:42:10Z", "vpn-gw-01 ipsec: down"}, + // The host is taken off with the timestamp now: this is an RFC 3164 + // body, and leaving "vpn-gw-01 ipsec:" inside the message is what #50 + // reported as a prefix the importer had added. + {"BSD, no year", "Mar 17 21:42:10 vpn-gw-01 ipsec: down", "2026-03-17T21:42:10Z", "ipsec: down"}, {"no timestamp at all", "something happened", "", "something happened"}, } @@ -54,6 +57,18 @@ func TestDetect_Timestamps(t *testing.T) { // A date in the middle of a sentence is data, not the moment the line was // written. Reading it as the line's own time would reorder the file around a // coincidence. +// The host of an RFC 3164 body is read out of the line, not left in the text. +func TestDetect_SplitsTheHostOffAnRFC3164Body(t *testing.T) { + got := Detect("Mar 17 21:42:10 vpn-gw-01 ipsec[42]: down", 2026, utc) + + if !got.HasHost || got.Host != "vpn-gw-01" { + t.Fatalf("Host = %q (HasHost=%v), want vpn-gw-01", got.Host, got.HasHost) + } + if got.Rest != "ipsec[42]: down" { + t.Errorf("Rest = %q, want the tag and the message", got.Rest) + } +} + func TestDetect_IgnoresATimestampThatIsNotAtTheFront(t *testing.T) { got := Detect("service restarted at 2026-01-01 00:00:00 by operator", 2026, utc) if got.HasTime { diff --git a/internal/syslog/parser.go b/internal/syslog/parser.go index 3b606ee..465ebdf 100644 --- a/internal/syslog/parser.go +++ b/internal/syslog/parser.go @@ -349,6 +349,16 @@ func parseRFC3164(remainder string, msg *models.SyslogMessage) { // extractAppFromMsg tries to extract app name and PID from the message TAG field. // Common format: "appname[pid]: message" or "appname: message" +// ExtractTag pulls an RFC 3164 TAG — "app[pid]:" or "app:" — off the front of +// a message, filling AppName and ProcID. +// +// Exported for the importer. A file written by rsyslog carries the same TAG as +// a line off the wire does, and a second definition of what a tag looks like +// would drift from this one the first time either changed. +func ExtractTag(msg *models.SyslogMessage) { + extractAppFromMsg(msg) +} + func extractAppFromMsg(msg *models.SyslogMessage) { if msg.Message == "" { return From fd81e21398e372976c062374a9320f3a10320eaa Mon Sep 17 00:00:00 2001 From: Wasabules <39313803+Wasabules@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:29:55 +0200 Subject: [PATCH 2/4] feat(import): widen what automatic detection recognises, and pin it with a corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Detection read a timestamp at the front of a line and a severity word standing on its own. Measured against the formats people actually have on disk, that left whole families unread: the level glued to the date (klog), three integers between the timestamp and the level (logcat), a bracketed header with the year last (Apache), an epoch and nothing else (Squid), a timestamp in the middle of the line (access logs), and the two structured formats — JSON and logfmt — whose fields it never looked at because they are not where it looks. Automatic detection is now a chain, from the formats that announce themselves to the ones that must be inferred. Order is precedence, and every branch before the last is anchored and specific, so a line reaches the general detector only when nothing recognised it outright. That is what lets the panel be wide without the wide part being a guess: a JSON object, a klog line and an access line each look like exactly one thing. - shapes.go holds the four that a wider timestamp list could never reach - JSON, access and logfmt are delegated to the readers already written for them, each behind a test of what the line must start with - slash dates (Go, nginx), numeric zones (zap), a zone after a space (Serilog), day-and-month without a year (logcat), Ruby's letter-and-comma prefix, and the three-letter level spellings INF, WRN, DBG, VRB and FTL - the preview now counts the lines whose host and application were read The corpus test is the specification: 27 lines, one per format, taken from what each tool really writes — and four that must stay unrecognised, because a wide recogniser earns its width only by staying silent on a line it does not understand. An invented hostname is worse than an unparsed line: the text still shows while the field quietly lies. tools/sample-logs/auto-formats.txt is the same corpus as a file to import. --- app_import_test.go | 17 +- frontend/src/components/ImportDialog.svelte | 3 + frontend/src/lib/api.ts | 2 + frontend/src/lib/i18n/de.json | 1 + frontend/src/lib/i18n/en.json | 1 + frontend/src/lib/i18n/es.json | 1 + frontend/src/lib/i18n/fr.json | 1 + frontend/src/lib/i18n/it.json | 1 + frontend/src/lib/i18n/ja.json | 1 + frontend/src/lib/i18n/pt.json | 1 + frontend/src/lib/i18n/zh.json | 1 + frontend/wailsjs/go/models.ts | 2 + internal/importer/corpus_test.go | 246 ++++++++++++++++++++ internal/importer/format.go | 30 ++- internal/importer/plain.go | 54 ++++- internal/importer/shapes.go | 213 +++++++++++++++++ tools/sample-logs/README.md | 8 + tools/sample-logs/auto-formats.txt | 24 ++ 18 files changed, 596 insertions(+), 11 deletions(-) create mode 100644 internal/importer/corpus_test.go create mode 100644 internal/importer/shapes.go create mode 100644 tools/sample-logs/auto-formats.txt diff --git a/app_import_test.go b/app_import_test.go index 25f15b3..c8cbd36 100644 --- a/app_import_test.go +++ b/app_import_test.go @@ -198,30 +198,35 @@ func TestImportLogFile_DoesNotRememberAFormatThatFailed(t *testing.T) { } // A declared format has to reach the importer, or the dialog is decoration. +// +// Detection reads a JSON line on its own now, so the contrast has to come from +// something it cannot know: a level under a field name of the application's +// own choosing. func TestPreviewLogFile_AppliesTheDeclaredFormat(t *testing.T) { app, _ := importApp(t) p := filepath.Join(t.TempDir(), "app.json.log") - line := `{"time":"2026-03-17T21:42:10Z","level":"error","msg":"connection refused"}` + "\n" + line := `{"time":"2026-03-17T21:42:10Z","prio":"error","msg":"connection refused"}` + "\n" if err := os.WriteFile(p, []byte(line), 0o600); err != nil { t.Fatal(err) } - // Detection reads nothing out of a JSON line, which is the gap the modes - // exist to close. auto, err := app.PreviewLogFile(p, models.ImportFormat{Mode: models.ImportAuto}) if err != nil { t.Fatal(err) } if auto.Result.LevelDetected != 0 { - t.Errorf("detection claims to read a level out of JSON: %d", auto.Result.LevelDetected) + t.Errorf("a level was read from a field nothing could have guessed: %d", + auto.Result.LevelDetected) } - declared, err := app.PreviewLogFile(p, models.ImportFormat{Mode: models.ImportJSON}) + declared, err := app.PreviewLogFile(p, models.ImportFormat{ + Mode: models.ImportJSON, JSONLevel: "prio", + }) if err != nil { t.Fatal(err) } if declared.Result.LevelDetected != 1 || declared.Messages[0].SeverityLabel != "Error" { - t.Errorf("declaring JSON changed nothing: level=%d severity=%q", + t.Errorf("naming the field changed nothing: level=%d severity=%q", declared.Result.LevelDetected, declared.Messages[0].SeverityLabel) } } diff --git a/frontend/src/components/ImportDialog.svelte b/frontend/src/components/ImportDialog.svelte index 9249c6f..2de2436 100644 --- a/frontend/src/components/ImportDialog.svelte +++ b/frontend/src/components/ImportDialog.svelte @@ -169,6 +169,9 @@
{preview.result.imported.toLocaleString()}{$_('import.linesSampled')}
{preview.result.syslog.toLocaleString()}{$_('import.syslogLines')}
{plain.toLocaleString()}{$_('import.plainLines')}
+ {#if preview.result.hostDetected > 0} +
{preview.result.hostDetected.toLocaleString()}{$_('import.hostDetected')}
+ {/if} {#if preview.result.unmatched > 0}
{preview.result.unmatched.toLocaleString()}{$_('import.unmatched')}
{/if} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 437fe36..b903d84 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -272,6 +272,8 @@ export interface ImportResult { syslog: number; timeDetected: number; levelDetected: number; + // Lines whose host and application were read out of an RFC 3164 body. + hostDetected: number; // Lines that did not fit the declared format, and continuation lines folded // into the record above them. unmatched: number; diff --git a/frontend/src/lib/i18n/de.json b/frontend/src/lib/i18n/de.json index df8b9f0..71f4330 100644 --- a/frontend/src/lib/i18n/de.json +++ b/frontend/src/lib/i18n/de.json @@ -73,6 +73,7 @@ "linesSampled": "Zeilen als Stichprobe", "syslogLines": "mit Syslog-Priorität", "plainLines": "reiner Text", + "hostDetected": "mit Host und Anwendung", "inferred": "Von {total} Klartextzeilen wurde bei {time} ein Zeitstempel und bei {level} ein Level erkannt. Der Rest behält die Vorgabe.", "persist": "Auch in die Datenbank speichern", "persistHint": "Standardmäßig aus: Importierte Zeilen erscheinen dann im Verlauf neben empfangenem Verkehr.", diff --git a/frontend/src/lib/i18n/en.json b/frontend/src/lib/i18n/en.json index d2fca86..549b40c 100644 --- a/frontend/src/lib/i18n/en.json +++ b/frontend/src/lib/i18n/en.json @@ -73,6 +73,7 @@ "linesSampled": "lines sampled", "syslogLines": "with a syslog priority", "plainLines": "plain text", + "hostDetected": "with host and app", "inferred": "Out of {total} plain lines, a timestamp was recognised on {time} and a level on {level}. The rest keep the default.", "persist": "Also save to the database", "persistHint": "Off by default: imported lines would then appear in History next to received traffic.", diff --git a/frontend/src/lib/i18n/es.json b/frontend/src/lib/i18n/es.json index c629dfa..4247a6e 100644 --- a/frontend/src/lib/i18n/es.json +++ b/frontend/src/lib/i18n/es.json @@ -73,6 +73,7 @@ "linesSampled": "líneas muestreadas", "syslogLines": "con prioridad syslog", "plainLines": "texto sin formato", + "hostDetected": "con host y aplicación", "inferred": "De {total} líneas de texto sin formato, se reconoció una marca de tiempo en {time} y un nivel en {level}. El resto conserva el valor predeterminado.", "persist": "Guardar también en la base de datos", "persistHint": "Desactivado por defecto: las líneas importadas aparecerían en el historial junto al tráfico recibido.", diff --git a/frontend/src/lib/i18n/fr.json b/frontend/src/lib/i18n/fr.json index b233ec6..167b970 100644 --- a/frontend/src/lib/i18n/fr.json +++ b/frontend/src/lib/i18n/fr.json @@ -73,6 +73,7 @@ "linesSampled": "lignes échantillonnées", "syslogLines": "avec une priorité syslog", "plainLines": "texte brut", + "hostDetected": "avec hôte et application", "inferred": "Sur {total} lignes de texte brut, un horodatage a été reconnu sur {time} et un niveau sur {level}. Les autres conservent la valeur par défaut.", "persist": "Enregistrer aussi dans la base de données", "persistHint": "Désactivé par défaut : les lignes importées apparaîtraient alors dans l’historique à côté du trafic reçu.", diff --git a/frontend/src/lib/i18n/it.json b/frontend/src/lib/i18n/it.json index 181509e..db8fb85 100644 --- a/frontend/src/lib/i18n/it.json +++ b/frontend/src/lib/i18n/it.json @@ -73,6 +73,7 @@ "linesSampled": "righe campionate", "syslogLines": "con priorità syslog", "plainLines": "testo semplice", + "hostDetected": "con host e applicazione", "inferred": "Su {total} righe di testo semplice, è stata riconosciuta una data/ora in {time} e un livello in {level}. Le altre mantengono il valore predefinito.", "persist": "Salva anche nel database", "persistHint": "Disattivato per impostazione predefinita: le righe importate comparirebbero nella cronologia accanto al traffico ricevuto.", diff --git a/frontend/src/lib/i18n/ja.json b/frontend/src/lib/i18n/ja.json index c46729e..055899e 100644 --- a/frontend/src/lib/i18n/ja.json +++ b/frontend/src/lib/i18n/ja.json @@ -73,6 +73,7 @@ "linesSampled": "サンプルした行数", "syslogLines": "syslog 優先度あり", "plainLines": "プレーンテキスト", + "hostDetected": "ホストとアプリあり", "inferred": "プレーンテキスト {total} 行のうち、タイムスタンプを {time} 行、レベルを {level} 行で認識しました。残りは既定値のままです。", "persist": "データベースにも保存する", "persistHint": "既定ではオフです。オンにすると、インポートした行が受信トラフィックと並んで履歴に表示されます。", diff --git a/frontend/src/lib/i18n/pt.json b/frontend/src/lib/i18n/pt.json index ca0687d..a7103cb 100644 --- a/frontend/src/lib/i18n/pt.json +++ b/frontend/src/lib/i18n/pt.json @@ -73,6 +73,7 @@ "linesSampled": "linhas amostradas", "syslogLines": "com prioridade syslog", "plainLines": "texto simples", + "hostDetected": "com anfitrião e aplicação", "inferred": "De {total} linhas de texto simples, foi reconhecida uma data/hora em {time} e um nível em {level}. As restantes mantêm o valor predefinido.", "persist": "Guardar também na base de dados", "persistHint": "Desativado por predefinição: as linhas importadas passariam a aparecer no histórico ao lado do tráfego recebido.", diff --git a/frontend/src/lib/i18n/zh.json b/frontend/src/lib/i18n/zh.json index 2772519..c118c1f 100644 --- a/frontend/src/lib/i18n/zh.json +++ b/frontend/src/lib/i18n/zh.json @@ -73,6 +73,7 @@ "linesSampled": "采样行数", "syslogLines": "含 syslog 优先级", "plainLines": "纯文本", + "hostDetected": "含主机和应用", "inferred": "在 {total} 行纯文本中，识别出 {time} 行的时间戳和 {level} 行的级别。其余保持默认值。", "persist": "同时保存到数据库", "persistHint": "默认关闭：开启后导入的行会与接收到的流量一起出现在历史记录中。", diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index 34fb5c7..39bb7ab 100644 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -9,6 +9,7 @@ export namespace importer { syslog: number; timeDetected: number; levelDetected: number; + hostDetected: number; unmatched: number; joined: number; stopped: boolean; @@ -28,6 +29,7 @@ export namespace importer { this.syslog = source["syslog"]; this.timeDetected = source["timeDetected"]; this.levelDetected = source["levelDetected"]; + this.hostDetected = source["hostDetected"]; this.unmatched = source["unmatched"]; this.joined = source["joined"]; this.stopped = source["stopped"]; diff --git a/internal/importer/corpus_test.go b/internal/importer/corpus_test.go new file mode 100644 index 0000000..6a978b9 --- /dev/null +++ b/internal/importer/corpus_test.go @@ -0,0 +1,246 @@ +package importer + +import ( + "testing" + "time" + + "SyslogStudio/internal/models" +) + +// What automatic detection is expected to make of the formats people actually +// have on disk. +// +// One line per format, taken from what the tool that writes it really emits. +// The table is the specification: a format that is not in it is a format +// nobody has checked, and a format in it that changes behaviour fails here +// before it reaches anyone's screen. +// +// The negative half matters as much. A wide recogniser earns its width only if +// it stays silent on a line it does not understand — an invented hostname or a +// severity read out of an ordinary sentence is worse than an unparsed line, +// because the text still shows while the field quietly lies. + +type shapeCase struct { + name string + line string + // The moment, formatted with `layout` (the date alone when the format + // carries no year of its own and the panel supplies it). + when string + layout string + sev string + host string + app string + pid string + msg string + // local marks a line whose timestamp carries no zone AND goes through the + // wire parser, which reads it in the machine's zone — the same instant the + // receiver would record for that line (#24). The format panel's timezone + // governs everything this package parses itself. + local bool +} + +const defaultLayout = "2006-01-02 15:04:05" + +func TestAuto_Corpus(t *testing.T) { + cases := []shapeCase{ + // --- the syslog family --------------------------------------------- + { + name: "RFC 5424 with a priority", + line: `<134>1 2026-03-17T21:42:10Z web-1 sshd 4242 - - Accepted publickey for deploy`, + when: "2026-03-17 21:42:10", sev: "Info", host: "web-1", app: "sshd", pid: "4242", + msg: "Accepted publickey for deploy", + }, + { + name: "RFC 3164 with a priority", + line: `<38>Mar 17 21:42:10 web-1 sshd[4242]: Failed password for root`, + when: "03-17 21:42:10", layout: "01-02 15:04:05", local: true, + sev: "Info", host: "web-1", app: "sshd", pid: "4242", + msg: "Failed password for root", + }, + { + name: "rsyslog traditional file format, no priority", + line: `Sep 18 08:05:13 nbb-ad-01.vms.nbb.nod Microsoft-Windows-Security-Auditing[756]: An account was logged on.`, + when: "2026-09-18 08:05:13", sev: "Notice", + host: "nbb-ad-01.vms.nbb.nod", app: "Microsoft-Windows-Security-Auditing", pid: "756", + msg: "An account was logged on.", + }, + { + name: "rsyslog FileFormat, ISO stamp", + line: `2026-09-18T08:05:13.123456+02:00 web-1 sshd[4242]: Accepted publickey for deploy`, + when: "2026-09-18 06:05:13", sev: "Notice", host: "web-1", app: "sshd", pid: "4242", + msg: "Accepted publickey for deploy", + }, + { + name: "systemd, journalctl short", + line: `Sep 18 08:05:13 web-1 systemd[1]: Started Daily apt upgrade.`, + when: "2026-09-18 08:05:13", sev: "Notice", host: "web-1", app: "systemd", pid: "1", + msg: "Started Daily apt upgrade.", + }, + + // --- application loggers ------------------------------------------- + { + name: "Go standard logger", + line: `2026/03/17 21:42:10 starting worker pool`, + when: "2026-03-17 21:42:10", sev: "Notice", msg: "starting worker pool", + }, + { + name: "nginx error log", + line: `2026/03/17 21:42:10 [error] 1234#0: *1 connect() failed while connecting to upstream`, + when: "2026-03-17 21:42:10", sev: "Error", + }, + { + name: "Apache error log", + line: `[Mon Mar 17 21:42:10.123456 2026] [core:error] [pid 1234] AH00037: Symbolic link not allowed`, + when: "2026-03-17 21:42:10", sev: "Error", pid: "1234", + msg: "AH00037: Symbolic link not allowed", + }, + { + name: "Apache/nginx access log", + line: `198.51.100.7 - - [17/Mar/2026:21:42:10 +0000] "GET /health HTTP/1.1" 500 172 "-" "curl/8.5.0"`, + when: "2026-03-17 21:42:10", sev: "Error", host: "198.51.100.7", + }, + { + name: "Kubernetes klog", + line: `I0317 21:42:10.123456 1234 controller.go:212] Starting workers`, + when: "2026-03-17 21:42:10", sev: "Info", app: "controller.go", pid: "1234", + msg: "Starting workers", + }, + { + name: "Android logcat, threadtime", + line: `03-17 21:42:10.123 1234 5678 E ActivityManager: ANR in com.example.app`, + when: "2026-03-17 21:42:10", sev: "Error", app: "ActivityManager", pid: "1234", + msg: "ANR in com.example.app", + }, + { + name: "Squid, epoch at the front", + line: `1774388530.123 123 198.51.100.7 TCP_MISS/200 4021 GET http://example.com/`, + when: "2026", layout: "2006", sev: "Notice", + }, + { + name: "Java, logback", + line: `2026-03-17 21:42:10,123 [http-nio-8080-exec-3] ERROR c.e.Service - boom`, + when: "2026-03-17 21:42:10", sev: "Error", + }, + { + name: "Python, logging module", + line: `2026-03-17 21:42:10,123 - mymodule - ERROR - connection lost`, + when: "2026-03-17 21:42:10", sev: "Error", + }, + { + name: "Serilog, three-letter level", + line: `2026-03-17 21:42:10.123 +02:00 [INF] Now listening on http://localhost:5000`, + when: "2026-03-17 19:42:10", sev: "Info", + }, + { + name: "MySQL error log", + line: `2026-03-17T21:42:10.123456Z 0 [Warning] [MY-010068] CA certificate is self signed`, + when: "2026-03-17 21:42:10", sev: "Warning", + }, + { + name: "zap, console encoder", + line: "2026-03-17T21:42:10.123+0200\tINFO\tpkg/file.go:42\tserver started", + when: "2026-03-17 19:42:10", sev: "Info", + }, + { + name: "Docker, --timestamps", + line: `2026-03-17T21:42:10.123456789Z Starting container`, + when: "2026-03-17 21:42:10", sev: "Notice", msg: "Starting container", + }, + { + name: "Ruby and Rails logger", + line: `I, [2026-03-17T21:42:10.123456 #1234] INFO -- : Completed 200 OK`, + when: "2026-03-17 21:42:10", sev: "Info", + }, + { + name: ".NET console, level then category", + line: `info: Microsoft.Hosting.Lifetime[0] Now listening`, + when: "", sev: "Info", + }, + + // --- structured ---------------------------------------------------- + { + name: "JSON, numeric level and epoch milliseconds", + line: `{"level":50,"time":1774388532123,"msg":"pool exhausted","service":"api"}`, + when: "2026", layout: "2006", sev: "Error", app: "api", msg: "pool exhausted", + }, + { + name: "JSON, string level and RFC 3339", + line: `{"time":"2026-03-17T21:42:10Z","level":"warn","msg":"disk almost full","host":"web-1"}`, + when: "2026-03-17 21:42:10", sev: "Warning", host: "web-1", msg: "disk almost full", + }, + { + name: "logfmt", + line: `ts=2026-03-17T21:42:10Z level=error msg="connection refused" err="dial tcp"`, + when: "2026-03-17 21:42:10", sev: "Error", + }, + + // --- what must NOT be recognised ----------------------------------- + { + name: "a sentence with a colon in it", + line: `2026-03-17 21:42:10 something happened here: it failed`, + when: "2026-03-17 21:42:10", sev: "Notice", + msg: "something happened here: it failed", + }, + { + name: "a level word followed by a colon", + line: `2026-03-17 21:42:10 ERROR: something failed`, + when: "2026-03-17 21:42:10", sev: "Error", msg: "ERROR: something failed", + }, + { + name: "a stack trace line", + line: "\tat java.base/java.lang.Thread.run(Thread.java:840)", + when: "", sev: "Notice", + }, + { + name: "a line that says nothing", + line: `just some text with no shape at all`, + when: "", sev: "Notice", msg: "just some text with no shape at all", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, msgs := readFormat(t, c.line+"\n", models.ImportFormat{Mode: models.ImportAuto}) + if len(msgs) != 1 { + t.Fatalf("got %d messages, want 1", len(msgs)) + } + m := msgs[0] + + layout := c.layout + if layout == "" { + layout = defaultLayout + } + if c.when == "" { + // No timestamp in the line: the message keeps the moment it was + // read, which is today rather than the line's own year. + if m.Timestamp.UTC().Format("2006-01-02") != time.Now().UTC().Format("2006-01-02") { + t.Errorf("read a timestamp where the line has none: %s", m.Timestamp) + } + } else { + at := m.Timestamp.UTC() + if c.local { + at = m.Timestamp.In(time.Local) + } + if got := at.Format(layout); got != c.when { + t.Errorf("time = %s, want %s", got, c.when) + } + } + + if m.SeverityLabel != c.sev { + t.Errorf("severity = %q, want %q", m.SeverityLabel, c.sev) + } + if m.Hostname != c.host { + t.Errorf("hostname = %q, want %q", m.Hostname, c.host) + } + if m.AppName != c.app { + t.Errorf("app = %q, want %q", m.AppName, c.app) + } + if m.ProcID != c.pid { + t.Errorf("pid = %q, want %q", m.ProcID, c.pid) + } + if c.msg != "" && m.Message != c.msg { + t.Errorf("message = %q, want %q", m.Message, c.msg) + } + }) + } +} diff --git a/internal/importer/format.go b/internal/importer/format.go index abfc150..78fd341 100644 --- a/internal/importer/format.go +++ b/internal/importer/format.go @@ -189,11 +189,39 @@ func (p *parser) applyLevel(msg *models.SyslogMessage, raw string, r *record) { // --- automatic --------------------------------------------------------------- +// parseAuto reads a line without being told what it is. +// +// A chain, from the formats that announce themselves to the ones that have to +// be inferred. Order is precedence, and every branch before the last is +// anchored and specific: a line reaches the general detector only when nothing +// has recognised it outright. That is what lets the panel be wide without the +// wide part being a guess — a JSON object, a klog line and an access line each +// look like exactly one thing. func (p *parser) parseAuto(line, file string) record { + // A priority is not a guess at all. if isSyslogLine(line) { return record{msg: syslog.Parse([]byte(line), file, "file"), start: true, syslog: true} } - + // One JSON object per line, the shape most applications write today. + if looksLikeJSON(line) { + if r := p.parseJSON(line, file); r.start { + return r + } + } + // klog, logcat, Apache's error log, an epoch at the front: shapes that a + // wider timestamp list could never reach. + if r, ok := p.parseShape(line, file); ok { + return r + } + // An access line carries its timestamp in the middle, where nothing looking + // at the front of a line will ever find it. + if accessPattern.MatchString(line) { + return p.parseAccess(line, file) + } + // key=value, when the FIRST pair is one of the known fields. + if looksLikeLogfmt(line) { + return p.parseLogfmt(line, file) + } return p.parsePlain(line, file) } diff --git a/internal/importer/plain.go b/internal/importer/plain.go index ab83c30..8830d49 100644 --- a/internal/importer/plain.go +++ b/internal/importer/plain.go @@ -43,6 +43,16 @@ var severityWords = map[string]models.Severity{ "NOTICE": models.SevNotice, "INFO": models.SevInformational, "INFORMATION": models.SevInformational, "DEBUG": models.SevDebug, "TRACE": models.SevDebug, "FINE": models.SevDebug, + + // The three-letter spellings Serilog, NLog and several Go loggers write. + // "ERR" is above; the rest are here. Each still has to stand on its own to + // match, so a word ending in "inf" or "dbg" is not a level. + "FTL": models.SevCritical, + "WRN": models.SevWarning, + "INF": models.SevInformational, + "DBG": models.SevDebug, "VRB": models.SevDebug, + // java.util.logging, whose FINER and FINEST sit below FINE. + "FINER": models.SevDebug, "FINEST": models.SevDebug, } // severityPattern finds a severity word standing on its own. @@ -52,8 +62,9 @@ var severityWords = map[string]models.Severity{ // also inside "TERRAFORM", "REFERRAL" and every other word with those three // letters in the middle. A word that is part of a longer word is not a level. var severityPattern = regexp.MustCompile( - `(?i)(?:^|[\s\[\(<|:=,/])(EMERG(?:ENCY)?|PANIC|ALERT|CRIT(?:ICAL)?|FATAL|ERR(?:OR)?|SEVERE|` + - `WARN(?:ING)?|NOTICE|INFO(?:RMATION)?|DEBUG|TRACE|FINE)(?:$|[\s\]\)>|:=,/\"])`) + `(?i)(?:^|[\s\[\(<|:=,/])(EMERG(?:ENCY)?|PANIC|ALERT|CRIT(?:ICAL)?|FATAL|FTL|ERR(?:OR)?|SEVERE|` + + `WARN(?:ING)?|WRN|NOTICE|INFO(?:RMATION)?|INF|DEBUG|DBG|TRACE|VRB|FINEST|FINER|FINE)` + + `(?:$|[\s\]\)>|:=,/\"])`) // timeToken matches the timestamp shapes people actually write, anchored to the // start of the line. @@ -66,7 +77,15 @@ var severityPattern = regexp.MustCompile( var timeToken = regexp.MustCompile( `^(?:` + // ISO 8601 / RFC 3339, with or without a fraction and a zone. - `\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?` + + `\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?: ?(?:Z|[+-]\d{2}:?\d{2}))?` + + `|` + + // Slashes instead of dashes: Go's standard logger, and nginx's error + // log, which between them account for a great many files. + `\d{4}/\d{2}/\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?` + + `|` + + // Day and month only, as Android and several embedded loggers write. + // The year comes from the format panel. + `\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[.,]\d+)?` + `|` + // Apache and nginx access logs. `\d{2}/[A-Za-z]{3}/\d{4}:\d{2}:\d{2}:\d{2}(?: [+-]\d{4})?` + @@ -78,11 +97,15 @@ var timeToken = regexp.MustCompile( // timeLayouts are tried against a token that already looks like a timestamp. var timeLayouts = []string{ "2006-01-02T15:04:05.999999999Z07:00", + "2006-01-02T15:04:05.999999999Z0700", "2006-01-02T15:04:05Z07:00", + "2006-01-02T15:04:05Z0700", "2006-01-02T15:04:05.999999999", "2006-01-02T15:04:05", "2006-01-02 15:04:05.999999999 -07:00", + "2006-01-02 15:04:05.999999999 -0700", "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02 15:04:05.999999999Z0700", "2006-01-02 15:04:05.999999999", "2006-01-02 15:04:05Z07:00", "2006-01-02 15:04:05 -07:00", @@ -91,6 +114,16 @@ var timeLayouts = []string{ "02/Jan/2006:15:04:05", "Jan _2 15:04:05", "Jan 2 15:04:05", + // Go's standard logger and nginx. + "2006/01/02 15:04:05.999999999", + "2006/01/02 15:04:05", + "2006/01/02T15:04:05", + // Apache's error log, where the year comes last. + "Mon Jan _2 15:04:05.999999999 2006", + "Mon Jan 2 15:04:05 2006", + // Android's logcat and Kubernetes' klog, neither of which writes a year. + "01-02 15:04:05.999999999", + "0102 15:04:05.999999999", } // syslogBody matches what follows the timestamp in an RFC 3164 line: a @@ -153,6 +186,13 @@ type Detection struct { func detectTime(line string, year int, loc *time.Location) (time.Time, string, bool) { trimmed := strings.TrimLeft(line, " ") + // Ruby's Logger, and Rails with it, writes the severity letter and a comma + // before the bracket: "I, [2026-03-17T21:42:10.123456 #1234]". + if len(trimmed) > 3 && trimmed[1] == ',' && trimmed[2] == ' ' && + trimmed[0] >= 'A' && trimmed[0] <= 'Z' { + trimmed = trimmed[3:] + } + // Many formats bracket the stamp: [2026-03-17 21:42:10]. bracketed := strings.HasPrefix(trimmed, "[") if bracketed { @@ -181,7 +221,13 @@ func detectTime(line string, year int, loc *time.Location) (time.Time, string, b } rest := strings.TrimSpace(trimmed[len(token):]) if bracketed { - rest = strings.TrimSpace(strings.TrimPrefix(rest, "]")) + // Close the bracket the stamp opened, along with whatever else was + // inside it: Ruby writes "[