diff --git a/README.md b/README.md index 83b022b..2025151 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,27 @@ in a ticket without going through a redaction tool first. - **Filter, sort and group** by severity, facility, host, application, source IP or time range - **Explicit timezones** — follow the machine, pin to UTC, or name a zone; the column header says which one it is showing -- **Export** as CSV or plain text +- **Right-click a line** to copy it (message, raw line, JSON, or the cell you aimed at), to + narrow the view to that host, application, severity or the five minutes around it, or to turn + it into an alert rule. In anonymous mode copying yields what is on screen, and the real value + is a separate entry — you cannot paste a real address believing it was masked +- **Columns you arrange** — drag an edge to resize, double-click it to fit the widest value, + drag a heading to move the column, and right-click any heading to add or remove one: facility, + process id, message id, RFC version and received time are all there, hidden until wanted. + Widths, order and choice are remembered +- **Keyboard navigation** — arrows, Page Up/Down, Home/End walk the list, Escape closes the + detail, `/` jumps to the search box. Shift+arrows extend a selection +- **Pick several lines** — click, Ctrl-click, Shift-click, then copy them or export exactly + those. In anonymous mode you copy what is on screen, not what is behind it +- **Freeze the stream** while you read it — nothing is dropped, and the list says how many + arrived and catches up when you release it +- **Saved filters** — name the set of criteria you keep retyping and recall it in one click +- **Drop a log file on the window** to import it, with the same preview as the file picker +- **A detail panel you can widen** — drag the edge between the list and the message, double-click + it to go back to the default +- **Export** as CSV, plain text, NDJSON, syslog (RFC 5424 or RFC 3164, replayable into any + collector — including this one) or a self-contained HTML report for someone who does not have + the application. In anonymous mode the export follows the screen by default, and says so - **Import a log file** already on disk — `.log`, `.txt` or a rotated `.gz`. A captured syslog file is parsed exactly as it would be off the wire; a plain application log has its timestamp and level read out of the text, and a preview says how much was read and how much diff --git a/app.go b/app.go index e865ca2..982c9b1 100644 --- a/app.go +++ b/app.go @@ -891,20 +891,66 @@ func (a *App) SelectCAFile() (string, error) { // the screen it was taken from; an empty or unknown name falls back to the // machine's zone rather than failing the export. func (a *App) ExportLogs(filter models.FilterCriteria, format string, timezone string) (string, error) { - var defaultFilename string - var filters []wailsRuntime.FileFilter + return a.writeMessagesTo(a.server.GetMessages(filter), "syslog_export", format, timezone) +} - if format == "csv" { - defaultFilename = "syslog_export.csv" - filters = []wailsRuntime.FileFilter{ - {DisplayName: "CSV Files (*.csv)", Pattern: "*.csv"}, - } - } else { - defaultFilename = "syslog_export.txt" - filters = []wailsRuntime.FileFilter{ - {DisplayName: "Text Files (*.txt)", Pattern: "*.txt"}, +// ExportSelection writes only the messages whose ids are given. +// +// The same writers and the same dialog as a full export; what differs is +// which messages. Picking a handful of lines out of a stream and handing +// exactly those to someone is a different act from exporting everything a +// filter matched, and doing it by narrowing the filter until only those +// remain is not a thing anyone should have to do. +func (a *App) ExportSelection(ids []string, format string, timezone string) (string, error) { + if len(ids) == 0 { + return "", fmt.Errorf("nothing selected") + } + + wanted := make(map[string]bool, len(ids)) + for _, id := range ids { + wanted[id] = true + } + // Taken from the buffer in buffer order, not in the order they were + // clicked: an export that reordered a log would be a strange thing to hand + // to anyone. + var messages []models.SyslogMessage + for _, msg := range a.server.GetMessages(models.FilterCriteria{}) { + if wanted[msg.ID] { + messages = append(messages, msg) } } + if len(messages) == 0 { + return "", fmt.Errorf("the selected messages are no longer in the buffer") + } + return a.writeMessagesTo(messages, "syslog_selection", format, timezone) +} + +// ExportMessages writes messages the interface hands over, as it has them. +// +// This is how an export can match the screen. Anonymous mode substitutes +// hostnames and addresses for DISPLAY, and that substitution lives in the +// interface — so an export written from the server's own copy contains the +// real values, whatever the screen says. Someone attaching that file to a +// ticket would be publishing exactly what they thought they had masked. +// +// Rather than teaching the backend to redact (a second implementation, whose +// stand-ins would not even match the ones on screen), the interface sends what +// it is showing. +func (a *App) ExportMessages(messages []models.SyslogMessage, format string, timezone string) (string, error) { + if len(messages) == 0 { + return "", fmt.Errorf("nothing to export") + } + return a.writeMessagesTo(messages, "syslog_export", format, timezone) +} + +// writeMessagesTo asks where, then writes there. One place that knows how an +// export is named, filtered and written, for all three ways in. +func (a *App) writeMessagesTo(messages []models.SyslogMessage, base, format, timezone string) (string, error) { + defaultFilename, wanted := exportFile(format, base) + filters := make([]wailsRuntime.FileFilter, 0, len(wanted)) + for _, f := range wanted { + filters = append(filters, wailsRuntime.FileFilter{DisplayName: f.Display, Pattern: f.Pattern}) + } path, err := wailsRuntime.SaveFileDialog(a.ctx, wailsRuntime.SaveDialogOptions{ Title: "Export Logs", @@ -918,15 +964,7 @@ func (a *App) ExportLogs(filter models.FilterCriteria, format string, timezone s return "", nil } - messages := a.server.GetMessages(filter) - loc := resolveLocation(timezone) - if format == "csv" { - err = writeCSV(path, messages, loc) - } else { - err = writeText(path, messages, loc) - } - - if err != nil { + if err := writeExport(path, format, messages, resolveLocation(timezone)); err != nil { return "", fmt.Errorf("failed to write export: %w", err) } return path, nil diff --git a/export_formats.go b/export_formats.go new file mode 100644 index 0000000..6a2999f --- /dev/null +++ b/export_formats.go @@ -0,0 +1,336 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "html" + "os" + "strings" + "time" + + "SyslogStudio/internal/models" +) + +// The formats an export can be written in, beyond the two it started with. +// +// Each one exists for something someone does with the file afterwards. CSV +// goes to a spreadsheet and text goes to a reader; these go to a machine, to +// another collector, and to somebody who does not have this application. +// +// What they share is that they carry the message AS RECEIVED. A relay may +// rewrite a message's origin on the way out — that is what forwarding is for — +// but an export that rewrote what was captured would be an export of something +// that never happened. + +const ( + formatCSV = "csv" + formatText = "txt" + formatNDJSON = "ndjson" + formatRFC5424 = "rfc5424" + formatRFC3164 = "rfc3164" + formatHTML = "html" + maxHTMLPreview = 50000 +) + +// exportFile is the name and the dialog filter a format asks for. +func exportFile(format, base string) (string, []exportFilter) { + switch format { + case formatCSV: + return base + ".csv", []exportFilter{{"CSV Files (*.csv)", "*.csv"}} + case formatNDJSON: + return base + ".ndjson", []exportFilter{{"JSON Lines (*.ndjson, *.jsonl)", "*.ndjson;*.jsonl"}} + case formatRFC5424, formatRFC3164: + return base + ".log", []exportFilter{{"Syslog (*.log)", "*.log"}} + case formatHTML: + return base + ".html", []exportFilter{{"HTML (*.html)", "*.html"}} + default: + return base + ".txt", []exportFilter{{"Text Files (*.txt)", "*.txt"}} + } +} + +type exportFilter struct { + Display string + Pattern string +} + +// writeExport puts the messages on disk in the format asked for. +func writeExport(path, format string, messages []models.SyslogMessage, loc *time.Location) error { + switch format { + case formatCSV: + return writeCSV(path, messages, loc) + case formatNDJSON: + return writeNDJSON(path, messages, loc) + case formatRFC5424: + return writeSyslog(path, messages, loc, true) + case formatRFC3164: + return writeSyslog(path, messages, loc, false) + case formatHTML: + return writeHTML(path, messages, loc) + default: + return writeText(path, messages, loc) + } +} + +// exportRecord is one message as a machine reads it. +// +// Named fields in a fixed order rather than the model itself: the model is +// free to change for the application's own reasons, and a file someone has +// written a script against is not. +type exportRecord struct { + Timestamp string `json:"timestamp"` + ReceivedAt string `json:"receivedAt"` + Severity int `json:"severity"` + SeverityLabel string `json:"severityLabel"` + Facility int `json:"facility"` + FacilityLabel string `json:"facilityLabel"` + Version int `json:"version,omitempty"` + Hostname string `json:"hostname"` + AppName string `json:"appName"` + ProcID string `json:"procID,omitempty"` + MsgID string `json:"msgID,omitempty"` + StructuredData string `json:"structuredData,omitempty"` + Message string `json:"message"` + SourceIP string `json:"sourceIP"` + Protocol string `json:"protocol"` +} + +// writeNDJSON writes one JSON object per line. +// +// Not a JSON array: a file of lines can be read by `jq`, streamed into a bulk +// index, or tailed while it is still being written, and none of those work on +// a document that has to be closed before it parses. +func writeNDJSON(path string, messages []models.SyslogMessage, loc *time.Location) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + w := bufio.NewWriter(f) + defer w.Flush() + + enc := json.NewEncoder(w) + for _, msg := range messages { + record := exportRecord{ + Timestamp: msg.Timestamp.In(loc).Format(time.RFC3339Nano), + ReceivedAt: msg.ReceivedAt.In(loc).Format(time.RFC3339Nano), + Severity: int(msg.Severity), + SeverityLabel: msg.SeverityLabel, + Facility: int(msg.Facility), + FacilityLabel: msg.FacilityLabel, + Version: msg.Version, + Hostname: msg.Hostname, + AppName: msg.AppName, + ProcID: msg.ProcID, + MsgID: msg.MsgID, + StructuredData: msg.StructuredData, + Message: msg.Message, + SourceIP: msg.SourceIP, + Protocol: msg.Protocol, + } + if err := enc.Encode(record); err != nil { + return err + } + } + return w.Flush() +} + +// writeSyslog writes the messages back as protocol lines. +// +// The point is that the file can be replayed: into another collector, into +// this one, into anything that reads syslog. So the priority, the host and the +// application are the ones the message arrived with, and the framing is +// whichever RFC the receiving end understands. +func writeSyslog(path string, messages []models.SyslogMessage, loc *time.Location, rfc5424 bool) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + w := bufio.NewWriter(f) + defer w.Flush() + + for _, msg := range messages { + pri := int(msg.Facility)*8 + int(msg.Severity) + ts := msg.Timestamp + if ts.IsZero() { + ts = msg.ReceivedAt + } + ts = ts.In(loc) + + var line string + if rfc5424 { + sd := strings.TrimSpace(msg.StructuredData) + if sd == "" { + sd = "-" + } + line = fmt.Sprintf("<%d>1 %s %s %s %s %s %s %s", + pri, + ts.Format("2006-01-02T15:04:05.000Z07:00"), + orDash(msg.Hostname), + orDash(msg.AppName), + orDash(msg.ProcID), + orDash(msg.MsgID), + sd, + oneLine(msg.Message), + ) + } else { + // RFC 3164 has no field for a message id, and its timestamp has no + // year: this is the older wire, and writing it means accepting what + // it cannot carry. + tag := msg.AppName + if tag != "" && msg.ProcID != "" { + tag = fmt.Sprintf("%s[%s]", tag, msg.ProcID) + } + if tag != "" { + tag += ": " + } + line = fmt.Sprintf("<%d>%s %s %s%s", + pri, + ts.Format("Jan _2 15:04:05"), + orDash(msg.Hostname), + tag, + oneLine(msg.Message), + ) + } + if _, err := w.WriteString(line + "\n"); err != nil { + return err + } + } + return w.Flush() +} + +func orDash(s string) string { + if strings.TrimSpace(s) == "" { + return "-" + } + return sanitizeExportField(s) +} + +// A syslog line ends at a newline, so a message containing one would become +// two lines — the second of which would parse as something else entirely. +func oneLine(s string) string { + return strings.NewReplacer("\r\n", " ", "\n", " ", "\r", " ").Replace(s) +} + +// A space inside a header field would shift every field after it. +func sanitizeExportField(s string) string { + return strings.NewReplacer(" ", "_", "\r", "", "\n", "").Replace(s) +} + +// The colours the application shows, so a report looks like the screen it came +// from rather than a different reading of the same data. +var htmlSeverityColours = [8]string{ + "#ff0040", "#ff4444", "#ff6644", "#ff8800", + "#ffcc00", "#44aaff", "#66dd66", "#888888", +} + +// writeHTML writes a report for someone who does not have this application. +// +// One file, no assets, no network: it opens from an e-mail attachment on a +// machine that has never heard of SyslogStudio, which is the whole point of +// attaching it. +func writeHTML(path string, messages []models.SyslogMessage, loc *time.Location) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + w := bufio.NewWriter(f) + defer w.Flush() + + counts := map[string]int{} + for _, msg := range messages { + counts[msg.SeverityLabel]++ + } + + var summary strings.Builder + for level := models.SevEmergency; level <= models.SevDebug; level++ { + label := models.SeverityToLabel(level) + if counts[label] == 0 { + continue + } + summary.WriteString(fmt.Sprintf( + `%s %d`, + htmlSeverityColours[level], html.EscapeString(label), counts[label])) + } + + fmt.Fprintf(w, ` + + +Syslog export — %s + +

Syslog export

+
%d messages · written %s
+
%s
+ + + +`, + html.EscapeString(time.Now().In(loc).Format("2006-01-02")), + len(messages), + html.EscapeString(time.Now().In(loc).Format(exportTimeLayout)), + summary.String()) + + shown := messages + if len(shown) > maxHTMLPreview { + shown = shown[:maxHTMLPreview] + } + for _, msg := range shown { + colour := htmlSeverityColours[7] + if int(msg.Severity) >= 0 && int(msg.Severity) < len(htmlSeverityColours) { + colour = htmlSeverityColours[msg.Severity] + } + fmt.Fprintf(w, + ``+ + ``+"\n", + colour, + html.EscapeString(msg.SeverityLabel), + html.EscapeString(msg.Timestamp.In(loc).Format(exportTimeLayout)), + html.EscapeString(msg.SourceIP), + html.EscapeString(msg.Hostname), + html.EscapeString(msg.AppName), + html.EscapeString(msg.Message)) + } + + fmt.Fprint(w, "
SeverityTimestampSourceHostAppMessage
%s%s%s%s%s%s
\n") + if len(messages) > len(shown) { + // Said in the file rather than only in a toast at export time: whoever + // opens this may not be whoever wrote it. + fmt.Fprintf(w, + `
Showing the first %d of %d messages. A browser stops being usable long before the rest would fit; export as CSV or NDJSON for the whole set.
`+"\n", + len(shown), len(messages)) + } + fmt.Fprint(w, "\n") + return w.Flush() +} diff --git a/export_formats_test.go b/export_formats_test.go new file mode 100644 index 0000000..34db92d --- /dev/null +++ b/export_formats_test.go @@ -0,0 +1,231 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "SyslogStudio/internal/importer" + "SyslogStudio/internal/models" +) + +func exportSample() []models.SyslogMessage { + at := time.Date(2026, 3, 17, 21, 42, 10, 0, time.UTC) + return []models.SyslogMessage{ + { + ID: "1", Timestamp: at, ReceivedAt: at, + Severity: models.SevError, SeverityLabel: "Error", + Facility: models.FacLocal0, FacilityLabel: "local0", + Hostname: "vpn-gw-01", AppName: "ipsec", ProcID: "4242", + Message: "IKE_SA rekey failed", SourceIP: "10.0.0.7", Protocol: "TCP", + }, + { + ID: "2", Timestamp: at.Add(time.Second), ReceivedAt: at.Add(time.Second), + Severity: models.SevInformational, SeverityLabel: "Info", + Facility: models.FacUser, FacilityLabel: "user", + Hostname: "web-1", AppName: "sshd", + // A message that would break a line-based format if it were let + // through as it stands. + Message: "Accepted publickey\nfor deploy", SourceIP: "10.0.0.9", Protocol: "UDP", + }, + } +} + +func writeTo(t *testing.T, name, format string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := writeExport(path, format, exportSample(), time.UTC); err != nil { + t.Fatalf("writeExport(%s): %v", format, err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +// --- NDJSON ------------------------------------------------------------------ + +func TestExport_NDJSONIsOneObjectPerLine(t *testing.T) { + out := writeTo(t, "out.ndjson", formatNDJSON) + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + + if len(lines) != 2 { + t.Fatalf("got %d lines, want 2 — a message with a newline in it must not become two records", len(lines)) + } + + var first exportRecord + if err := json.Unmarshal([]byte(lines[0]), &first); err != nil { + t.Fatalf("line 1 is not JSON: %v", err) + } + if first.SeverityLabel != "Error" || first.Hostname != "vpn-gw-01" || first.ProcID != "4242" { + t.Errorf("fields lost: %+v", first) + } + if first.Timestamp != "2026-03-17T21:42:10Z" { + t.Errorf("timestamp = %q, want RFC 3339", first.Timestamp) + } + + var second exportRecord + if err := json.Unmarshal([]byte(lines[1]), &second); err != nil { + t.Fatalf("line 2 is not JSON: %v", err) + } + // The newline survives inside the value, which is the whole reason to use + // JSON rather than another line-based format. + if second.Message != "Accepted publickey\nfor deploy" { + t.Errorf("message = %q, want the newline kept inside the field", second.Message) + } +} + +// --- syslog ------------------------------------------------------------------ + +// The point of writing syslog is that it can be read back. Ours is the parser +// nearest to hand, and if it cannot read what we wrote, nothing else will. +func TestExport_RFC5424RoundTripsThroughTheImporter(t *testing.T) { + path := filepath.Join(t.TempDir(), "replay.log") + if err := writeExport(path, formatRFC5424, exportSample(), time.UTC); err != nil { + t.Fatal(err) + } + + var back []models.SyslogMessage + res, err := importer.Read( + importer.Options{Path: path, Year: 2026, Location: time.UTC, + Format: models.ImportFormat{Mode: models.ImportSyslog}}, + func(m models.SyslogMessage) bool { back = append(back, m); return true }) + if err != nil { + t.Fatalf("reading back: %v", err) + } + + if res.Imported != 2 || res.Syslog != 2 { + t.Fatalf("read back %d messages (%d with a priority), want 2 and 2", res.Imported, res.Syslog) + } + if back[0].SeverityLabel != "Error" || back[0].FacilityLabel != "local0" { + t.Errorf("priority lost: %s / %s", back[0].SeverityLabel, back[0].FacilityLabel) + } + if back[0].Hostname != "vpn-gw-01" || back[0].AppName != "ipsec" || back[0].ProcID != "4242" { + t.Errorf("origin lost: %q / %q / %q", back[0].Hostname, back[0].AppName, back[0].ProcID) + } + if back[0].Message != "IKE_SA rekey failed" { + t.Errorf("message = %q", back[0].Message) + } + if got := back[0].Timestamp.UTC().Format(time.RFC3339); got != "2026-03-17T21:42:10Z" { + t.Errorf("timestamp = %s", got) + } + // The embedded newline had to go somewhere, and a space is the only place + // it can go in a format that ends a record at one. + if strings.Contains(back[1].Message, "\n") || !strings.Contains(back[1].Message, "for deploy") { + t.Errorf("second message = %q", back[1].Message) + } +} + +func TestExport_RFC3164RoundTripsThroughTheImporter(t *testing.T) { + path := filepath.Join(t.TempDir(), "replay-bsd.log") + if err := writeExport(path, formatRFC3164, exportSample(), time.UTC); err != nil { + t.Fatal(err) + } + + var back []models.SyslogMessage + if _, err := importer.Read( + importer.Options{Path: path, Year: 2026, Location: time.UTC, + Format: models.ImportFormat{Mode: models.ImportSyslog}}, + func(m models.SyslogMessage) bool { back = append(back, m); return true }); err != nil { + t.Fatal(err) + } + + if len(back) != 2 { + t.Fatalf("read back %d messages, want 2", len(back)) + } + if back[0].SeverityLabel != "Error" || back[0].Hostname != "vpn-gw-01" { + t.Errorf("got %q / %q", back[0].SeverityLabel, back[0].Hostname) + } + if back[0].AppName != "ipsec" || back[0].ProcID != "4242" { + t.Errorf("tag lost: %q[%q]", back[0].AppName, back[0].ProcID) + } +} + +func TestExport_SyslogFillsEmptyFieldsWithADash(t *testing.T) { + msgs := []models.SyslogMessage{{ + Timestamp: time.Date(2026, 3, 17, 21, 42, 10, 0, time.UTC), + Severity: models.SevNotice, SeverityLabel: "Notice", + Facility: models.FacUser, Message: "bare", + }} + path := filepath.Join(t.TempDir(), "bare.log") + if err := writeExport(path, formatRFC5424, msgs, time.UTC); err != nil { + t.Fatal(err) + } + data, _ := os.ReadFile(path) + line := strings.TrimSpace(string(data)) + + // <13>1