From 48ee934be417dd162e92743a2be5c633bab2ead6 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Wed, 1 Apr 2026 21:30:27 +0800 Subject: [PATCH 01/12] add to --- TODO.md | 27 --------- dd.sql | 11 ++++ src/defs.go | 29 ++++++--- src/host.go | 159 +++++++++++++++++++++++++++++++++++++++++--------- src/sender.go | 152 +++++++++++++++++++++++++++++++++++------------ src/store.go | 145 +++++++++++++++++++++++++++++++++++++++++---- 6 files changed, 411 insertions(+), 112 deletions(-) diff --git a/TODO.md b/TODO.md index 24cc22a..0f0e22c 100644 --- a/TODO.md +++ b/TODO.md @@ -8,20 +8,6 @@ correctness issues, then enhancements. ## P0 — Foundational (blocks most other work) -### 2. Fix `Encode()` to produce the full message header per spec -**File:** `defs.go` `Encode()` -Currently encodes only version through type. Missing: add-to field, size -(uint32), attachment headers (uint8 count + headers). Topic is always encoded -but must be absent when pid is set. Type is always length-prefixed but must be a -single uint8 index when the common-type flag (bit 2) is set. The header hash -(SHA-256 of encoded header) is wrong without these fields, breaking challenge -verification and pid references. - -### 3. Add `AddTo` field to `FMsgHeader` -**File:** `defs.go` struct -Add `AddTo []FMsgAddress` field. Required before any add-to parsing, encoding, -storage, or per-recipient response ordering can work. - ### 4. Add `Attachments` field to `FMsgHeader` **File:** `defs.go` struct Add `Attachments []FMsgAttachmentHeader` to store parsed attachment headers @@ -35,14 +21,6 @@ Add a `ChallengeCompleted bool` to distinguish "challenge was completed and hash verification check in `downloadMessage` erroneously fails when the challenge was skipped. -### ~~6. Fix response code constants~~ -**File:** `host.go` constants -~~`RejectCodeMustChallenge` (11) and `RejectCodeCannotChallenge` (12) do not -exist in the spec. Code 11 = "accept header" (add-to notification success). -Add missing per-user codes: 102 (user not accepting), 103 (user undisclosed).~~ -**DONE:** Replaced with `AcceptCodeHeader` (11), added `RejectCodeUserNotAccepting` (102) -and `RejectCodeUserUndisclosed` (103). - --- ## P1 — Receiving path (host.go) correctness @@ -54,11 +32,6 @@ connection's IP is authorised. If not → TERMINATE (no reject code). Currently this only happens inside `challenge()` and is skipped when skip-challenge is allowed. -### 8. Parse "add to" field when has-add-to flag is set -**File:** `host.go` `readHeader()` -Spec 1.4.v.b: Read uint8 count + addresses. Verify distinct from each other and -from "to" (case-insensitive). Implement the pid/add-to decision tree (add-to -requires pid; if no add-to recipients on our domain → accept header code 11). ### 9. Make topic conditional on pid absence **File:** `host.go` `readHeader()` diff --git a/dd.sql b/dd.sql index 3ca9abc..c904589 100644 --- a/dd.sql +++ b/dd.sql @@ -33,6 +33,17 @@ create table if not exists msg_to ( ); create index on msg_to ((lower(addr))); +create table if not exists msg_add_to ( + id bigserial primary key, + msg_id bigint not null references msg (id), + addr varchar(255) not null, + time_delivered double precision, -- if sending, time sending host recieved delivery confirmation, if receiving, time successfully received message + time_last_attempt double precision, -- only used when sending, time of last delivery attempt if failed; otherwise null + response_code smallint, -- only used when sending, response code of last delivery attempt if failed; otherwise null + unique (msg_id, addr) +); +create index on msg_add_to ((lower(addr))); + create table if not exists msg_attachment ( msg_id bigint references msg (id), filename varchar(255) not null, diff --git a/src/defs.go b/src/defs.go index 5ad8ab3..87936bf 100644 --- a/src/defs.go +++ b/src/defs.go @@ -24,13 +24,12 @@ type FMsgAttachmentHeader struct { } type FMsgHeader struct { - Version uint8 - Flags uint8 - Pid []byte - From FMsgAddress - To []FMsgAddress - // TODO [Spec]: Add AddTo []FMsgAddress field for the "add to" recipients - // (present when has-add-to flag bit 1 is set). + Version uint8 + Flags uint8 + Pid []byte + From FMsgAddress + To []FMsgAddress + AddTo []FMsgAddress Timestamp float64 Topic string Type string @@ -63,7 +62,6 @@ func (addr *FMsgAddress) ToString() string { // instead of returning one. // TODO [Spec]: The spec defines "message header" as all fields up to and // including the attachment headers field. This Encode() is missing: -// - The "add to" field (uint8 count + addresses, when has-add-to flag set). // - The "size" field (uint32). // - The "attachment headers" field (uint8 count + list of attachment headers). // @@ -86,6 +84,14 @@ func (h *FMsgHeader) Encode() []byte { b.WriteByte(byte(len(str))) b.WriteString(str) } + if h.Flags&FlagHasAddTo != 0 { + b.WriteByte(byte(len(h.AddTo))) + for _, addr := range h.AddTo { + str = addr.ToString() + b.WriteByte(byte(len(str))) + b.WriteString(str) + } + } if err := binary.Write(&b, binary.LittleEndian, h.Timestamp); err != nil { panic(err) } @@ -111,6 +117,13 @@ func (h *FMsgHeader) String() string { fmt.Fprintf(&b, "\n\t%s", addr.ToString()) } } + for i, addr := range h.AddTo { + if i == 0 { + fmt.Fprintf(&b, "\nadd to:\t%s", addr.ToString()) + } else { + fmt.Fprintf(&b, "\n\t%s", addr.ToString()) + } + } fmt.Fprintf(&b, "\ntopic:\t%s", h.Topic) fmt.Fprintf(&b, "\ntype:\t%s", h.Type) fmt.Fprintf(&b, "\nsize:\t%d", h.Size) diff --git a/src/host.go b/src/host.go index 107896b..5c18bd8 100644 --- a/src/host.go +++ b/src/host.go @@ -374,22 +374,46 @@ func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { num-- } - // TODO [Spec 1.4.v.b / flag bit 1]: Parse the "add to" field when the - // has-add-to flag (bit 1) is set. This requires: - // - Reading uint8 count + list of additional recipient addresses. - // - Verifying add-to addresses are distinct from each other and from "to" - // addresses (case-insensitive), per spec step 1.4.i.a. - // - Adding an AddTo []FMsgAddress field to FMsgHeader. - // - Implementing the pid/add-to decision tree from spec step 1.4.v.b: - // * If add-to exists, pid MUST also exist; else reject code 1. - // * If any add-to recipients are for our domain, continue normally - // (pid message does NOT have to be stored). - // * If NO add-to recipients are for our domain: - // - pid MUST refer to a stored message per "Verifying Message Stored"; - // else reject code 6. - // - At least one "to" recipient must be for our domain; else reject 1. - // - Record message header, respond ACCEPT code 11 (accept header), - // close connection (no further data to download). + // parse "add to" field when has-add-to flag (bit 1) is set + if flags&FlagHasAddTo != 0 { + // add to requires pid per spec 1.4.v.b.a + if flags&FlagHasPid == 0 { + codes := []byte{RejectCodeInvalid} + if err := rejectAccept(c, codes); err != nil { + return h, r, err + } + return h, r, fmt.Errorf("add to exists but pid does not") + } + + addToCount, err := r.ReadByte() + if err != nil { + return h, r, err + } + if addToCount == 0 { + codes := []byte{RejectCodeInvalid} + if err := rejectAccept(c, codes); err != nil { + return h, r, err + } + return h, r, fmt.Errorf("add to flag set but count is 0") + } + for addToCount > 0 { + addr, err := readAddress(r) + if err != nil { + return h, r, err + } + key := strings.ToLower(addr.ToString()) + if seen[key] { + codes := []byte{RejectCodeInvalid} + if err := rejectAccept(c, codes); err != nil { + return h, r, err + } + return h, r, fmt.Errorf("duplicate recipient address in add to: %s", addr.ToString()) + } + seen[key] = true + h.AddTo = append(h.AddTo, *addr) + addToCount-- + } + } // read timestamp if err := binary.Read(r, binary.LittleEndian, &h.Timestamp); err != nil { @@ -478,6 +502,68 @@ func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { // Currently this check only happens inside challenge() and is skipped // entirely when skip-challenge is allowed. + // add-to pid decision tree per spec 1.4.v.b + if len(h.AddTo) > 0 { + // check if any add-to recipients are for our domain + addToHasOurDomain := false + for _, addr := range h.AddTo { + if strings.EqualFold(addr.Domain, Domain) { + addToHasOurDomain = true + break + } + } + if !addToHasOurDomain { + // none of the add-to recipients are for our domain — this is an + // add-to notification for existing "to" recipients on our domain. + + // pid must refer to a stored message per "Verifying Message Stored" + parentID, err := lookupMsgIdByHash(h.Pid) + if err != nil { + return h, r, err + } + if parentID == 0 { + codes := []byte{RejectCodeParentNotFound} + if err := rejectAccept(c, codes); err != nil { + return h, r, err + } + return h, r, fmt.Errorf("add-to notification: parent not found for pid %s", hex.EncodeToString(h.Pid)) + } + + // at least one "to" recipient must be for our domain + toHasOurDomain := false + for _, addr := range h.To { + if strings.EqualFold(addr.Domain, Domain) { + toHasOurDomain = true + break + } + } + if !toHasOurDomain { + codes := []byte{RejectCodeInvalid} + if err := rejectAccept(c, codes); err != nil { + return h, r, err + } + return h, r, fmt.Errorf("add-to notification: no 'to' recipients for our domain %s", Domain) + } + + // record this message header and respond accept header (code 11) + if err := storeMsgHeaderOnly(h); err != nil { + codes := []byte{RejectCodeUndisclosed} + if err2 := rejectAccept(c, codes); err2 != nil { + return h, r, err2 + } + return h, r, fmt.Errorf("add-to notification: storing header: %w", err) + } + codes := []byte{AcceptCodeHeader} + if err := rejectAccept(c, codes); err != nil { + return h, r, err + } + log.Printf("INFO: add-to notification accepted (code 11) for pid %s", hex.EncodeToString(h.Pid)) + return nil, r, nil // nil header signals handled (like challenge) + } + // else: add-to recipients are for our domain, continue normally + // (pid message does NOT have to be already stored per spec 1.4.v.b.b) + } + // TODO [Spec 1.4.v.c]: When pid exists and add-to does not: // a. The message or message header pid refers to MUST be verified as stored // per "Verifying Message Stored"; else reject code 6 (parent not found). @@ -621,18 +707,20 @@ func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { // code 10 (duplicate) and close. Currently the duplicate check happens // AFTER downloading all data, wasting bandwidth on duplicates. - // filter to our domain recipients - // TODO [Spec 3.4.i]: Per spec step 3.4, response codes must be sent for - // each recipient on our domain in the order they appear in "to" THEN - // "add to". Currently only "to" is considered (add-to not implemented). + // filter to our domain recipients in "to" then "add to" order per spec 3.4.i addrs := []FMsgAddress{} for _, addr := range h.To { - if addr.Domain == Domain { + if strings.EqualFold(addr.Domain, Domain) { + addrs = append(addrs, addr) + } + } + for _, addr := range h.AddTo { + if strings.EqualFold(addr.Domain, Domain) { addrs = append(addrs, addr) } } if len(addrs) == 0 { - return fmt.Errorf("%w our domain: %s, not in recipient list: %s", ErrProtocolViolation, Domain, h.To) + return fmt.Errorf("%w our domain: %s, not in recipient list", ErrProtocolViolation, Domain) } codes := make([]byte, len(addrs)) @@ -717,7 +805,13 @@ func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { } // validate each recipient and copy message for accepted ones - acceptedAddrs := []FMsgAddress{} + // Build a set of add-to addresses for later classification + addToSet := make(map[string]bool) + for _, addr := range h.AddTo { + addToSet[strings.ToLower(addr.ToString())] = true + } + acceptedTo := []FMsgAddress{} + acceptedAddTo := []FMsgAddress{} var primaryFilepath string for i, addr := range addrs { code, err := validateMsgRecvForAddr(h, &addr) @@ -764,20 +858,27 @@ func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { } codes[i] = RejectCodeAccept - acceptedAddrs = append(acceptedAddrs, addr) + if addToSet[strings.ToLower(addr.ToString())] { + acceptedAddTo = append(acceptedAddTo, addr) + } else { + acceptedTo = append(acceptedTo, addr) + } if primaryFilepath == "" { primaryFilepath = fp } } // store message details once for all accepted recipients - if len(acceptedAddrs) > 0 { + if len(acceptedTo) > 0 || len(acceptedAddTo) > 0 { origTo := h.To - h.To = acceptedAddrs + origAddTo := h.AddTo + h.To = acceptedTo + h.AddTo = acceptedAddTo h.Filepath = primaryFilepath if err := storeMsgDetail(h); err != nil { log.Printf("ERROR: storing message: %s", err) h.To = origTo + h.AddTo = origAddTo for i := range codes { if codes[i] == RejectCodeAccept { codes[i] = RejectCodeUndisclosed @@ -785,8 +886,10 @@ func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { } } else { h.To = origTo - for i := range acceptedAddrs { - if err := postMsgStatRecv(&acceptedAddrs[i], h.Timestamp, int(h.Size)); err != nil { + h.AddTo = origAddTo + allAccepted := append(acceptedTo, acceptedAddTo...) + for i := range allAccepted { + if err := postMsgStatRecv(&allAccepted[i], h.Timestamp, int(h.Size)); err != nil { log.Printf("WARN: Failed to post msg recv stat: %s", err) } } diff --git a/src/sender.go b/src/sender.go index fcb6757..a4e6bd3 100644 --- a/src/sender.go +++ b/src/sender.go @@ -39,11 +39,6 @@ type pendingTarget struct { // retryable recipients. This is a lightweight read-only query — row-level // locks are acquired per-delivery in deliverMessage. // -// TODO [Spec]: This query does not distinguish between "to" and "add to" -// recipients — all are stored in msg_to. Once add-to support is implemented -// (FMsgHeader.AddTo, storeMsgDetail, loadMsg), ensure add-to recipients are -// also discovered here and their response codes handled correctly. -// // TODO [Spec]: Spec says to retry "with back-off" (e.g. exponential back-off). // Currently uses a fixed RetryInterval. Implement an exponential back-off // strategy — e.g. double the wait after each failed attempt per (msg, domain). @@ -60,6 +55,7 @@ func findPendingTargets() ([]pendingTarget, error) { now := timeutil.TimestampNow().Float64() + // query both msg_to and msg_add_to for pending targets rows, err := db.Query(` SELECT mt.msg_id, mt.addr FROM msg_to mt @@ -69,6 +65,15 @@ func findPendingTargets() ([]pendingTarget, error) { AND (mt.response_code IS NULL OR mt.response_code IN (3, 5)) AND (mt.time_last_attempt IS NULL OR ($1 - mt.time_last_attempt) > $2) AND ($1 - m.time_sent) < $3 + UNION ALL + SELECT mat.msg_id, mat.addr + FROM msg_add_to mat + INNER JOIN msg m ON m.id = mat.msg_id + WHERE mat.time_delivered IS NULL + AND m.time_sent IS NOT NULL + AND (mat.response_code IS NULL OR mat.response_code IN (3, 5)) + AND (mat.time_last_attempt IS NULL OR ($1 - mat.time_last_attempt) > $2) + AND ($1 - m.time_sent) < $3 `, now, RetryInterval, RetryMaxAge) if err != nil { return nil, err @@ -182,7 +187,6 @@ func deliverMessage(target pendingTarget) { log.Printf("ERROR: sender: scan locked addr: %s", err) return } - // keep only addresses on the target domain lastAt := strings.LastIndex(addr, "@") if lastAt != -1 && strings.EqualFold(addr[lastAt+1:], target.Domain) { lockedAddrs = append(lockedAddrs, addr) @@ -193,7 +197,45 @@ func deliverMessage(target pendingTarget) { log.Printf("ERROR: sender: lock rows err for msg %d: %s", target.MsgID, err) return } - if len(lockedAddrs) == 0 { + + // Also lock pending msg_add_to rows for this message on the target domain. + lockAddToRows, err := tx.Query(` + SELECT mat.addr + FROM msg_add_to mat + INNER JOIN msg m ON m.id = mat.msg_id + WHERE mat.msg_id = $1 + AND mat.time_delivered IS NULL + AND m.time_sent IS NOT NULL + AND (mat.response_code IS NULL OR mat.response_code IN (3, 5)) + AND (mat.time_last_attempt IS NULL OR ($2 - mat.time_last_attempt) > $3) + AND ($2 - m.time_sent) < $4 + FOR UPDATE OF mat SKIP LOCKED + `, target.MsgID, now, RetryInterval, RetryMaxAge) + if err != nil { + log.Printf("ERROR: sender: lock add-to rows for msg %d: %s", target.MsgID, err) + return + } + + var lockedAddToAddrs []string + for lockAddToRows.Next() { + var addr string + if err := lockAddToRows.Scan(&addr); err != nil { + lockAddToRows.Close() + log.Printf("ERROR: sender: scan locked add-to addr: %s", err) + return + } + lastAt := strings.LastIndex(addr, "@") + if lastAt != -1 && strings.EqualFold(addr[lastAt+1:], target.Domain) { + lockedAddToAddrs = append(lockedAddToAddrs, addr) + } + } + lockAddToRows.Close() + if err := lockAddToRows.Err(); err != nil { + log.Printf("ERROR: sender: lock add-to rows err for msg %d: %s", target.MsgID, err) + return + } + + if len(lockedAddrs) == 0 && len(lockedAddToAddrs) == 0 { return // already locked by another sender or no longer eligible } @@ -213,22 +255,21 @@ func deliverMessage(target pendingTarget) { // Build the list of recipients on the target domain (in order) and // note which ones we locked (i.e. are pending delivery this round). Per spec, - // response codes arrive per-recipient in To-field order excluding other domains. + // response codes arrive per-recipient in to then add-to order excluding other domains. // TODO check retry logic when some recipients on this domain are not locked // (e.g. already delivered or locked by another sender) — do we still get // per-recipient codes for the locked ones? - // - // TODO [Spec]: This loop only iterates h.To. Per spec, per-recipient response - // codes arrive "in the order they appear in to then add to, excluding recipients - // for other domains." Once add-to is implemented (FMsgHeader.AddTo), append - // add-to recipients on the target domain AFTER the to recipients. lockedSet := make(map[string]bool) for _, a := range lockedAddrs { lockedSet[strings.ToLower(a)] = true } + for _, a := range lockedAddToAddrs { + lockedSet[strings.ToLower(a)] = true + } type domainRecip struct { addr string isLocked bool + isAddTo bool } var domainRecips []domainRecip for _, addr := range h.To { @@ -237,6 +278,17 @@ func deliverMessage(target pendingTarget) { domainRecips = append(domainRecips, domainRecip{ addr: s, isLocked: lockedSet[strings.ToLower(s)], + isAddTo: false, + }) + } + } + for _, addr := range h.AddTo { + if strings.EqualFold(addr.Domain, target.Domain) { + s := addr.ToString() + domainRecips = append(domainRecips, domainRecip{ + addr: s, + isLocked: lockedSet[strings.ToLower(s)], + isAddTo: true, }) } } @@ -267,15 +319,11 @@ func deliverMessage(target pendingTarget) { } defer conn.Close() - // Send header (FMsgHeader.Encode writes version through type) - // TODO [Spec]: Encode() is incomplete — it is missing the "add to" field - // (uint8 count + addresses when has-add-to flag set), and it always encodes - // the topic field even when pid IS set (spec: topic must be absent when pid - // is set). It also always encodes the type as a length-prefixed string, but - // spec says common-type flag (bit 2) means type is a single uint8 index. - // Because the header hash is computed from Encode() output, the outgoing - // registration hash (used by the challenge handler) will be WRONG until - // Encode() is fixed — challenge verification will fail. + // Send header (FMsgHeader.Encode writes version through type, including add-to) + // TODO [Spec]: Encode() always encodes the topic field even when pid IS set + // (spec: topic must be absent when pid is set). It also always encodes the + // type as a length-prefixed string, but spec says common-type flag (bit 2) + // means type is a single uint8 index. headerBytes := h.Encode() if _, err := conn.Write(headerBytes); err != nil { log.Printf("ERROR: sender: writing header: %s", err) @@ -339,11 +387,34 @@ func deliverMessage(target pendingTarget) { now = timeutil.TimestampNow().Float64() if code < 100 { + // Code 11 (accept header) is a success for add-to notification. + if code == AcceptCodeHeader { + log.Printf("INFO: sender: msg %d add-to notification accepted by %s (code 11)", target.MsgID, target.Domain) + for _, a := range lockedAddrs { + if _, err := tx.Exec(` + UPDATE msg_to SET time_delivered = $1, response_code = $2 + WHERE msg_id = $3 AND addr = $4 + `, now, int(code), target.MsgID, a); err != nil { + log.Printf("ERROR: sender: update delivered for %s: %s", a, err) + } + } + for _, a := range lockedAddToAddrs { + if _, err := tx.Exec(` + UPDATE msg_add_to SET time_delivered = $1, response_code = $2 + WHERE msg_id = $3 AND addr = $4 + `, now, int(code), target.MsgID, a); err != nil { + log.Printf("ERROR: sender: update delivered add-to for %s: %s", a, err) + } + } + if err := tx.Commit(); err != nil { + log.Printf("ERROR: sender: commit tx for msg %d: %s", target.MsgID, err) + } else { + committed = true + } + return + } + // global rejection — update all locked recipients - // TODO [Spec]: Code 11 (accept header) is < 100 but is NOT a rejection. - // It means "message header received (add-to notification only)" and should - // be treated as a success for add-to-only scenarios. Handle code 11 - // separately before falling into this rejection branch. // // TODO [Spec]: Permanent failures (1 invalid, 2 unsupported version, // 4 too big, 10 duplicate) should NOT be retried. Currently all global @@ -360,6 +431,14 @@ func deliverMessage(target pendingTarget) { log.Printf("ERROR: sender: update last attempt for %s: %s", a, err) } } + for _, a := range lockedAddToAddrs { + if _, err := tx.Exec(` + UPDATE msg_add_to SET time_last_attempt = $1, response_code = $2 + WHERE msg_id = $3 AND addr = $4 + `, now, int(code), target.MsgID, a); err != nil { + log.Printf("ERROR: sender: update last attempt add-to for %s: %s", a, err) + } + } if err := tx.Commit(); err != nil { log.Printf("ERROR: sender: commit tx for msg %d: %s", target.MsgID, err) } else { @@ -386,26 +465,25 @@ func deliverMessage(target pendingTarget) { // TODO well receiving host still attempted delivery to this recipient — do we update response code for it? Spec is not clear on this scenario. } c := codes[i] - // TODO [Spec]: Code 11 (accept header) is a success for add-to-only - // recipients — treat it like code 200 (mark as delivered). Also handle - // codes 102 (user not accepting) and 103 (user undisclosed) which are - // per-user codes defined in spec but missing from the response code - // constants. Code 102 "not accepting at this time" may be transient. + table := "msg_to" + if dr.isAddTo { + table = "msg_add_to" + } if c == RejectCodeAccept { log.Printf("INFO: sender: delivered msg %d to %s", target.MsgID, dr.addr) - if _, err := tx.Exec(` - UPDATE msg_to SET time_delivered = $1, response_code = 200 + if _, err := tx.Exec(fmt.Sprintf(` + UPDATE %s SET time_delivered = $1, response_code = 200 WHERE msg_id = $2 AND addr = $3 - `, now, target.MsgID, dr.addr); err != nil { + `, table), now, target.MsgID, dr.addr); err != nil { log.Printf("ERROR: sender: update delivered for %s: %s", dr.addr, err) } } else { log.Printf("WARN: sender: msg %d to %s rejected: %s (%d)", target.MsgID, dr.addr, responseCodeName(c), c) - if _, err := tx.Exec(` - UPDATE msg_to SET time_last_attempt = $1, response_code = $2 + if _, err := tx.Exec(fmt.Sprintf(` + UPDATE %s SET time_last_attempt = $1, response_code = $2 WHERE msg_id = $3 AND addr = $4 - `, now, int(c), target.MsgID, dr.addr); err != nil { + `, table), now, int(c), target.MsgID, dr.addr); err != nil { log.Printf("ERROR: sender: update last attempt for %s: %s", dr.addr, err) } } diff --git a/src/store.go b/src/store.go index 0364a86..8805ba0 100644 --- a/src/store.go +++ b/src/store.go @@ -28,7 +28,7 @@ func testDb() error { log.Printf("INFO: Database connected: %s@%s:%s/%s", user, host, port, dbName) // verify required tables exist - for _, table := range []string{"msg", "msg_to", "msg_attachment"} { + for _, table := range []string{"msg", "msg_to", "msg_add_to", "msg_attachment"} { var exists bool err = db.QueryRow(`SELECT EXISTS ( SELECT FROM information_schema.tables @@ -126,11 +126,25 @@ values ($1, $2, $3)`) } } - // TODO [Spec]: Also insert add-to recipients (msg.AddTo) into msg_to, - // preserving their order AFTER to recipients. The DB should distinguish - // to vs add-to (e.g. a boolean column or separate table) so loadMsg can - // reconstruct the correct FMsgHeader.To and FMsgHeader.AddTo slices. - // Per-recipient response codes must arrive in "to then add to" order. + // insert add-to recipients into msg_add_to + if len(msg.AddTo) > 0 { + addToStmt, err := tx.Prepare(`insert into msg_add_to (msg_id, addr, time_delivered) +values ($1, $2, $3)`) + if err != nil { + return err + } + defer addToStmt.Close() + + for _, addr := range msg.AddTo { + var delivered interface{} + if addr.Domain == Domain { + delivered = now + } + if _, err := addToStmt.Exec(msgID, addr.ToString(), delivered); err != nil { + return err + } + } + } // resolve pid from psha256 (parent message hash) if len(msg.Pid) > 0 { @@ -150,15 +164,97 @@ values ($1, $2, $3)`) } +// storeMsgHeaderOnly stores just the message header for add-to notifications +// (spec code 11). Only the header is recorded so the header hash can be +// faithfully computed for subsequent messages referencing this one via pid. +func storeMsgHeaderOnly(msg *FMsgHeader) error { + db, err := sql.Open("postgres", "") + if err != nil { + return err + } + defer db.Close() + + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + headerHash := msg.GetHeaderHash() + + var msgID int64 + err = tx.QueryRow(`insert into msg (version + , flags + , time_sent + , from_addr + , topic + , type + , sha256 + , psha256 + , size + , filepath) +values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) +returning id`, + msg.Version, + msg.Flags, + msg.Timestamp, + msg.From.ToString(), + msg.Topic, + msg.Type, + headerHash, + msg.Pid, + int(msg.Size), + "").Scan(&msgID) + if err != nil { + return err + } + + // insert to recipients (for record keeping) + toStmt, err := tx.Prepare(`insert into msg_to (msg_id, addr) values ($1, $2)`) + if err != nil { + return err + } + defer toStmt.Close() + for _, addr := range msg.To { + if _, err := toStmt.Exec(msgID, addr.ToString()); err != nil { + return err + } + } + + // insert add-to recipients + if len(msg.AddTo) > 0 { + addToStmt, err := tx.Prepare(`insert into msg_add_to (msg_id, addr) values ($1, $2)`) + if err != nil { + return err + } + defer addToStmt.Close() + for _, addr := range msg.AddTo { + if _, err := addToStmt.Exec(msgID, addr.ToString()); err != nil { + return err + } + } + } + + // resolve pid from psha256 + if len(msg.Pid) > 0 { + var parentID sql.NullInt64 + err = tx.QueryRow("SELECT id FROM msg WHERE sha256 = $1", msg.Pid).Scan(&parentID) + if err != nil && err != sql.ErrNoRows { + return err + } + if parentID.Valid { + if _, err = tx.Exec("UPDATE msg SET pid = $1 WHERE id = $2", parentID.Int64, msgID); err != nil { + return err + } + } + } + + return tx.Commit() +} + // loadMsg loads a message and all its recipients from the database within the // given transaction and returns a fully populated FMsgHeader. // -// TODO [Spec]: All recipients are loaded into FMsgHeader.To with no distinction -// between "to" and "add to". Once the DB schema distinguishes the two groups -// (see storeMsgDetail TODO), populate FMsgHeader.To and FMsgHeader.AddTo -// separately, preserving their original wire-format order. The sender relies -// on this ordering for correct per-recipient response code mapping. -// // TODO [Spec]: Attachment headers are not loaded. Once the msg_attachment table // stores attachment metadata (flags, type, filename, size, filepath), loadMsg // should populate FMsgHeader.Attachments so the sender can write attachment @@ -207,12 +303,37 @@ func loadMsg(tx *sql.Tx, msgID int64) (*FMsgHeader, error) { allTo = append(allTo, *addr) } + // load add-to recipients from msg_add_to + addToRows, err := tx.Query(`SELECT addr FROM msg_add_to WHERE msg_id = $1 ORDER BY id`, msgID) + if err != nil { + return nil, fmt.Errorf("load add-to recipients for msg %d: %w", msgID, err) + } + var allAddTo []FMsgAddress + for addToRows.Next() { + var a string + if err := addToRows.Scan(&a); err != nil { + addToRows.Close() + return nil, fmt.Errorf("scan add-to addr: %w", err) + } + addr, err := parseAddress([]byte(a)) + if err != nil { + addToRows.Close() + return nil, fmt.Errorf("invalid add-to address %s: %w", a, err) + } + allAddTo = append(allAddTo, *addr) + } + addToRows.Close() + if err := addToRows.Err(); err != nil { + return nil, fmt.Errorf("add-to recipients query err for msg %d: %w", msgID, err) + } + return &FMsgHeader{ Version: uint8(version), Flags: uint8(flags), Pid: pid, From: *from, To: allTo, + AddTo: allAddTo, Timestamp: timeSent, Topic: topic, Type: typ, From c48decf98a9642a74179bc32214d32d95fa070db Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Wed, 1 Apr 2026 21:47:13 +0800 Subject: [PATCH 02/12] some unit tests --- src/defs_test.go | 320 +++++++++++++++++++++++++++++++++++++++++++ src/host_test.go | 188 +++++++++++++++++++++++++ src/outgoing_test.go | 94 +++++++++++++ 3 files changed, 602 insertions(+) create mode 100644 src/defs_test.go create mode 100644 src/host_test.go create mode 100644 src/outgoing_test.go diff --git a/src/defs_test.go b/src/defs_test.go new file mode 100644 index 0000000..070176d --- /dev/null +++ b/src/defs_test.go @@ -0,0 +1,320 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "math" + "testing" +) + +func TestAddressToString(t *testing.T) { + tests := []struct { + addr FMsgAddress + want string + }{ + {FMsgAddress{User: "alice", Domain: "example.com"}, "@alice@example.com"}, + {FMsgAddress{User: "Bob", Domain: "EXAMPLE.COM"}, "@Bob@EXAMPLE.COM"}, + {FMsgAddress{User: "a-b.c", Domain: "x.y.z"}, "@a-b.c@x.y.z"}, + } + for _, tt := range tests { + got := tt.addr.ToString() + if got != tt.want { + t.Errorf("FMsgAddress{%q, %q}.ToString() = %q, want %q", tt.addr.User, tt.addr.Domain, got, tt.want) + } + } +} + +func TestEncodeMinimalHeader(t *testing.T) { + h := &FMsgHeader{ + Version: 1, + Flags: 0, + From: FMsgAddress{User: "alice", Domain: "a.com"}, + To: []FMsgAddress{{User: "bob", Domain: "b.com"}}, + Timestamp: 1700000000.0, + Topic: "hello", + Type: "text/plain", + } + b := h.Encode() + + r := bytes.NewReader(b) + + // version + ver, _ := r.ReadByte() + if ver != 1 { + t.Fatalf("version = %d, want 1", ver) + } + + // flags + flags, _ := r.ReadByte() + if flags != 0 { + t.Fatalf("flags = %d, want 0", flags) + } + + // from address + fromLen, _ := r.ReadByte() + fromBytes := make([]byte, fromLen) + r.Read(fromBytes) + if string(fromBytes) != "@alice@a.com" { + t.Fatalf("from = %q, want %q", string(fromBytes), "@alice@a.com") + } + + // to count + toCount, _ := r.ReadByte() + if toCount != 1 { + t.Fatalf("to count = %d, want 1", toCount) + } + + // to[0] + toLen, _ := r.ReadByte() + toBytes := make([]byte, toLen) + r.Read(toBytes) + if string(toBytes) != "@bob@b.com" { + t.Fatalf("to[0] = %q, want %q", string(toBytes), "@bob@b.com") + } + + // timestamp + var ts float64 + binary.Read(r, binary.LittleEndian, &ts) + if ts != 1700000000.0 { + t.Fatalf("timestamp = %f, want 1700000000.0", ts) + } + + // topic + topicLen, _ := r.ReadByte() + topicBytes := make([]byte, topicLen) + r.Read(topicBytes) + if string(topicBytes) != "hello" { + t.Fatalf("topic = %q, want %q", string(topicBytes), "hello") + } + + // type + typeLen, _ := r.ReadByte() + typeBytes := make([]byte, typeLen) + r.Read(typeBytes) + if string(typeBytes) != "text/plain" { + t.Fatalf("type = %q, want %q", string(typeBytes), "text/plain") + } + + // should have consumed entire buffer + if r.Len() != 0 { + t.Fatalf("unexpected %d trailing bytes", r.Len()) + } +} + +func TestEncodeWithPid(t *testing.T) { + pid := make([]byte, 32) + for i := range pid { + pid[i] = byte(i) + } + h := &FMsgHeader{ + Version: 1, + Flags: FlagHasPid, + Pid: pid, + From: FMsgAddress{User: "a", Domain: "b.com"}, + To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + Timestamp: 0, + Topic: "", + Type: "text/plain", + } + b := h.Encode() + r := bytes.NewReader(b) + + r.ReadByte() // version + r.ReadByte() // flags + + // pid should be next 32 bytes + pidOut := make([]byte, 32) + n, _ := r.Read(pidOut) + if n != 32 { + t.Fatalf("pid bytes read = %d, want 32", n) + } + if !bytes.Equal(pidOut, pid) { + t.Fatalf("pid mismatch") + } +} + +func TestEncodeWithAddTo(t *testing.T) { + pid := make([]byte, 32) + h := &FMsgHeader{ + Version: 1, + Flags: FlagHasPid | FlagHasAddTo, + Pid: pid, + From: FMsgAddress{User: "a", Domain: "b.com"}, + To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + AddTo: []FMsgAddress{{User: "e", Domain: "f.com"}}, + Timestamp: 0, + Topic: "", + Type: "text/plain", + } + b := h.Encode() + r := bytes.NewReader(b) + + r.ReadByte() // version + r.ReadByte() // flags + + // skip pid (32 bytes) + pidBuf := make([]byte, 32) + r.Read(pidBuf) + + // skip from + fLen, _ := r.ReadByte() + fBuf := make([]byte, fLen) + r.Read(fBuf) + + // skip to count + to[0] + toCount, _ := r.ReadByte() + if toCount != 1 { + t.Fatalf("to count = %d, want 1", toCount) + } + tLen, _ := r.ReadByte() + tBuf := make([]byte, tLen) + r.Read(tBuf) + + // add to count + addToCount, _ := r.ReadByte() + if addToCount != 1 { + t.Fatalf("add to count = %d, want 1", addToCount) + } + + // add to[0] + atLen, _ := r.ReadByte() + atBuf := make([]byte, atLen) + r.Read(atBuf) + if string(atBuf) != "@e@f.com" { + t.Fatalf("add to[0] = %q, want %q", string(atBuf), "@e@f.com") + } +} + +func TestEncodeNoAddToWhenFlagUnset(t *testing.T) { + // When FlagHasAddTo is NOT set, add-to addresses should not appear on the wire + // even if the AddTo slice is populated. + h := &FMsgHeader{ + Version: 1, + Flags: 0, + From: FMsgAddress{User: "a", Domain: "b.com"}, + To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + AddTo: []FMsgAddress{{User: "e", Domain: "f.com"}}, // should be ignored + Timestamp: 0, + Topic: "", + Type: "text/plain", + } + withAddTo := h.Encode() + + h2 := &FMsgHeader{ + Version: 1, + Flags: 0, + From: FMsgAddress{User: "a", Domain: "b.com"}, + To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + Timestamp: 0, + Topic: "", + Type: "text/plain", + } + withoutAddTo := h2.Encode() + + if !bytes.Equal(withAddTo, withoutAddTo) { + t.Fatalf("encoded bytes differ when AddTo populated but flag unset") + } +} + +func TestGetHeaderHash(t *testing.T) { + h := &FMsgHeader{ + Version: 1, + Flags: 0, + From: FMsgAddress{User: "alice", Domain: "a.com"}, + To: []FMsgAddress{{User: "bob", Domain: "b.com"}}, + Timestamp: 1700000000.0, + Topic: "test", + Type: "text/plain", + } + hash := h.GetHeaderHash() + if len(hash) != 32 { + t.Fatalf("hash length = %d, want 32", len(hash)) + } + + // Must be deterministic + hash2 := h.GetHeaderHash() + if !bytes.Equal(hash, hash2) { + t.Fatal("GetHeaderHash not deterministic") + } + + // Must match manual SHA-256 of Encode() + expected := sha256.Sum256(h.Encode()) + if !bytes.Equal(hash, expected[:]) { + t.Fatal("GetHeaderHash does not match sha256(Encode())") + } +} + +func TestStringOutput(t *testing.T) { + h := &FMsgHeader{ + Version: 1, + Flags: 0, + From: FMsgAddress{User: "alice", Domain: "a.com"}, + To: []FMsgAddress{{User: "bob", Domain: "b.com"}, {User: "carol", Domain: "c.com"}}, + Timestamp: 0, + Topic: "greetings", + Type: "text/plain", + Size: 42, + } + s := h.String() + + // Check key substrings are present + for _, want := range []string{ + "v1", + "@alice@a.com", + "@bob@b.com", + "@carol@c.com", + "greetings", + "text/plain", + "42", + } { + if !bytes.Contains([]byte(s), []byte(want)) { + t.Errorf("String() missing %q", want) + } + } +} + +func TestStringWithAddTo(t *testing.T) { + h := &FMsgHeader{ + Version: 1, + Flags: FlagHasAddTo, + From: FMsgAddress{User: "alice", Domain: "a.com"}, + To: []FMsgAddress{{User: "bob", Domain: "b.com"}}, + AddTo: []FMsgAddress{{User: "dave", Domain: "d.com"}}, + Topic: "t", + Type: "text/plain", + } + s := h.String() + if !bytes.Contains([]byte(s), []byte("add to:")) { + t.Error("String() missing 'add to:' label") + } + if !bytes.Contains([]byte(s), []byte("@dave@d.com")) { + t.Error("String() missing add-to address") + } +} + +func TestEncodeTimestampEncoding(t *testing.T) { + // Verify the timestamp is encoded as little-endian float64 + h := &FMsgHeader{ + Version: 1, + Flags: 0, + From: FMsgAddress{User: "a", Domain: "b.com"}, + To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + Timestamp: 1700000000.5, + Topic: "", + Type: "", + } + b := h.Encode() + + // Find timestamp position: version(1) + flags(1) + from(1+len) + to_count(1) + to[0](1+len) + fromStr := "@a@b.com" + toStr := "@c@d.com" + offset := 1 + 1 + 1 + len(fromStr) + 1 + 1 + len(toStr) // = 2 + 9 + 10 = 21 + tsBytes := b[offset : offset+8] + + bits := binary.LittleEndian.Uint64(tsBytes) + ts := math.Float64frombits(bits) + if ts != 1700000000.5 { + t.Fatalf("timestamp = %f, want 1700000000.5", ts) + } +} diff --git a/src/host_test.go b/src/host_test.go new file mode 100644 index 0000000..6d56e33 --- /dev/null +++ b/src/host_test.go @@ -0,0 +1,188 @@ +package main + +import ( + "bytes" + "testing" + "time" +) + +func TestIsValidUser(t *testing.T) { + valid := []string{"alice", "Bob", "a-b", "a_b", "a.b", "user123", "A"} + for _, u := range valid { + if !isValidUser(u) { + t.Errorf("isValidUser(%q) = false, want true", u) + } + } + + invalid := []string{"", " ", "a b", "a@b", "a/b", string(make([]byte, 65))} + for _, u := range invalid { + if isValidUser(u) { + t.Errorf("isValidUser(%q) = true, want false", u) + } + } +} + +func TestIsValidDomain(t *testing.T) { + valid := []string{"example.com", "a.b.c", "foo-bar.com", "localhost"} + for _, d := range valid { + if !isValidDomain(d) { + t.Errorf("isValidDomain(%q) = false, want true", d) + } + } + + invalid := []string{ + "", + "nodot", // no dots and not localhost + ".leading.dot", // empty label + "trailing.", // empty label + "-start.com", // label starts with hyphen + "end-.com", // label ends with hyphen + "has space.com", + } + for _, d := range invalid { + if isValidDomain(d) { + t.Errorf("isValidDomain(%q) = true, want false", d) + } + } +} + +func TestParseAddress(t *testing.T) { + tests := []struct { + input string + wantErr bool + user string + domain string + }{ + {"@alice@example.com", false, "alice", "example.com"}, + {"@Bob@EXAMPLE.COM", false, "Bob", "EXAMPLE.COM"}, + {"@a-b.c@x.y.z", false, "a-b.c", "x.y.z"}, + // errors + {"alice@example.com", true, "", ""}, // missing leading @ + {"@alice", true, "", ""}, // missing second @ + {"@", true, "", ""}, // too short + {"ab", true, "", ""}, // too short + {"@@example.com", true, "", ""}, // empty user + {"@alice@", true, "", ""}, // empty domain (not valid) + {"@alice@nodot", true, "", ""}, // domain with no dot (not localhost) + } + for _, tt := range tests { + addr, err := parseAddress([]byte(tt.input)) + if tt.wantErr { + if err == nil { + t.Errorf("parseAddress(%q) = nil error, want error", tt.input) + } + continue + } + if err != nil { + t.Errorf("parseAddress(%q) error: %v", tt.input, err) + continue + } + if addr.User != tt.user || addr.Domain != tt.domain { + t.Errorf("parseAddress(%q) = {%q, %q}, want {%q, %q}", tt.input, addr.User, addr.Domain, tt.user, tt.domain) + } + } +} + +func TestReadUInt8Slice(t *testing.T) { + // Build a buffer: uint8 length = 5, then "hello" + var buf bytes.Buffer + buf.WriteByte(5) + buf.WriteString("hello") + // Extra trailing bytes should not be consumed + buf.WriteString("extra") + + slice, err := ReadUInt8Slice(&buf) + if err != nil { + t.Fatalf("ReadUInt8Slice error: %v", err) + } + if string(slice) != "hello" { + t.Fatalf("ReadUInt8Slice = %q, want %q", string(slice), "hello") + } + // "extra" should remain + rest := make([]byte, 5) + n, _ := buf.Read(rest) + if string(rest[:n]) != "extra" { + t.Fatalf("remaining bytes = %q, want %q", string(rest[:n]), "extra") + } +} + +func TestReadUInt8SliceEmpty(t *testing.T) { + var buf bytes.Buffer + buf.WriteByte(0) // zero-length slice + + slice, err := ReadUInt8Slice(&buf) + if err != nil { + t.Fatalf("ReadUInt8Slice error: %v", err) + } + if len(slice) != 0 { + t.Fatalf("expected empty slice, got len %d", len(slice)) + } +} + +func TestCalcNetIODuration(t *testing.T) { + // Small sizes should return MinNetIODeadline + d := calcNetIODuration(100, 5000) + if d < MinNetIODeadline { + t.Fatalf("calcNetIODuration(100, 5000) = %v, want >= %v", d, MinNetIODeadline) + } + + // Large sizes should exceed MinNetIODeadline + d = calcNetIODuration(1_000_000, 5000) + expected := time.Duration(float64(1_000_000) / 5000 * float64(time.Second)) // 200s + if d != expected { + t.Fatalf("calcNetIODuration(1000000, 5000) = %v, want %v", d, expected) + } +} + +func TestResponseCodeName(t *testing.T) { + tests := []struct { + code uint8 + want string + }{ + {RejectCodeInvalid, "invalid"}, + {RejectCodeUnsupportedVersion, "unsupported version"}, + {RejectCodeUndisclosed, "undisclosed"}, + {RejectCodeTooBig, "too big"}, + {RejectCodeInsufficentResources, "insufficient resources"}, + {RejectCodeParentNotFound, "parent not found"}, + {RejectCodePastTime, "past time"}, + {RejectCodeFutureTime, "future time"}, + {RejectCodeTimeTravel, "time travel"}, + {RejectCodeDuplicate, "duplicate"}, + {AcceptCodeHeader, "accept header"}, + {RejectCodeUserUnknown, "user unknown"}, + {RejectCodeUserFull, "user full"}, + {RejectCodeUserNotAccepting, "user not accepting"}, + {RejectCodeUserUndisclosed, "user undisclosed"}, + {RejectCodeAccept, "accept"}, + {99, "unknown(99)"}, + } + for _, tt := range tests { + got := responseCodeName(tt.code) + if got != tt.want { + t.Errorf("responseCodeName(%d) = %q, want %q", tt.code, got, tt.want) + } + } +} + +func TestFlagConstants(t *testing.T) { + // Verify flag bit assignments match SPEC.md + if FlagHasPid != 1 { + t.Errorf("FlagHasPid = %d, want 1 (bit 0)", FlagHasPid) + } + if FlagHasAddTo != 2 { + t.Errorf("FlagHasAddTo = %d, want 2 (bit 1)", FlagHasAddTo) + } + if FlagCommonType != 4 { + t.Errorf("FlagCommonType = %d, want 4 (bit 2)", FlagCommonType) + } + if FlagImportant != 8 { + t.Errorf("FlagImportant = %d, want 8 (bit 3)", FlagImportant) + } + if FlagNoReply != 16 { + t.Errorf("FlagNoReply = %d, want 16 (bit 4)", FlagNoReply) + } + if FlagDeflate != 32 { + t.Errorf("FlagDeflate = %d, want 32 (bit 5)", FlagDeflate) + } +} diff --git a/src/outgoing_test.go b/src/outgoing_test.go new file mode 100644 index 0000000..6585d48 --- /dev/null +++ b/src/outgoing_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "testing" +) + +func TestOutgoingMapOperations(t *testing.T) { + initOutgoing() + + h := &FMsgHeader{ + Version: 1, + Flags: 0, + From: FMsgAddress{User: "alice", Domain: "a.com"}, + To: []FMsgAddress{{User: "bob", Domain: "b.com"}}, + Topic: "test", + Type: "text/plain", + } + + var hash [32]byte + copy(hash[:], h.GetHeaderHash()) + + // Lookup before register should fail + _, ok := lookupOutgoing(hash) + if ok { + t.Fatal("lookupOutgoing found entry before register") + } + + // Register + registerOutgoing(hash, h) + + // Lookup after register should succeed + got, ok := lookupOutgoing(hash) + if !ok { + t.Fatal("lookupOutgoing failed after register") + } + if got != h { + t.Fatal("lookupOutgoing returned different pointer") + } + + // Delete + deleteOutgoing(hash) + + _, ok = lookupOutgoing(hash) + if ok { + t.Fatal("lookupOutgoing found entry after delete") + } +} + +func TestOutgoingMapMultipleEntries(t *testing.T) { + initOutgoing() + + h1 := &FMsgHeader{ + Version: 1, + From: FMsgAddress{User: "a", Domain: "b.com"}, + To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + Timestamp: 1.0, + Type: "text/plain", + } + h2 := &FMsgHeader{ + Version: 1, + From: FMsgAddress{User: "x", Domain: "y.com"}, + To: []FMsgAddress{{User: "z", Domain: "w.com"}}, + Timestamp: 2.0, + Type: "text/plain", + } + + var hash1, hash2 [32]byte + copy(hash1[:], h1.GetHeaderHash()) + copy(hash2[:], h2.GetHeaderHash()) + + registerOutgoing(hash1, h1) + registerOutgoing(hash2, h2) + + got1, ok1 := lookupOutgoing(hash1) + got2, ok2 := lookupOutgoing(hash2) + + if !ok1 || got1 != h1 { + t.Error("failed to look up h1") + } + if !ok2 || got2 != h2 { + t.Error("failed to look up h2") + } + + // Delete one, other should remain + deleteOutgoing(hash1) + _, ok1 = lookupOutgoing(hash1) + _, ok2 = lookupOutgoing(hash2) + if ok1 { + t.Error("h1 still present after delete") + } + if !ok2 { + t.Error("h2 missing after deleting h1") + } +} From 2460843190aaa3fa76c70ae1acd8255ac3f3c8db Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Fri, 3 Apr 2026 12:06:05 +0800 Subject: [PATCH 03/12] readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eb0bc60..e2ef674 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Tested with Go 1.25 on Linux and Windows, AMD64 and ARM ## Running -An up and running [fmsg Id API](https://github.com/markmnl/fmsgid) needs to be reachable by fmsgd to know users and their quotas for this fmsgd service. +An up and running [fmsg Id API](https://github.com/markmnl/fmsgid) needs to be reachable by fmsgd to know users and their quotas for this fmsgd service. See also [fmsg-docker](https://github.com/markmnl/fmsg-docker) - a docker compose stack for a fmsg host including fmsgid, fmsg-webpi and fmsgd. IP address to bind to and listen on is the only argument, `127.0.0.1` is used if argument not supplied. e.g. on Linux: From 5cb275939e2b41f0b82e3be778278659df40a1a7 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Fri, 3 Apr 2026 19:11:30 +0800 Subject: [PATCH 04/12] calc flags instead of storing bitfield --- TODO.md | 7 +++---- dd.sql | 11 ++++++++++- src/store.go | 49 +++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/TODO.md b/TODO.md index 0f0e22c..e144d6e 100644 --- a/TODO.md +++ b/TODO.md @@ -98,11 +98,10 @@ Currently only v==255 is handled. ## P2 — Sending path (sender.go) correctness -### 21. Include add-to recipients in domain recipient list +### ~~21. Include add-to recipients in domain recipient list~~ DONE **File:** `sender.go` `deliverMessage()` -`domainRecips` only iterates `h.To`. Per spec, per-recipient codes arrive in -"to then add to" order. Append add-to recipients on the target domain after to -recipients. +`domainRecips` already iterates both `h.To` and `h.AddTo` in order, with +`isAddTo` flag set appropriately. Implemented as part of the add-to feature. ### 22. Write attachment headers and attachment bodies **File:** `sender.go` `deliverMessage()` diff --git a/dd.sql b/dd.sql index c904589..a720b60 100644 --- a/dd.sql +++ b/dd.sql @@ -10,7 +10,9 @@ create table if not exists msg ( id bigserial primary key, version int not null, pid bigint references msg (id), - flags int not null, + no_reply boolean not null default false, + is_important boolean not null default false, + is_deflate boolean not null default false, time_sent double precision, -- time sending host recieved message for sending, message timestamp field, NULL means message not ready for sending i.e. draft from_addr varchar(255) not null, topic varchar(255) not null, @@ -68,3 +70,10 @@ drop trigger if exists trg_msg_to_insert on msg_to; create trigger trg_msg_to_insert after insert on msg_to for each row execute function notify_msg_to_insert(); + +-- notify when a new msg_add_to row is inserted with null time_delivered so the +-- sender can pick it up immediately instead of waiting for the next poll. +drop trigger if exists trg_msg_add_to_insert on msg_add_to; +create trigger trg_msg_add_to_insert + after insert on msg_add_to + for each row execute function notify_msg_to_insert(); diff --git a/src/store.go b/src/store.go index 8805ba0..75804d6 100644 --- a/src/store.go +++ b/src/store.go @@ -82,7 +82,9 @@ func storeMsgDetail(msg *FMsgHeader) error { var msgID int64 err = tx.QueryRow(`insert into msg (version - , flags + , no_reply + , is_important + , is_deflate , time_sent , from_addr , topic @@ -91,10 +93,12 @@ func storeMsgDetail(msg *FMsgHeader) error { , psha256 , size , filepath) -values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) +values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) returning id`, msg.Version, - msg.Flags, + msg.Flags&FlagNoReply != 0, + msg.Flags&FlagImportant != 0, + msg.Flags&FlagDeflate != 0, msg.Timestamp, msg.From.ToString(), msg.Topic, @@ -184,7 +188,9 @@ func storeMsgHeaderOnly(msg *FMsgHeader) error { var msgID int64 err = tx.QueryRow(`insert into msg (version - , flags + , no_reply + , is_important + , is_deflate , time_sent , from_addr , topic @@ -193,10 +199,12 @@ func storeMsgHeaderOnly(msg *FMsgHeader) error { , psha256 , size , filepath) -values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) +values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) returning id`, msg.Version, - msg.Flags, + msg.Flags&FlagNoReply != 0, + msg.Flags&FlagImportant != 0, + msg.Flags&FlagDeflate != 0, msg.Timestamp, msg.From.ToString(), msg.Topic, @@ -260,14 +268,15 @@ returning id`, // should populate FMsgHeader.Attachments so the sender can write attachment // headers and data on the wire and compute a correct header/message hash. func loadMsg(tx *sql.Tx, msgID int64) (*FMsgHeader, error) { - var version, flags, size int + var version, size int + var noReply, isImportant, isDeflate bool var pid []byte var fromAddr, topic, typ, filepath string var timeSent float64 err := tx.QueryRow(` - SELECT version, flags, psha256, from_addr, topic, type, time_sent, size, filepath + SELECT version, no_reply, is_important, is_deflate, psha256, from_addr, topic, type, time_sent, size, filepath FROM msg WHERE id = $1 - `, msgID).Scan(&version, &flags, &pid, &fromAddr, &topic, &typ, &timeSent, &size, &filepath) + `, msgID).Scan(&version, &noReply, &isImportant, &isDeflate, &pid, &fromAddr, &topic, &typ, &timeSent, &size, &filepath) if err != nil { return nil, fmt.Errorf("load msg %d: %w", msgID, err) } @@ -327,9 +336,29 @@ func loadMsg(tx *sql.Tx, msgID int64) (*FMsgHeader, error) { return nil, fmt.Errorf("add-to recipients query err for msg %d: %w", msgID, err) } + // Compute flags bitfield from stored booleans and loaded data. + // has_pid and has_add_to are derived from actual data rather than stored, + // so add-to recipients added after the original message are included. + var flags uint8 + if len(pid) > 0 { + flags |= FlagHasPid + } + if len(allAddTo) > 0 { + flags |= FlagHasAddTo + } + if noReply { + flags |= FlagNoReply + } + if isImportant { + flags |= FlagImportant + } + if isDeflate { + flags |= FlagDeflate + } + return &FMsgHeader{ Version: uint8(version), - Flags: uint8(flags), + Flags: flags, Pid: pid, From: *from, To: allTo, From a3abd5a220310213686c880e2f717de674d9f77b Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Fri, 3 Apr 2026 20:19:43 +0800 Subject: [PATCH 05/12] sha256 nullable (during drafting) --- dd.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd.sql b/dd.sql index a720b60..8e05dc9 100644 --- a/dd.sql +++ b/dd.sql @@ -17,7 +17,7 @@ create table if not exists msg ( from_addr varchar(255) not null, topic varchar(255) not null, type varchar(255) not null, - sha256 bytea unique not null, + sha256 bytea unique, psha256 bytea, size int not null, -- spec allows uint32 but we don't enforced by FMSG_MAX_MSG_SIZE filepath text not null From 2ce02181b6fcd6f326aac84ca4079c1d8ae5def0 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Sat, 4 Apr 2026 17:01:38 +0800 Subject: [PATCH 06/12] add to --- TODO.md | 26 -------------------------- src/host.go | 46 ++++++++++++---------------------------------- src/host_test.go | 2 +- src/sender.go | 6 +++--- src/store.go | 14 +++++++++++--- 5 files changed, 27 insertions(+), 67 deletions(-) diff --git a/TODO.md b/TODO.md index e144d6e..e700774 100644 --- a/TODO.md +++ b/TODO.md @@ -70,16 +70,6 @@ Currently `ChallengeHash` is zero-valued when skipped, causing false mismatch. Spec 3.2: Download sequential attachment byte sequences after message body, bounded by attachment header sizes. Currently only message body is downloaded. -### 16. Include add-to recipients in per-recipient response codes -**File:** `host.go` `downloadMessage()` -Spec 3.4: Response codes in order of "to" then "add to", excluding other -domains. Currently only "to" is considered. - -### 17. Return correct code for "not accepting" users -**File:** `host.go` `validateMsgRecvForAddr()` -When user is known but not accepting, return code 102 (user not accepting) or -103 (user undisclosed), not 100 (user unknown). - ### 18. Validate at least one "to" recipient **File:** `host.go` `readHeader()` Spec 1.4.i.a: If to count is 0, reject code 1 (invalid). @@ -108,12 +98,6 @@ Currently only v==255 is handled. Attachment count is hardcoded to 0. Write actual attachment headers (flags, type, filename, size) and send attachment data after message body. -### 23. Handle code 11 (accept header) as success -**File:** `sender.go` `deliverMessage()` -Code 11 is < 100 but is NOT a rejection — it means "header received" for -add-to-only scenarios. Handle separately from the global rejection branch. -Also treat code 11 as success in per-recipient handling for add-to recipients. - ### 24. Handle missing per-user codes 102 and 103 **File:** `sender.go` `deliverMessage()` Code 102 (user not accepting) may be transient — consider retry. Code 103 (user @@ -138,22 +122,12 @@ and not part of the spec. ## P4 — Storage layer -### 27. Distinguish to vs add-to in msg_to table -**File:** `store.go` `storeMsgDetail()` -Need a way (boolean column, separate table, or ordering convention) to tell "to" -from "add to" recipients so `loadMsg` can reconstruct both slices in order. - ### 28. Store and load attachment metadata **File:** `store.go` `storeMsgDetail()` / `loadMsg()` Insert attachment headers into `msg_attachment` and load them back into `FMsgHeader.Attachments` so the sender can write them on the wire and hashing is correct. -### 29. Load add-to recipients separately in loadMsg -**File:** `store.go` `loadMsg()` -Currently all recipients go into `FMsgHeader.To`. Populate `.To` and `.AddTo` -separately, preserving wire-format order. - --- ## P5 — Hash computation diff --git a/src/host.go b/src/host.go index 5c18bd8..b4bf2fe 100644 --- a/src/host.go +++ b/src/host.go @@ -49,7 +49,7 @@ const ( RejectCodeFutureTime uint8 = 8 RejectCodeTimeTravel uint8 = 9 RejectCodeDuplicate uint8 = 10 - AcceptCodeHeader uint8 = 11 + AcceptCodeAddTo uint8 = 11 RejectCodeUserUnknown uint8 = 100 RejectCodeUserFull uint8 = 101 @@ -82,8 +82,8 @@ func responseCodeName(code uint8) string { return "time travel" case RejectCodeDuplicate: return "duplicate" - case AcceptCodeHeader: - return "accept header" + case AcceptCodeAddTo: + return "accept add to" case RejectCodeUserUnknown: return "user unknown" case RejectCodeUserFull: @@ -396,20 +396,21 @@ func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { } return h, r, fmt.Errorf("add to flag set but count is 0") } + addToSeen := make(map[string]bool) for addToCount > 0 { addr, err := readAddress(r) if err != nil { return h, r, err } key := strings.ToLower(addr.ToString()) - if seen[key] { + if addToSeen[key] { codes := []byte{RejectCodeInvalid} if err := rejectAccept(c, codes); err != nil { return h, r, err } return h, r, fmt.Errorf("duplicate recipient address in add to: %s", addr.ToString()) } - seen[key] = true + addToSeen[key] = true h.AddTo = append(h.AddTo, *addr) addToCount-- } @@ -513,8 +514,8 @@ func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { } } if !addToHasOurDomain { - // none of the add-to recipients are for our domain — this is an - // add-to notification for existing "to" recipients on our domain. + // none of the add-to recipients are for our domain — record + // the additional recipients and respond code 11. // pid must refer to a stored message per "Verifying Message Stored" parentID, err := lookupMsgIdByHash(h.Pid) @@ -529,23 +530,7 @@ func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { return h, r, fmt.Errorf("add-to notification: parent not found for pid %s", hex.EncodeToString(h.Pid)) } - // at least one "to" recipient must be for our domain - toHasOurDomain := false - for _, addr := range h.To { - if strings.EqualFold(addr.Domain, Domain) { - toHasOurDomain = true - break - } - } - if !toHasOurDomain { - codes := []byte{RejectCodeInvalid} - if err := rejectAccept(c, codes); err != nil { - return h, r, err - } - return h, r, fmt.Errorf("add-to notification: no 'to' recipients for our domain %s", Domain) - } - - // record this message header and respond accept header (code 11) + // record add-to recipients for future hash reconstruction, respond code 11 if err := storeMsgHeaderOnly(h); err != nil { codes := []byte{RejectCodeUndisclosed} if err2 := rejectAccept(c, codes); err2 != nil { @@ -553,11 +538,11 @@ func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { } return h, r, fmt.Errorf("add-to notification: storing header: %w", err) } - codes := []byte{AcceptCodeHeader} + codes := []byte{AcceptCodeAddTo} if err := rejectAccept(c, codes); err != nil { return h, r, err } - log.Printf("INFO: add-to notification accepted (code 11) for pid %s", hex.EncodeToString(h.Pid)) + log.Printf("INFO: additional recipients received (code 11) for pid %s", hex.EncodeToString(h.Pid)) return nil, r, nil // nil header signals handled (like challenge) } // else: add-to recipients are for our domain, continue normally @@ -657,10 +642,6 @@ func validateMsgRecvForAddr(h *FMsgHeader, addr *FMsgAddress) (code uint8, err e } // check user accepting new - // TODO [Spec 3.4.i.d]: When a user is known but not accepting new messages - // the spec requires reject code 102 (user not accepting) or 103 (user - // undisclosed). Currently RejectCodeUserUnknown (100) is returned which is - // incorrect — code 100 means the address is unknown to the host. if !detail.AcceptingNew { return RejectCodeUserNotAccepting, nil } @@ -777,11 +758,8 @@ func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { // the current message's timestamp (reject code 9 = time travel), and the // verification must follow the "Verifying Message Stored" semantics: // - The digest must match a previously accepted message (code 200) or - // accepted header (code 11). + // accepted add-to (code 11). // - The message/header must currently exist and be retrievable. - // - pid references a message hash if from was in "to" of parent; references - // the message header hash if from was only in "add to" of parent. Both - // must be checked. if h.Flags&FlagHasPid != 0 { parentID, err := lookupMsgIdByHash(h.Pid) if err != nil { diff --git a/src/host_test.go b/src/host_test.go index 6d56e33..2f8a2c8 100644 --- a/src/host_test.go +++ b/src/host_test.go @@ -149,7 +149,7 @@ func TestResponseCodeName(t *testing.T) { {RejectCodeFutureTime, "future time"}, {RejectCodeTimeTravel, "time travel"}, {RejectCodeDuplicate, "duplicate"}, - {AcceptCodeHeader, "accept header"}, + {AcceptCodeAddTo, "accept add to"}, {RejectCodeUserUnknown, "user unknown"}, {RejectCodeUserFull, "user full"}, {RejectCodeUserNotAccepting, "user not accepting"}, diff --git a/src/sender.go b/src/sender.go index a4e6bd3..66b8ec2 100644 --- a/src/sender.go +++ b/src/sender.go @@ -387,9 +387,9 @@ func deliverMessage(target pendingTarget) { now = timeutil.TimestampNow().Float64() if code < 100 { - // Code 11 (accept header) is a success for add-to notification. - if code == AcceptCodeHeader { - log.Printf("INFO: sender: msg %d add-to notification accepted by %s (code 11)", target.MsgID, target.Domain) + // Code 11 (accept add to) — additional recipients received. + if code == AcceptCodeAddTo { + log.Printf("INFO: sender: msg %d additional recipients received by %s (code 11)", target.MsgID, target.Domain) for _, a := range lockedAddrs { if _, err := tx.Exec(` UPDATE msg_to SET time_delivered = $1, response_code = $2 diff --git a/src/store.go b/src/store.go index 75804d6..bbc65d5 100644 --- a/src/store.go +++ b/src/store.go @@ -270,13 +270,13 @@ returning id`, func loadMsg(tx *sql.Tx, msgID int64) (*FMsgHeader, error) { var version, size int var noReply, isImportant, isDeflate bool - var pid []byte + var pid, msgHash []byte var fromAddr, topic, typ, filepath string var timeSent float64 err := tx.QueryRow(` - SELECT version, no_reply, is_important, is_deflate, psha256, from_addr, topic, type, time_sent, size, filepath + SELECT version, no_reply, is_important, is_deflate, psha256, sha256, from_addr, topic, type, time_sent, size, filepath FROM msg WHERE id = $1 - `, msgID).Scan(&version, &noReply, &isImportant, &isDeflate, &pid, &fromAddr, &topic, &typ, &timeSent, &size, &filepath) + `, msgID).Scan(&version, &noReply, &isImportant, &isDeflate, &pid, &msgHash, &fromAddr, &topic, &typ, &timeSent, &size, &filepath) if err != nil { return nil, fmt.Errorf("load msg %d: %w", msgID, err) } @@ -339,6 +339,14 @@ func loadMsg(tx *sql.Tx, msgID int64) (*FMsgHeader, error) { // Compute flags bitfield from stored booleans and loaded data. // has_pid and has_add_to are derived from actual data rather than stored, // so add-to recipients added after the original message are included. + // + // When add-to recipients exist on a root message (no pid), set pid to the + // message's own hash so the wire format is valid: spec requires pid when + // add-to is present. This turns the outgoing message into an add-to + // notification referencing the original message. + if len(allAddTo) > 0 && len(pid) == 0 { + pid = msgHash + } var flags uint8 if len(pid) > 0 { flags |= FlagHasPid From 41661f8c5e377345940b56cb40344ed41636c712 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Sat, 4 Apr 2026 17:02:08 +0800 Subject: [PATCH 07/12] updated SPEC --- SPEC.md | 89 ++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/SPEC.md b/SPEC.md index 4075eb8..dd3db2c 100644 --- a/SPEC.md +++ b/SPEC.md @@ -10,13 +10,13 @@ ## Wire Format (Field Order -- MUST NOT CHANGE) -1. version (uint8) -- 1..127 = fmsg version; 128..255 = CHALLENGE (version = 256 - value) +1. version (uint8) -- 1..127 = fmsg version; 129..255 = CHALLENGE (version = 256 - value); 0 and 128 are undefined 2. flags (uint8) 3. [pid] (32 byte SHA-256 hash) -- present only if flag bit 0 set 4. from (uint8 + UTF-8 address) 5. to (uint8 count + list of addresses, distinct case-insensitive, at least one) 6. [add to] (uint8 count + list of addresses) -- present only if flag bit 1 set, distinct case-insensitive, at least one -7. time (float64 POSIX timestamp) +7. time (float64 POSIX timestamp, stamped by sender's host when message acquired) 8. [topic] (uint8 + UTF-8 string) -- present only when pid is NOT set, may be 0-length 9. type (uint8 common-type index when flag bit 2 set; otherwise uint8 length + ASCII MIME string per RFC 6838) 10. size (uint32, data size in bytes, 0 or greater) @@ -26,6 +26,8 @@ All multi-byte integers are little-endian. All strings are prefixed with uint8 length. +_case-insensitive_ means byte-wise equality after Unicode default case folding (locale-independent). +TERMINATE means tear down all connection(s) with the remote host immediately. ------------------------------------------------------------------------ @@ -35,9 +37,11 @@ All strings are prefixed with uint8 length. - When pid exists the entire topic field MUST NOT be included on the wire. - Root messages MUST NOT include pid. - Reply messages MUST include pid. -- All recipients (to + add to combined) MUST be distinct (case-insensitive). -- pid references previous message hash if from was in to of previous message; - references previous message header hash if from was only in add to. +- All recipients in to MUST be distinct (case-insensitive). +- All recipients in add to MUST be distinct (case-insensitive). +- pid always references the previous message hash. +- When add to exists, pid MUST also exist. +- A sender (from) MUST have been a participant in the message referenced by pid. - Data size MUST match the declared size field. - Attachment count MUST fit within uint8 range. @@ -136,13 +140,14 @@ in _to_ then _add to_, excluding recipients for other domains. | 7 | too old | timestamp is too far in the past for this host to accept | | 8 | future time | timestamp is too far in the future for this host to accept | | 9 | time travel | timestamp is before parent timestamp | -| 10 | duplicate | message has already been received | -| 11 | accept header | message header received (add-to notification only) | +| 10 | duplicate | message has already been received for all recipients on this host | +| 11 | accept add to | additional recipients received | | | | | | 100 | user unknown | the recipient is unknown by this host | | 101 | user full | insufficient resources for specific recipient | | 102 | user not accepting | user is known but not accepting new messages at this time | -| 103 | user undisclosed | no reason given (MAY be used instead of 100-102 to avoid disclosure)| +| 103 | user duplicate | message has already been received for this recipient | +| 105 | user undisclosed | no reason given (MAY be used instead of 100-103 to avoid disclosure)| | | | | | 200 | accept | message received for recipient | @@ -154,10 +159,16 @@ A **message** is verified stored if: the SHA-256 digest exactly matches a previously accepted (code 200) message, and that message currently exists and can be retrieved. -A **message header** is verified stored if: the SHA-256 digest exactly matches -a previously accepted (code 11) message header, and that header currently exists +OR the SHA-256 digest exactly matches the digest computed over message bytes +with add to recipients included and add to flag set, previously accepted +(code 11, additional recipients received), and that message currently exists and can be retrieved. +NOTE: Multiple add to messages may arrive for the same pid, each with a +different batch of additional recipients. The host MUST record each batch +individually (not accumulate) so the exact message bytes can be reconstructed +per batch for hash verification. + ------------------------------------------------------------------------ ## Protocol Steps Configuration @@ -177,16 +188,44 @@ and can be retrieved. Host A connects to Host B via first responsive authorised IP from Domain Resolution. Host A transmits the message. Host B reads the first byte to determine type (version vs CHALLENGE), then downloads and verifies the remaining header. -Verification includes: recipient uniqueness, sender IP authorisation, size limits, -time bounds, common type mappings, and pid/add-to rules. + +Verification includes: at least one recipient in to, recipient distinctness +(to and add to checked separately), at least one recipient for Host B's domain, +sender IP authorisation, size limits, time bounds, and common type mappings. + +pid/add-to rules: +- No pid, no add to → first message in thread, continue normally. +- pid exists, no add to → pid MUST be verified stored; parent time MUST be + before message time. NOTE: Verified stored checks the host has the parent, + not that every recipient still has it in their mailbox. Implementations + SHOULD consider restoring the parent to a recipient's mailbox if deleted. +- add to exists → pid MUST also exist, otherwise reject (code 1). + - If any add to recipients are for Host B → message download continues in + full (pid references previous message Host B might not have). + - If no add to recipients are for Host B → only the message header is + sent (no data/attachments). pid MUST match a message originally accepted + with code 200 (not code 11, preventing add to chaining); code 6 if not + found; parent time MUST be before message time (code 9). + Host B records the new add to recipients for future hash reconstruction, + responds code 11 (additional recipients received), closes connection. + NOTE: Implementations SHOULD consider restoring the referenced message to + a recipient's mailbox if previously deleted, so newly added recipients + have proper thread context. ### 2. The Automatic Challenge Host B MAY challenge Host A (modes: NEVER, ALWAYS, HAS_NOT_PARTICIPATED, DIFFERENT_DOMAIN). -Host B opens Connection 2 to the **same IP** as Connection 1, sends a CHALLENGE -(version byte + message header hash). Host A verifies the header hash matches its -outgoing message; if not → TERMINATE. Host A responds with the message hash. -Host B keeps the response hash for later verification. Both close Connection 2. +Host B MUST verify sender IP is in the authorised set for the from domain +**before** opening Connection 2. Host B opens Connection 2 to the **same IP** as +Connection 1, sends a CHALLENGE (version byte + message header hash). Host A +verifies the header hash matches its outgoing message; if not → TERMINATE. Host A +responds with the message hash. Host B keeps the response hash for later +verification. Both close Connection 2. + +HAS_NOT_PARTICIPATED is particularly important for messages with add to recipients +for Host B's domain — Host B may not have the parent referenced by pid and cannot +verify it is stored, making such messages indistinguishable from unsolicited ones +without a challenge. ### 3. Integrity Verification, Per-Recipient Response and Disposition @@ -194,7 +233,11 @@ Before downloading remaining data: if challenge was completed, check for duplica via message hash → code 10. Host B downloads data + attachments. If challenge was completed, verify computed hash matches challenge response → TERMINATE on mismatch. Host B sends per-recipient ACCEPT/REJECT codes in _to_ then _add to_ order -(excluding other domains). +(excluding other domains). For each recipient, check in order: already received +(103 user duplicate) → unknown (100) → exceeds quota (101 user full) → not +accepting (102) → otherwise accept (200). For any per-user REJECT, 105 +(user undisclosed) MAY be used instead. Global duplicate (code 10) is for the +entire message across all recipients. ### 4. Sending a Message @@ -206,6 +249,8 @@ to all recipients on that domain; per-recipient codes (≥100) arrive in _to_ th _add to_ order. Host A records codes and retries transient failures (3 undisclosed, 5 insufficient resources) with back-off. Permanent failures (1 invalid, 2 unsupported version, 4 too big, 10 duplicate) are not retried. +Per-user codes: 101 (user full) MAY warrant retry; 100 (user unknown) or +103 (user duplicate) typically would not. ### Handling a Challenge @@ -236,8 +281,12 @@ Implementors should be mindful of concurrent access to this record. ## Rejection Conditions (MUST Reject) - Cannot decode / malformed structure → TERMINATE -- Duplicate recipients (to + add to, case-insensitive) → code 1 +- No recipients in to → code 1 +- Duplicate recipients in to (case-insensitive) → code 1 +- Duplicate recipients in add to (case-insensitive) → code 1 +- No recipients in to or add to for receiving host's domain → code 1 - Common type flag set but value has no mapping → code 1 +- Attachment common type flag set but value has no mapping → code 1 - add to exists but pid does not → code 1 - Unauthorised sender IP → TERMINATE - DNSSEC validation failure → TERMINATE @@ -245,7 +294,7 @@ Implementors should be mindful of concurrent access to this record. - Message too old (DELTA > MAX_MESSAGE_AGE) → code 7 - Message too far in future (|DELTA| > MAX_TIME_SKEW) → code 8 - pid parent not found (when required per add-to rules) → code 6 -- Parent time ≥ message time → code 9 +- Parent time ≥ message time (when parent is required and found) → code 9 - Duplicate message (via challenge hash lookup) → code 10 - Hash mismatch after full download (challenge was completed) → TERMINATE @@ -258,7 +307,7 @@ When generating or modifying code: - Always serialize and parse fields exactly in defined order. - Never use TXT, MX, or SRV for host discovery. - Always resolve `_fmsg.` using A/AAAA (with CNAME support). -- Enforce recipient uniqueness across to and add to. +- Enforce recipient uniqueness within to and within add to separately. - Validate sender IP before issuing CHALLENGE. - Terminate immediately on DNSSEC failure. - Respect all flag semantics strictly (use spec bit assignments). From 7e8a2c8cede959088ebfb30d93fea4f4bff5600a Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Sat, 4 Apr 2026 17:11:04 +0800 Subject: [PATCH 08/12] implement attachments --- TODO2.md | 240 +++++++++++++++++++++++++++++++++++++++++++++++ src/defs.go | 64 ++++++++----- src/defs_test.go | 160 ++++++++++++++++++++++++++++++- src/sender.go | 47 ++++------ 4 files changed, 459 insertions(+), 52 deletions(-) create mode 100644 TODO2.md diff --git a/TODO2.md b/TODO2.md new file mode 100644 index 0000000..371585c --- /dev/null +++ b/TODO2.md @@ -0,0 +1,240 @@ +# TODO2 + +Gap analysis of the codebase against SPEC.md. + +--- + +## P0 — Wire Format / Encoding (defs.go) + +These are foundational: the header hash (used for challenge verification and pid +references) and message hash (used for duplicate detection, challenge response, +and pid) are both wrong until Encode() is complete. + +### ~~1. Encode() must include size and attachment headers~~ DONE +**File:** `defs.go` `Encode()` +Spec defines "message header" as all fields through the attachment headers. +Encode() currently stops after the type field — it omits: + - size (uint32 LE) + - attachment count (uint8) + attachment headers (flags, type, filename, size) + +The header hash (SHA-256 of the encoded header) will be incorrect without these +fields, which breaks challenge verification and pid references. + +### 2. Encode() must omit topic when pid is set +**File:** `defs.go` `Encode()` +Spec: "When pid exists the entire topic field MUST NOT be included on the wire." +Encode() always writes the topic. It must only write topic when pid is absent. + +### 3. Encode() must support common-type encoding for type field +**File:** `defs.go` `Encode()` +When common-type flag (bit 2) is set, type on the wire is a single uint8 index, +not a length-prefixed string. Encode() always writes length-prefixed. + +### ~~4. Add Attachments field to FMsgHeader~~ DONE +**File:** `defs.go` struct +Add `Attachments []FMsgAttachmentHeader` to store parsed attachment headers. +Required before Encode() can include attachment headers, before attachment +parsing/validation/download, and before hash computation is correct. + +### ~~5. Complete FMsgAttachmentHeader struct~~ DONE +**File:** `defs.go` struct +Currently only has Filename, Size, Filepath. Missing per-spec fields: + - Flags (uint8) — including per-attachment common-type (bit 0) and deflate (bit 1) + - Type (string) — the attachment's media type + +### 6. Add ChallengeCompleted flag to FMsgHeader +**File:** `defs.go` struct +Add `ChallengeCompleted bool` to distinguish "challenge was completed and +ChallengeHash is valid" from "challenge was not performed." Without this, the +hash check in downloadMessage erroneously fails when the challenge was skipped. + +### ~~7. GetMessageHash() must include attachment data~~ DONE +**File:** `defs.go` `GetMessageHash()` +Spec: message hash is SHA-256 of the entire message — header + data + +attachment data. Currently attachment data (sequential byte sequences following +the message body) is not included. + +--- + +## P1 — Receiving: Header Exchange (host.go readHeader) + +### 8. Generalise first-byte version/challenge detection +**File:** `host.go` `readHeader()` +Spec step 1.3: 1..127 = version, 129..255 = CHALLENGE (version = 256 − value), +0 and 128 are undefined. Currently only v==255 is handled as a challenge. Must +handle any value > 128 where (256 − value) is a supported version. + +### 9. Send reject code 2 for unsupported version +**File:** `host.go` `readHeader()` +Spec 1.3.iii: Send code 2 on the connection before closing. Currently just +returns an error without sending any code. + +### 10. Validate at least one "to" recipient +**File:** `host.go` `readHeader()` +Spec 1.4.i.a: If to count is 0, reject code 1 (invalid). Currently no check. + +### 11. Make topic conditional on pid absence +**File:** `host.go` `readHeader()` +Spec: topic field is only present when pid is NOT set. Currently topic is +read unconditionally regardless of pid. + +### 12. Handle common-type flag for type field +**File:** `host.go` `readHeader()` +When common-type flag (bit 2) is set, type is a single uint8 index into the +Common Media Types table. If the value has no mapping → reject code 1. +Currently always reads type as a length-prefixed string. + +### 13. Parse and validate attachment headers +**File:** `host.go` `readHeader()` +Currently rejects any non-zero attachment count. Must: + - Parse each header: flags (uint8), type, filename (uint8 + UTF-8), size (uint32). + - Validate per-attachment common-type flag mapping (reject code 1 if unmapped). + - Validate filenames per spec (Unicode letters/numbers, limited special chars, + no consecutive dots, not at start/end, unique case-insensitive, < 256 bytes). + - Include all attachment sizes in MAX_SIZE check (data size + Σ attachment sizes). + - Store parsed headers on FMsgHeader.Attachments. + +### 14. DNS-verify sender IP during header exchange +**File:** `host.go` `readHeader()` +Spec 1.4.ii / "Sender IP Verification": Host B MUST resolve +`_fmsg.` and verify the incoming connection's IP is in the +authorised set. If not authorised → TERMINATE (no reject code, no challenge). +Currently this check only happens inside challenge() and is skipped when +challenge is not performed. + +### 15. Perform pid verification during header exchange +**File:** `host.go` `readHeader()` +Spec 1.4.v.c: When pid exists and add to does not: + a. pid must be verified stored per "Verifying Message Stored"; else code 6. + b. Parent time must be before message time; else code 9 (time travel). +Currently deferred to downloadMessage(). + +### 16. Distinguish code-200 vs code-11 parents for add-to header-only path +**File:** `host.go` `readHeader()` + `store.go` +Spec 1.4.v.b.a (no add-to recipients for our domain): pid MUST match a message +originally accepted with code 200 (not code 11), preventing add-to chaining. +Currently lookupMsgIdByHash does not distinguish how the message was accepted. + +### 17. Verify sender participated in parent message +**File:** `host.go` `readHeader()` +Spec invariant: "A sender (from) MUST have been a participant in the message +referenced by pid." Not currently checked. + +--- + +## P2 — Receiving: Challenge (host.go challenge) + +### 18. Connection 2 must target same IP as Connection 1 +**File:** `host.go` `challenge()` +Spec 2.1: Dial conn.RemoteAddr() IP, not h.From.Domain. Dialling the domain +may resolve to a different IP. + +### 19. Make challenge mode configurable +**File:** `host.go` `challenge()` / `handleConn()` +Spec defines challenge modes (NEVER, ALWAYS, HAS_NOT_PARTICIPATED, +DIFFERENT_DOMAIN) as Host B's implementation choice. Currently always challenges. +At minimum, support skipping the challenge so the ChallengeCompleted guard +(item 6) works correctly. + +--- + +## P3 — Receiving: Download and Response (host.go downloadMessage) + +### 20. Pre-download duplicate check via challenge hash +**File:** `host.go` `downloadMessage()` +Spec 3.1: BEFORE downloading data, if challenge was completed, use the message +hash to check for duplicate → code 10. Currently the duplicate check is after +download, wasting bandwidth. + +### 21. Guard hash verification behind ChallengeCompleted +**File:** `host.go` `downloadMessage()` +Spec 3.3: Hash verification must only run when challenge was completed. Currently +ChallengeHash is zero-valued when skipped, causing false mismatch on every +non-challenged message. + +### 22. Download attachment data +**File:** `host.go` `downloadMessage()` +Spec 3.2: Download sequential attachment byte sequences after message body, +bounded by attachment header sizes. Currently only message body is downloaded. + +### 23. Per-recipient user-duplicate check (code 103) +**File:** `host.go` `downloadMessage()` / `validateMsgRecvForAddr()` +Spec 3.4.i ordering: user duplicate (103) → unknown (100) → full (101) → +not accepting (102) → accept (200). Currently there is no check for whether +the message was already received for a specific recipient (code 103). + +--- + +## P4 — Sending (sender.go) + +### 24. Write actual attachment headers +**File:** `sender.go` `deliverMessage()` +Attachment count is hardcoded to 0. Must write attachment count + each +attachment header (flags, type, filename, size) from FMsgHeader.Attachments. + +### 25. Send attachment data after message body +**File:** `sender.go` `deliverMessage()` +Spec: sequential attachment byte sequences following data, bounded by header +sizes. Currently not sent. + +### 26. Implement exponential back-off for retries +**File:** `sender.go` `findPendingTargets()` +Spec says "SHOULD apply a back-off strategy." Currently uses a fixed +RetryInterval. + +### 27. Add code 101 (user full) to retryable set +**File:** `sender.go` `findPendingTargets()` +Per-user code 101 is analogous to global code 5 and is likely transient. +Spec says MAY warrant retry. + +--- + +## P5 — Storage (store.go / dd.sql) + +### 28. Store and load attachment metadata +**Files:** `store.go` `storeMsgDetail()` / `loadMsg()`, `dd.sql` msg_attachment + - dd.sql: msg_attachment is missing `flags` (uint8) and `type` (varchar) + columns needed for wire-format reconstruction and hash computation. + - storeMsgDetail: insert attachment headers into msg_attachment. + - loadMsg: load attachment rows into FMsgHeader.Attachments. + +### 29. Distinguish acceptance mode in stored messages +**File:** `store.go` / `dd.sql` +For item 16 (preventing add-to chaining), need a way to distinguish messages +accepted via code 200 (full message) from code 11 (header-only add-to +notification). Options: boolean column, separate lookup, or filepath="" check. + +--- + +## P6 — Address Validation (host.go) + +### 30. Support Unicode in address recipient part +**File:** `host.go` `isValidUser()` +Spec says recipient may contain Unicode letters/numbers (`\p{L}`, `\p{N}`). +Currently only ASCII a-z, A-Z, 0-9 are accepted. Must use Unicode-aware +checks (e.g. `unicode.IsLetter`, `unicode.IsNumber`). + +### 31. Enforce dot placement rules in address recipient part +**File:** `host.go` `isValidUser()` +Spec says dot `.` must not be consecutive and not at start/end. Currently dots +are allowed anywhere with no positional checks. + +--- + +## P7 — DNS / DNSSEC (dns.go) + +### 32. Perform DNSSEC validation +**Files:** `dns.go` `lookupAuthorisedIPs()`, `sender.go` +Spec: DNSSEC validation SHOULD be performed. If validation fails → connection +MUST terminate (no retry). Currently not performed or reported. + +--- + +## P8 — Handling a Challenge (host.go) + +### 33. Incoming challenge must send reject code 2 for unsupported version +**File:** `host.go` `readHeader()` / `handleChallenge()` +Spec "Handling a Challenge" step 1: if the first byte is not a supported +version and not a valid challenge, send reject code 2 (unsupported version) +and close. Currently handleChallenge is only entered for v==255 and other +unsupported values return an error without sending a code. diff --git a/src/defs.go b/src/defs.go index 87936bf..87e2245 100644 --- a/src/defs.go +++ b/src/defs.go @@ -17,6 +17,8 @@ type FMsgAddress struct { } type FMsgAttachmentHeader struct { + Flags uint8 + Type string Filename string Size uint32 @@ -35,11 +37,9 @@ type FMsgHeader struct { Type string // Size in bytes of entire message - Size uint32 - // TODO [Spec]: Add Attachments []FMsgAttachmentHeader field to store parsed - // attachment headers (flags, type, filename, size) from the wire format. + Size uint32 + Attachments []FMsgAttachmentHeader - // Hash up to and including Type HeaderHash []byte // Hash of message from challenge response ChallengeHash [32]byte @@ -58,16 +58,9 @@ func (addr *FMsgAddress) ToString() string { return fmt.Sprintf("@%s@%s", addr.User, addr.Domain) } -// Encode the header up to and including type field to a []byte. This function will panic on error -// instead of returning one. -// TODO [Spec]: The spec defines "message header" as all fields up to and -// including the attachment headers field. This Encode() is missing: -// - The "size" field (uint32). -// - The "attachment headers" field (uint8 count + list of attachment headers). -// -// The header hash (SHA-256 of the encoded header) will be incorrect without -// these fields, breaking challenge verification and pid references. -// Additionally, the topic field should only be encoded when pid is NOT set. +// Encode the message header to wire format as a []byte. This includes all +// fields up to and including the attachment headers per spec. This function +// will panic on error instead of returning one. func (h *FMsgHeader) Encode() []byte { var b bytes.Buffer b.WriteByte(h.Version) @@ -95,10 +88,29 @@ func (h *FMsgHeader) Encode() []byte { if err := binary.Write(&b, binary.LittleEndian, h.Timestamp); err != nil { panic(err) } - b.WriteByte(byte(len(h.Topic))) - b.WriteString(h.Topic) + // topic is only present when pid is NOT set + if h.Flags&FlagHasPid == 0 { + b.WriteByte(byte(len(h.Topic))) + b.WriteString(h.Topic) + } b.WriteByte(byte(len(h.Type))) b.WriteString(h.Type) + // size (uint32 LE) + if err := binary.Write(&b, binary.LittleEndian, h.Size); err != nil { + panic(err) + } + // attachment headers + b.WriteByte(byte(len(h.Attachments))) + for _, att := range h.Attachments { + b.WriteByte(att.Flags) + b.WriteByte(byte(len(att.Type))) + b.WriteString(att.Type) + b.WriteByte(byte(len(att.Filename))) + b.WriteString(att.Filename) + if err := binary.Write(&b, binary.LittleEndian, att.Size); err != nil { + panic(err) + } + } return b.Bytes() } @@ -153,17 +165,23 @@ func (h *FMsgHeader) GetMessageHash() ([]byte, error) { return nil, err } - // TODO [Spec]: Encode() is missing size, attachment count, and - // attachment headers. Once Encode() includes the full message header - // per spec, the size and attachment header bytes will automatically be - // included in this hash. - if _, err := io.Copy(hash, f); err != nil { return nil, err } - // TODO: include attachment data (sequential byte sequences following - // the message body, bounded by attachment header sizes) in the hash. + // include attachment data (sequential byte sequences following + // the message body, bounded by attachment header sizes) + for _, att := range h.Attachments { + af, err := os.Open(att.Filepath) + if err != nil { + return nil, fmt.Errorf("open attachment %s: %w", att.Filename, err) + } + if _, err := io.CopyN(hash, af, int64(att.Size)); err != nil { + af.Close() + return nil, fmt.Errorf("read attachment %s: %w", att.Filename, err) + } + af.Close() + } h.messageHash = hash.Sum(nil) } diff --git a/src/defs_test.go b/src/defs_test.go index 070176d..4dca209 100644 --- a/src/defs_test.go +++ b/src/defs_test.go @@ -96,6 +96,19 @@ func TestEncodeMinimalHeader(t *testing.T) { t.Fatalf("type = %q, want %q", string(typeBytes), "text/plain") } + // size (uint32 LE) + var size uint32 + binary.Read(r, binary.LittleEndian, &size) + if size != 0 { + t.Fatalf("size = %d, want 0", size) + } + + // attachment count + attachCount, _ := r.ReadByte() + if attachCount != 0 { + t.Fatalf("attach count = %d, want 0", attachCount) + } + // should have consumed entire buffer if r.Len() != 0 { t.Fatalf("unexpected %d trailing bytes", r.Len()) @@ -114,7 +127,7 @@ func TestEncodeWithPid(t *testing.T) { From: FMsgAddress{User: "a", Domain: "b.com"}, To: []FMsgAddress{{User: "c", Domain: "d.com"}}, Timestamp: 0, - Topic: "", + Topic: "should be omitted", Type: "text/plain", } b := h.Encode() @@ -132,6 +145,44 @@ func TestEncodeWithPid(t *testing.T) { if !bytes.Equal(pidOut, pid) { t.Fatalf("pid mismatch") } + + // skip from + fLen, _ := r.ReadByte() + fBuf := make([]byte, fLen) + r.Read(fBuf) + + // skip to count + to[0] + toCount, _ := r.ReadByte() + if toCount != 1 { + t.Fatalf("to count = %d, want 1", toCount) + } + tLen, _ := r.ReadByte() + tBuf := make([]byte, tLen) + r.Read(tBuf) + + // skip timestamp + var ts float64 + binary.Read(r, binary.LittleEndian, &ts) + + // topic must NOT be present when pid is set — next byte should be type length + typeLen, _ := r.ReadByte() + typeBytes := make([]byte, typeLen) + r.Read(typeBytes) + if string(typeBytes) != "text/plain" { + t.Fatalf("expected type field directly after timestamp, got %q", string(typeBytes)) + } + + // size + attachment count + var size uint32 + binary.Read(r, binary.LittleEndian, &size) + attachCount, _ := r.ReadByte() + if attachCount != 0 { + t.Fatalf("attach count = %d, want 0", attachCount) + } + + if r.Len() != 0 { + t.Fatalf("unexpected %d trailing bytes", r.Len()) + } } func TestEncodeWithAddTo(t *testing.T) { @@ -317,4 +368,111 @@ func TestEncodeTimestampEncoding(t *testing.T) { if ts != 1700000000.5 { t.Fatalf("timestamp = %f, want 1700000000.5", ts) } + + // After timestamp: topic(1+0) + type(1+0) + size(4) + attach_count(1) = 7 bytes + if r := bytes.NewReader(b[offset+8:]); r.Len() != 7 { + t.Fatalf("trailing bytes after timestamp = %d, want 7", r.Len()) + } +} + +func TestEncodeWithAttachments(t *testing.T) { + h := &FMsgHeader{ + Version: 1, + Flags: 0, + From: FMsgAddress{User: "a", Domain: "b.com"}, + To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + Timestamp: 0, + Topic: "", + Type: "text/plain", + Size: 100, + Attachments: []FMsgAttachmentHeader{ + {Flags: 0, Type: "image/png", Filename: "pic.png", Size: 2048}, + {Flags: 1, Type: "a", Filename: "doc.txt", Size: 512}, + }, + } + b := h.Encode() + r := bytes.NewReader(b) + + r.ReadByte() // version + r.ReadByte() // flags + + // skip from + fLen, _ := r.ReadByte() + r.Read(make([]byte, fLen)) + // skip to count + to[0] + r.ReadByte() + tLen, _ := r.ReadByte() + r.Read(make([]byte, tLen)) + // skip timestamp + var ts float64 + binary.Read(r, binary.LittleEndian, &ts) + // skip topic + topicLen, _ := r.ReadByte() + r.Read(make([]byte, topicLen)) + // skip type + typeLen, _ := r.ReadByte() + r.Read(make([]byte, typeLen)) + + // size + var size uint32 + binary.Read(r, binary.LittleEndian, &size) + if size != 100 { + t.Fatalf("size = %d, want 100", size) + } + + // attachment count + attachCount, _ := r.ReadByte() + if attachCount != 2 { + t.Fatalf("attach count = %d, want 2", attachCount) + } + + // attachment 0 + att0Flags, _ := r.ReadByte() + if att0Flags != 0 { + t.Fatalf("att[0] flags = %d, want 0", att0Flags) + } + att0TypeLen, _ := r.ReadByte() + att0Type := make([]byte, att0TypeLen) + r.Read(att0Type) + if string(att0Type) != "image/png" { + t.Fatalf("att[0] type = %q, want %q", string(att0Type), "image/png") + } + att0FnLen, _ := r.ReadByte() + att0Fn := make([]byte, att0FnLen) + r.Read(att0Fn) + if string(att0Fn) != "pic.png" { + t.Fatalf("att[0] filename = %q, want %q", string(att0Fn), "pic.png") + } + var att0Size uint32 + binary.Read(r, binary.LittleEndian, &att0Size) + if att0Size != 2048 { + t.Fatalf("att[0] size = %d, want 2048", att0Size) + } + + // attachment 1 + att1Flags, _ := r.ReadByte() + if att1Flags != 1 { + t.Fatalf("att[1] flags = %d, want 1", att1Flags) + } + att1TypeLen, _ := r.ReadByte() + att1Type := make([]byte, att1TypeLen) + r.Read(att1Type) + if string(att1Type) != "a" { + t.Fatalf("att[1] type = %q, want %q", string(att1Type), "a") + } + att1FnLen, _ := r.ReadByte() + att1Fn := make([]byte, att1FnLen) + r.Read(att1Fn) + if string(att1Fn) != "doc.txt" { + t.Fatalf("att[1] filename = %q, want %q", string(att1Fn), "doc.txt") + } + var att1Size uint32 + binary.Read(r, binary.LittleEndian, &att1Size) + if att1Size != 512 { + t.Fatalf("att[1] size = %d, want 512", att1Size) + } + + if r.Len() != 0 { + t.Fatalf("unexpected %d trailing bytes", r.Len()) + } } diff --git a/src/sender.go b/src/sender.go index 66b8ec2..d7e61c4 100644 --- a/src/sender.go +++ b/src/sender.go @@ -2,7 +2,6 @@ package main import ( "database/sql" - "encoding/binary" "encoding/hex" "fmt" "io" @@ -319,36 +318,13 @@ func deliverMessage(target pendingTarget) { } defer conn.Close() - // Send header (FMsgHeader.Encode writes version through type, including add-to) - // TODO [Spec]: Encode() always encodes the topic field even when pid IS set - // (spec: topic must be absent when pid is set). It also always encodes the - // type as a length-prefixed string, but spec says common-type flag (bit 2) - // means type is a single uint8 index. + // Send header — Encode() writes all fields through attachment headers per spec headerBytes := h.Encode() if _, err := conn.Write(headerBytes); err != nil { log.Printf("ERROR: sender: writing header: %s", err) return } - // TODO [Spec]: Encode() should include size and attachment headers as part - // of the header (spec defines header as all fields through attachment headers). - // Currently size and attachment count are written separately here, which is - // correct on the wire but means they are NOT included in the header hash. - - // size (uint32 LE) - if err := binary.Write(conn, binary.LittleEndian, h.Size); err != nil { - log.Printf("ERROR: sender: writing size: %s", err) - return - } - - // TODO [Spec]: Write actual attachment headers here. Each attachment header - // is: flags (uint8), type (uint8 + [ASCII string]), filename (uint8 + UTF-8), - // size (uint32). Currently hardcoded to 0 attachment count. - if err := binary.Write(conn, binary.LittleEndian, uint8(0)); err != nil { - log.Printf("ERROR: sender: writing attachment count: %s", err) - return - } - // message data fd, err := os.Open(h.Filepath) if err != nil { @@ -368,9 +344,24 @@ func deliverMessage(target pendingTarget) { return } - // TODO [Spec]: Send attachment bodies — sequential byte sequences following - // the message data, with boundaries determined by the attachment header sizes. - // Each attachment's data length must match the size declared in its header. + // attachment data — sequential byte sequences bounded by header sizes + for _, att := range h.Attachments { + af, err := os.Open(att.Filepath) + if err != nil { + log.Printf("ERROR: sender: opening attachment %s: %s", att.Filename, err) + return + } + an, err := io.CopyN(conn, af, int64(att.Size)) + af.Close() + if an != int64(att.Size) { + log.Printf("ERROR: sender: attachment %s size mismatch: expected %d, got %d", att.Filename, att.Size, an) + return + } + if err != nil { + log.Printf("ERROR: sender: sending attachment %s: %s", att.Filename, err) + return + } + } // --- read response --- // A code < 100 is a global rejection (single byte for all recipients). From 96455453490922e5a1d3417de7655df520e36342 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Sat, 4 Apr 2026 17:59:26 +0800 Subject: [PATCH 09/12] no topic to read when pid --- TODO2.md | 4 ++-- src/host.go | 16 +++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/TODO2.md b/TODO2.md index 371585c..ed6ce20 100644 --- a/TODO2.md +++ b/TODO2.md @@ -20,7 +20,7 @@ Encode() currently stops after the type field — it omits: The header hash (SHA-256 of the encoded header) will be incorrect without these fields, which breaks challenge verification and pid references. -### 2. Encode() must omit topic when pid is set +### ~~2. Encode() must omit topic when pid is set~~ DONE **File:** `defs.go` `Encode()` Spec: "When pid exists the entire topic field MUST NOT be included on the wire." Encode() always writes the topic. It must only write topic when pid is absent. @@ -73,7 +73,7 @@ returns an error without sending any code. **File:** `host.go` `readHeader()` Spec 1.4.i.a: If to count is 0, reject code 1 (invalid). Currently no check. -### 11. Make topic conditional on pid absence +### ~~11. Make topic conditional on pid absence~~ DONE **File:** `host.go` `readHeader()` Spec: topic field is only present when pid is NOT set. Currently topic is read unconditionally regardless of pid. diff --git a/src/host.go b/src/host.go index b4bf2fe..f4913dd 100644 --- a/src/host.go +++ b/src/host.go @@ -437,16 +437,14 @@ func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { return h, r, fmt.Errorf("message timestamp: %f too far in future, delta: %fs", h.Timestamp, delta) } - // TODO [Spec definition]: Topic is only present when pid is NOT set (first - // message in a thread). When pid exists the entire topic field MUST NOT be - // included on the wire. Currently topic is read unconditionally. - - // read topic - topic, err := ReadUInt8Slice(r) - if err != nil { - return h, r, err + // read topic — only present when pid is NOT set (first message in a thread) + if flags&FlagHasPid == 0 { + topic, err := ReadUInt8Slice(r) + if err != nil { + return h, r, err + } + h.Topic = string(topic) } - h.Topic = string(topic) // TODO [Spec 1.4.i.b / flag bit 2]: When the "common type" flag (bit 2) is // set, the type field is a single uint8 that maps to a predefined Media Type From 9a43cb678e2dfaedc036cef7b8f31b2337dcf1db Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Sat, 4 Apr 2026 18:11:10 +0800 Subject: [PATCH 10/12] added FMSG_SKIP_AUTHORISED_IPS for testing --- .env-example | 28 ---------------------------- README.md | 1 + src/host.go | 46 ++++++++++++++++++++++++++-------------------- 3 files changed, 27 insertions(+), 48 deletions(-) delete mode 100644 .env-example diff --git a/.env-example b/.env-example deleted file mode 100644 index e17dc45..0000000 --- a/.env-example +++ /dev/null @@ -1,28 +0,0 @@ -# fmsgd Environment Variables - -# Required -FMSG_DATA_DIR=/opt/fmsg/data -FMSG_DOMAIN=example.com -FMSG_ID_URL=http://127.0.0.1:8080 - -# Optional -FMSG_MAX_MSG_SIZE=10240 -FMSG_PORT=4930 -FMSG_MAX_PAST_TIME_DELTA=604800 -FMSG_MAX_FUTURE_TIME_DELTA=300 -FMSG_MIN_DOWNLOAD_RATE=5000 -FMSG_MIN_UPLOAD_RATE=5000 -FMSG_READ_BUFFER_SIZE=1600 -FMSG_RETRY_INTERVAL=20 -FMSG_RETRY_MAX_AGE=86400 -FMSG_POLL_INTERVAL=10 -FMSG_MAX_CONCURRENT_SEND=1024 -FMSG_SKIP_DOMAIN_IP_CHECK=false - -# PostgreSQL connection variables (see https://www.postgresql.org/docs/current/libpq-envars.html) -PGHOST= -PGPORT= -PGUSER= -PGPASSWORD= -PGDATABASE= -PGSSLMODE=disable \ No newline at end of file diff --git a/README.md b/README.md index e2ef674..fa905eb 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Tested with Go 1.25 on Linux and Windows, AMD64 and ARM | FMSG_POLL_INTERVAL | 10 | Seconds. How often the sender polls the database for pending messages. | | FMSG_MAX_CONCURRENT_SEND | 1024 | Maximum number of concurrent outbound message deliveries. | | FMSG_SKIP_DOMAIN_IP_CHECK | false | Set to "true" to skip verifying this host's external IP is in the _fmsg DNS authorised IP set on startup. | +| FMSG_SKIP_AUTHORISED_IPS | false | Set to "true" to skip verifying remote hosts IP is in the _fmsg DNS authorised IP set during message exchange. WARNING setting this true is effectivly disables sender verification. | diff --git a/src/host.go b/src/host.go index f4913dd..ddb8b8c 100644 --- a/src/host.go +++ b/src/host.go @@ -111,6 +111,7 @@ var MinDownloadRate float64 = 5000 var MinUploadRate float64 = 5000 var ReadBufferSize = 1600 var MaxMessageSize = uint32(1024 * 10) +var SkipAuthorisedIPs = false var DataDir = "got on startup" var Domain = "got on startup" var IDURI = "got on startup" @@ -127,6 +128,7 @@ func loadEnvConfig() { MinUploadRate = env.GetFloatDefault("FMSG_MIN_UPLOAD_RATE", 5000) ReadBufferSize = env.GetIntDefault("FMSG_READ_BUFFER_SIZE", 1600) MaxMessageSize = uint32(env.GetIntDefault("FMSG_MAX_MSG_SIZE", 1024*10)) + SkipAuthorisedIPs = os.Getenv("FMSG_SKIP_AUTHORISED_IPS") == "true" } // Updates DataDir from environment, panics if not a valid directory. @@ -569,29 +571,33 @@ func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { func challenge(conn net.Conn, h *FMsgHeader) error { // verify remote IP is authorised by sender's _fmsg DNS record - remoteHost, _, err := net.SplitHostPort(conn.RemoteAddr().String()) - if err != nil { - return fmt.Errorf("failed to parse remote address: %w", err) - } - remoteIP := net.ParseIP(remoteHost) - if remoteIP == nil { - return fmt.Errorf("failed to parse remote IP: %s", remoteHost) - } + if SkipAuthorisedIPs { + log.Println("WARN: skipping authorised IP check (FMSG_SKIP_AUTHORISED_IPS=true)") + } else { + remoteHost, _, err := net.SplitHostPort(conn.RemoteAddr().String()) + if err != nil { + return fmt.Errorf("failed to parse remote address: %w", err) + } + remoteIP := net.ParseIP(remoteHost) + if remoteIP == nil { + return fmt.Errorf("failed to parse remote IP: %s", remoteHost) + } - authorisedIPs, err := lookupAuthorisedIPs(h.From.Domain) - if err != nil { - return err - } + authorisedIPs, err := lookupAuthorisedIPs(h.From.Domain) + if err != nil { + return err + } - found := false - for _, ip := range authorisedIPs { - if remoteIP.Equal(ip) { - found = true - break + found := false + for _, ip := range authorisedIPs { + if remoteIP.Equal(ip) { + found = true + break + } + } + if !found { + return fmt.Errorf("remote address %s not in _fmsg.%s authorised IPs", remoteIP.String(), h.From.Domain) } - } - if !found { - return fmt.Errorf("remote address %s not in _fmsg.%s authorised IPs", remoteIP.String(), h.From.Domain) } // okay lets give sender a call and confirm they are sending this message From 240c474c9fabd25875f09a8d8e264d322a5062ae Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Sat, 4 Apr 2026 18:23:01 +0800 Subject: [PATCH 11/12] dial remote ip --- TODO2.md | 2 +- src/host.go | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/TODO2.md b/TODO2.md index ed6ce20..54fd073 100644 --- a/TODO2.md +++ b/TODO2.md @@ -124,7 +124,7 @@ referenced by pid." Not currently checked. ## P2 — Receiving: Challenge (host.go challenge) -### 18. Connection 2 must target same IP as Connection 1 +### ~~18. Connection 2 must target same IP as Connection 1~~ DONE **File:** `host.go` `challenge()` Spec 2.1: Dial conn.RemoteAddr() IP, not h.From.Domain. Dialling the domain may resolve to a different IP. diff --git a/src/host.go b/src/host.go index ddb8b8c..f08a4af 100644 --- a/src/host.go +++ b/src/host.go @@ -600,13 +600,12 @@ func challenge(conn net.Conn, h *FMsgHeader) error { } } - // okay lets give sender a call and confirm they are sending this message - // TODO [Spec 2.1]: Connection 2 MUST be initiated to the same IP address as - // the incoming Connection 1 (conn.RemoteAddr()), NOT to h.From.Domain. - // The spec says: "Host B MUST initiate a separate new connection (Connection 2) - // back to Host A using the same incoming IP address of Connection 1." - // Dialling h.From.Domain may resolve to a different IP. - conn2, err := net.Dial("tcp", net.JoinHostPort(h.From.Domain, fmt.Sprintf("%d", RemotePort))) + // Connection 2 MUST target the same IP as Connection 1 (spec 2.1). + remoteHost, _, err := net.SplitHostPort(conn.RemoteAddr().String()) + if err != nil { + return fmt.Errorf("failed to parse remote address for challenge: %w", err) + } + conn2, err := net.Dial("tcp", net.JoinHostPort(remoteHost, fmt.Sprintf("%d", RemotePort))) if err != nil { return err } From 5408b15846abb83324537bcd8fdcdcb7d1660a79 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Sat, 4 Apr 2026 18:38:37 +0800 Subject: [PATCH 12/12] save sha256 --- src/sender.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/sender.go b/src/sender.go index d7e61c4..6f36137 100644 --- a/src/sender.go +++ b/src/sender.go @@ -245,6 +245,19 @@ func deliverMessage(target pendingTarget) { return } + // Ensure sha256 is populated for outgoing messages so future pid lookups + // (e.g. add-to notifications referencing this message) can find it. + msgHash, err := h.GetMessageHash() + if err != nil { + log.Printf("ERROR: sender: computing message hash for msg %d: %s", target.MsgID, err) + return + } + if _, err := tx.Exec(`UPDATE msg SET sha256 = $1 WHERE id = $2 AND sha256 IS NULL`, + msgHash, target.MsgID); err != nil { + log.Printf("ERROR: sender: storing sha256 for msg %d: %s", target.MsgID, err) + return + } + // Register in outgoing map so challenge handler can look up this message hash := h.GetHeaderHash() hashArr := *(*[32]byte)(hash)