diff --git a/.gitignore b/.gitignore index 475d0f9..0077797 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vscode/ .claude/ *.exe -.env \ No newline at end of file +.env +fmsgd \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index a6a3fcb..56b3030 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,8 +9,7 @@ All code MUST conform to the specification. When in doubt, re-read SPEC.md and f ## Key Rules - Serialize and parse wire fields in the exact order defined in SPEC.md. -- Use the flag bit assignments from SPEC.md (bit 0 = has pid, bit 1 = has add to, bit 2 = common type, etc.). -- Enforce recipient uniqueness across both to and add to (case-insensitive). +- Use the flag bit assignments from SPEC.md (bit 0 = has pid, bit 1 = has add to, bit 2 = common type, etc.).im - Reject/accept response codes must match SPEC.md — do not invent new codes. - Resolve `_fmsg.` using A/AAAA records only (never TXT, MX, or SRV). - Validate sender IP before issuing CHALLENGE. diff --git a/SPEC.md b/SPEC.md index dd3db2c..44702ab 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,316 +1,287 @@ -# fmsg Protocol -- Compact SPEC - -## Protocol Identity - -- Protocol: fmsg (f-message) -- Current Version: 1 -- All messages are binary encoded. - ------------------------------------------------------------------------- - -## Wire Format (Field Order -- MUST NOT CHANGE) - -1. version (uint8) -- 1..127 = fmsg version; 129..255 = CHALLENGE (version = 256 - value); 0 and 128 are undefined -2. flags (uint8) -3. [pid] (32 byte SHA-256 hash) -- present only if flag bit 0 set -4. from (uint8 + UTF-8 address) -5. to (uint8 count + list of addresses, distinct case-insensitive, at least one) -6. [add to] (uint8 count + list of addresses) -- present only if flag bit 1 set, distinct case-insensitive, at least one -7. time (float64 POSIX timestamp, stamped by sender's host when message acquired) -8. [topic] (uint8 + UTF-8 string) -- present only when pid is NOT set, may be 0-length -9. type (uint8 common-type index when flag bit 2 set; otherwise uint8 length + ASCII MIME string per RFC 6838) -10. size (uint32, data size in bytes, 0 or greater) -11. attachment headers (uint8 count + list of attachment headers, count may be 0) -12. data (byte array, length from size field) -13. [attachments data] (sequential byte sequences, boundaries from attachment header sizes) - -All multi-byte integers are little-endian. -All strings are prefixed with uint8 length. -_case-insensitive_ means byte-wise equality after Unicode default case folding (locale-independent). -TERMINATE means tear down all connection(s) with the remote host immediately. - ------------------------------------------------------------------------- - -## Invariants - -- Topic is present only on the first message in a thread (when pid absent) and is immutable. -- When pid exists the entire topic field MUST NOT be included on the wire. -- Root messages MUST NOT include pid. -- Reply messages MUST include pid. -- All recipients in to MUST be distinct (case-insensitive). -- All recipients in add to MUST be distinct (case-insensitive). -- pid always references the previous message hash. -- When add to exists, pid MUST also exist. -- A sender (from) MUST have been a participant in the message referenced by pid. -- Data size MUST match the declared size field. -- Attachment count MUST fit within uint8 range. - ------------------------------------------------------------------------- - -## Flags (uint8 bitmask) - -Bit 0: has pid -- pid field is present\ -Bit 1: has add to -- add to field is included\ -Bit 2: common type -- type field is a uint8 index into Common Media Types\ -Bit 3: important -- sender indicates message is IMPORTANT\ -Bit 4: no reply -- sender indicates any reply will be discarded\ -Bit 5: deflate -- message data compressed with zlib/deflate (RFC 1950/1951)\ -Bit 6: TBD (reserved)\ -Bit 7: TBD (reserved) - ------------------------------------------------------------------------- - -## Address - -Format: `@recipient@domain` (leading `@` distinguishes from email). -Encoded as: uint8 size + UTF-8 string. - -Recipient part MUST be: UTF-8, Unicode letters/numbers (`\p{L}`, `\p{N}`), -hyphen `-`, underscore `_`, dot `.` non-consecutively and not at start/end, -unique on host (case-insensitive), combined address < 256 bytes. - ------------------------------------------------------------------------- - -## Attachment Header - -Each attachment header: flags (uint8), type (uint8 + [ASCII string]), filename (uint8 + UTF-8), size (uint32). - -### Attachment Flags - -Bit 0: common type -- type is uint8 index into Common Media Types\ -Bit 1: deflate -- data compressed with zlib/deflate\ -Bits 2-7: TBD (reserved) - -### Filename Rules - -- UTF-8, Unicode letters/numbers (`\p{L}`, `\p{N}`) -- Hyphen `-`, underscore `_`, single space ` `, dot `.` non-consecutively, not at start/end -- Unique amongst attachments (case-insensitive) -- Less than 256 bytes - ------------------------------------------------------------------------- - -## Domain Resolution (Authoritative Host Discovery) - -1. Resolve `_fmsg.` for A and AAAA records (follow CNAME chains). -2. Only A and AAAA records are authoritative. -3. DNSSEC validation SHOULD be performed. -4. If DNSSEC validation fails → connection MUST terminate. - -### Sender IP Verification - -Receiving Host B MUST resolve `_fmsg.` and verify the incoming -connection's IP is in the authorised set. If not authorised → TERMINATE -(no reject code, no challenge). - -Infrastructure MUST preserve the true originating IP address. - ------------------------------------------------------------------------- - -## Challenge / Response - -- CHALLENGE version byte: 255 = protocol v1, 254 = v2, etc. -- Challenge consists of: version (uint8) + header hash (32 bytes SHA-256 of message header up to and including attachment headers). -- Challenge response: msg hash (32 bytes SHA-256 of entire message). -- Receiver initiates Connection 2 to the **same IP** as Connection 1. -- Sender verifies header hash matches message being sent; if not → TERMINATE. -- Response hash MUST be kept and verified against computed hash once message fully downloaded. -- Whether to challenge is at the discretion of the recipient host (example modes: NEVER, ALWAYS, HAS_NOT_PARTICIPATED, DIFFERENT_DOMAIN). - ------------------------------------------------------------------------- - -## Reject or Accept Response - -A code less than 100 indicates rejection or acceptance for all recipients and -will be the only value. Codes ≥ 100 are per recipient in the order they appear -in _to_ then _add to_, excluding recipients for other domains. - -| name | type | comment | -|-------|------------|-------------------------------------| -| codes | byte array | a single or sequence of uint8 codes | - -| code | name | description | -|-----:|------------------------|--------------------------------------------------------------------| -| 1 | invalid | the message header fails verification checks, i.e. not in spec | -| 2 | unsupported version | the version is not supported by the receiving host | -| 3 | undisclosed | no reason is given | -| 4 | too big | total size exceeds host's maximum permitted size of messages | -| 5 | insufficient resources | such as disk space to store the message | -| 6 | parent not found | parent referenced by pid SHA-256 not found and is required | -| 7 | too old | timestamp is too far in the past for this host to accept | -| 8 | future time | timestamp is too far in the future for this host to accept | -| 9 | time travel | timestamp is before parent timestamp | -| 10 | duplicate | message has already been received for all recipients on this host | -| 11 | accept add to | additional recipients received | -| | | | -| 100 | user unknown | the recipient is unknown by this host | -| 101 | user full | insufficient resources for specific recipient | -| 102 | user not accepting | user is known but not accepting new messages at this time | -| 103 | user duplicate | message has already been received for this recipient | -| 105 | user undisclosed | no reason given (MAY be used instead of 100-103 to avoid disclosure)| -| | | | -| 200 | accept | message received for recipient | - ------------------------------------------------------------------------- - -## Verifying Message Stored - -A **message** is verified stored if: the SHA-256 digest exactly matches a -previously accepted (code 200) message, and that message currently exists and -can be retrieved. - -OR the SHA-256 digest exactly matches the digest computed over message bytes -with add to recipients included and add to flag set, previously accepted -(code 11, additional recipients received), and that message currently exists -and can be retrieved. - -NOTE: Multiple add to messages may arrive for the same pid, each with a -different batch of additional recipients. The host MUST record each batch -individually (not accumulate) so the exact message bytes can be reconstructed -per batch for hash verification. - ------------------------------------------------------------------------- - -## Protocol Steps Configuration - -| Variable | Example Value | Description | -|-----------------|---------------|---------------------------------------------------------------------------| -| MAX_SIZE | 1048576 | Maximum allowed total size in bytes (data + all attachment sizes) | -| MAX_MESSAGE_AGE | 700000 | Maximum age since message time for acceptance (seconds) | -| MAX_TIME_SKEW | 20 | Maximum tolerance for message time ahead of current time (seconds) | - ------------------------------------------------------------------------- - -## Protocol Steps Summary - -### 1. Connection and Header Exchange - -Host A connects to Host B via first responsive authorised IP from Domain Resolution. -Host A transmits the message. Host B reads the first byte to determine type -(version vs CHALLENGE), then downloads and verifies the remaining header. - -Verification includes: at least one recipient in to, recipient distinctness -(to and add to checked separately), at least one recipient for Host B's domain, -sender IP authorisation, size limits, time bounds, and common type mappings. - -pid/add-to rules: -- No pid, no add to → first message in thread, continue normally. -- pid exists, no add to → pid MUST be verified stored; parent time MUST be - before message time. NOTE: Verified stored checks the host has the parent, - not that every recipient still has it in their mailbox. Implementations - SHOULD consider restoring the parent to a recipient's mailbox if deleted. -- add to exists → pid MUST also exist, otherwise reject (code 1). - - If any add to recipients are for Host B → message download continues in - full (pid references previous message Host B might not have). - - If no add to recipients are for Host B → only the message header is - sent (no data/attachments). pid MUST match a message originally accepted - with code 200 (not code 11, preventing add to chaining); code 6 if not - found; parent time MUST be before message time (code 9). - Host B records the new add to recipients for future hash reconstruction, - responds code 11 (additional recipients received), closes connection. - NOTE: Implementations SHOULD consider restoring the referenced message to - a recipient's mailbox if previously deleted, so newly added recipients - have proper thread context. - -### 2. The Automatic Challenge - -Host B MAY challenge Host A (modes: NEVER, ALWAYS, HAS_NOT_PARTICIPATED, DIFFERENT_DOMAIN). -Host B MUST verify sender IP is in the authorised set for the from domain -**before** opening Connection 2. Host B opens Connection 2 to the **same IP** as -Connection 1, sends a CHALLENGE (version byte + message header hash). Host A -verifies the header hash matches its outgoing message; if not → TERMINATE. Host A -responds with the message hash. Host B keeps the response hash for later -verification. Both close Connection 2. - -HAS_NOT_PARTICIPATED is particularly important for messages with add to recipients -for Host B's domain — Host B may not have the parent referenced by pid and cannot -verify it is stored, making such messages indistinguishable from unsolicited ones -without a challenge. - -### 3. Integrity Verification, Per-Recipient Response and Disposition - -Before downloading remaining data: if challenge was completed, check for duplicate -via message hash → code 10. Host B downloads data + attachments. If challenge was -completed, verify computed hash matches challenge response → TERMINATE on mismatch. -Host B sends per-recipient ACCEPT/REJECT codes in _to_ then _add to_ order -(excluding other domains). For each recipient, check in order: already received -(103 user duplicate) → unknown (100) → exceeds quota (101 user full) → not -accepting (102) → otherwise accept (200). For any per-user REJECT, 105 -(user undisclosed) MAY be used instead. Global duplicate (code 10) is for the -entire message across all recipients. - -### 4. Sending a Message - -Host A determines unique recipient domains (excluding own). For each domain: -resolve authorised IPs, connect in order until responsive, register outgoing message -header hash, transmit complete message in wire format order, handle any incoming -CHALLENGE per Handling a Challenge, read response codes. Global code (<100) applies -to all recipients on that domain; per-recipient codes (≥100) arrive in _to_ then -_add to_ order. Host A records codes and retries transient failures -(3 undisclosed, 5 insufficient resources) with back-off. Permanent failures -(1 invalid, 2 unsupported version, 4 too big, 10 duplicate) are not retried. -Per-user codes: 101 (user full) MAY warrant retry; 100 (user unknown) or -103 (user duplicate) typically would not. - -### Handling a Challenge - -A sending host MUST be listening for incoming connections on the same IP address -it uses to send outgoing messages. While a message is being transmitted on -Connection 1, the receiving host may open Connection 2 back to the sending host to -issue a CHALLENGE. The sending host handles this as follows: - -1. Download the first byte: - - If the value is less than 128 and a supported fmsg version — this is an - incoming message and should be processed per 1. Connection and Header Exchange. - - If the value is greater than 128 and (256 - value) is a supported fmsg - version, this is a CHALLENGE; continue. - - Otherwise send REJECT code 2 (unsupported version) and close the connection. -2. Download the next 32 bytes — the header hash supplied by the challenger. -3. Verify the header hash exactly matches the message header hash of a message - currently being transmitted. If no match → TERMINATE. -4. Compute the message hash (SHA-256 of entire message) and transmit it as - CHALLENGE RESPONSE. -5. Close Connection 2. The message exchange continues on Connection 1. - -Host A MUST maintain a record of outgoing messages keyed by header hash, -created before transmission begins and removed after the exchange completes. -Implementors should be mindful of concurrent access to this record. - ------------------------------------------------------------------------- - -## Rejection Conditions (MUST Reject) - -- Cannot decode / malformed structure → TERMINATE -- No recipients in to → code 1 -- Duplicate recipients in to (case-insensitive) → code 1 -- Duplicate recipients in add to (case-insensitive) → code 1 -- No recipients in to or add to for receiving host's domain → code 1 -- Common type flag set but value has no mapping → code 1 -- Attachment common type flag set but value has no mapping → code 1 -- add to exists but pid does not → code 1 -- Unauthorised sender IP → TERMINATE -- DNSSEC validation failure → TERMINATE -- size + all attachment sizes > MAX_SIZE → code 4 -- Message too old (DELTA > MAX_MESSAGE_AGE) → code 7 -- Message too far in future (|DELTA| > MAX_TIME_SKEW) → code 8 -- pid parent not found (when required per add-to rules) → code 6 -- Parent time ≥ message time (when parent is required and found) → code 9 -- Duplicate message (via challenge hash lookup) → code 10 -- Hash mismatch after full download (challenge was completed) → TERMINATE - ------------------------------------------------------------------------- - -## Agent Enforcement Rules - -When generating or modifying code: - -- Always serialize and parse fields exactly in defined order. -- Never use TXT, MX, or SRV for host discovery. -- Always resolve `_fmsg.` using A/AAAA (with CNAME support). -- Enforce recipient uniqueness within to and within add to separately. -- Validate sender IP before issuing CHALLENGE. -- Terminate immediately on DNSSEC failure. -- Respect all flag semantics strictly (use spec bit assignments). -- Topic field only present when pid is absent. -- Common type flag (bit 2) controls type field encoding (single uint8 index vs length-prefixed string). -- Connection 2 for challenge MUST target the same IP as Connection 1. +# fmsg — Concise Implementation Specification + +_NOTE_ This is a distilled version of the full specification attempting to capture all neccssary information such that implementations following should be correct. More context may be needed at times to explain the logic, please refer to the full [fmsg SPECIFICATION](https://github.com/markmnl/fmsg/blob/main/SPECIFICATION.md). + +All integers are little-endian. "case-insensitive" means Unicode default case folding on UTF-8 strings. "TERMINATE" means tear down all connections with the remote host immediately. + +--- + +## 1. Addresses + +Format: `@recipient@domain` — UTF-8, prefixed by uint8 byte length. + +Recipient part: Unicode letters/numbers (`\p{L}`, `\p{N}`), plus `-` `_` `.` non-consecutively and not at start/end. Whole address < 256 bytes. + +## 2. Message Binary Format + +All fields are read sequentially. `[ ]` = conditionally present. + +| # | Field | Type | Condition / Notes | +|---|-------|------|-------------------| +| 1 | version | uint8 | 1–127 = message version. 129–255 = CHALLENGE (version = 256 − value). 0 and 128 unused. | +| 2 | flags | uint8 | Bit field, see §3. | +| 3 | [pid] | 32 bytes | SHA-256 of parent message. Present iff flag bit 0 set. | +| 4 | from | address | Sender address. | +| 5 | to | uint8 count + addresses | ≥ 1 distinct (case-insensitive) addresses. | +| 6 | [add to from] | address | Present iff flag bit 1 set. Must be in _from_ or _to_. | +| 7 | [add to] | uint8 count + addresses | Present iff flag bit 1 set. ≥ 1 distinct addresses. | +| 8 | time | float64 | POSIX epoch, stamped by sending host. | +| 9 | [topic] | uint8 length + UTF-8 | Present iff pid is NOT present. Length may be 0. | +| 10 | type | uint8 + [US-ASCII string] | If flag bit 2 (common type) set: uint8 is a Common Media Type ID (see §4). Otherwise: uint8 is length of subsequent ASCII Media Type string. | +| 11 | size | uint32 | Byte length of data on the wire (after compression, if zlib-deflate set). | +| 12 | attachment headers | uint8 count + [headers] | Count may be 0. Each header: see §5. | +| 13 | data | bytes | Exactly _size_ bytes. | +| 14 | [attachments data] | bytes | Concatenated attachment payloads, sizes defined by headers. | + +**Message header** = fields 1–12. **Message header hash** = SHA-256(message header). **Message hash** = SHA-256(entire message, fields 1–14). + +The hash MUST be computed over the full message bytes: message header fields exactly as transmitted, followed by message data and any attachments data. When the zlib-deflate flag is set for message data or an attachment's data, that data MUST be decompressed prior to inclusion in the hash computation. + +**Sender** = _from_ when _has add to_ not set; _add to from_ when set. + +**Participants** = all addresses in _from_, _to_, _add to from_ (if any), _add to_ (if any). + +## 3. Flags (uint8 bit field) + +| Bit | Name | Description | +|----:|------|-------------| +| 0 | has pid | pid field present; message is a reply. | +| 1 | has add to | add to from + add to fields present; message adds recipients to an existing message. | +| 2 | common type | type field is a 1-byte Common Media Type ID instead of length-prefixed string. | +| 3 | important | Sender flags message as important. | +| 4 | no reply | Sender will discard any reply. | +| 5 | zlib-deflate | Message data compressed with zlib/deflate (RFC 1950/1951). | +| 6–7 | reserved | Must be 0. | + +## 4. Common Media Types + +When flag bit 2 is set, the type field is a single uint8 mapping to: + +| ID | Media Type | ID | Media Type | +|----|------------|----|------------| +| 1 | application/epub+zip | 33 | image/avif | +| 2 | application/gzip | 34 | image/bmp | +| 3 | application/json | 35 | image/gif | +| 4 | application/msword | 36 | image/heic | +| 5 | application/octet-stream | 37 | image/jpeg | +| 6 | application/pdf | 38 | image/png | +| 7 | application/rtf | 39 | image/svg+xml | +| 8 | application/vnd.amazon.ebook | 40 | image/tiff | +| 9 | application/vnd.ms-excel | 41 | image/webp | +| 10 | application/vnd.ms-powerpoint | 42 | model/3mf | +| 11 | application/vnd.oasis.opendocument.presentation | 43 | model/gltf-binary | +| 12 | application/vnd.oasis.opendocument.spreadsheet | 44 | model/obj | +| 13 | application/vnd.oasis.opendocument.text | 45 | model/step | +| 14 | application/vnd.openxmlformats-officedocument.presentationml.presentation | 46 | model/stl | +| 15 | application/vnd.openxmlformats-officedocument.spreadsheetml.sheet | 47 | model/vnd.usdz+zip | +| 16 | application/vnd.openxmlformats-officedocument.wordprocessingml.document | 48 | text/calendar | +| 17 | application/x-tar | 49 | text/css | +| 18 | application/xhtml+xml | 50 | text/csv | +| 19 | application/xml | 51 | text/html | +| 20 | application/zip | 52 | text/javascript | +| 21 | audio/aac | 53 | text/markdown | +| 22 | audio/midi | 54 | text/plain;charset=US-ASCII | +| 23 | audio/mpeg | 55 | text/plain;charset=UTF-16 | +| 24 | audio/ogg | 56 | text/plain;charset=UTF-8 | +| 25 | audio/opus | 57 | text/vcard | +| 26 | audio/vnd.wave | 58 | video/H264 | +| 27 | audio/webm | 59 | video/H265 | +| 28 | font/otf | 60 | video/H266 | +| 29 | font/ttf | 61 | video/ogg | +| 30 | font/woff | 62 | video/VP8 | +| 31 | font/woff2 | 63 | video/VP9 | +| 32 | image/apng | 64 | video/webm | + +An unmapped ID MUST be rejected (code 1 invalid). + +## 5. Attachment Header + +Each attachment header, in order: + +| Field | Type | Notes | +|-------|------|-------| +| flags | uint8 | Bit 0 = common type (same lookup as §4). Bit 1 = zlib-deflate. Bits 2–7 reserved. | +| type | uint8 + [ASCII string] | Same encoding rule as message type, using this attachment's own common type flag. | +| filename | uint8 length + UTF-8 | < 256 bytes. Unicode letters/numbers, plus `-` `_` ` ` `.` non-consecutively, not at start/end. Unique per message (case-insensitive). | +| size | uint32 | Byte length of this attachment's data on the wire (after compression, if zlib-deflate set). | + +Attachment data payloads follow all headers, concatenated in order. + +## 6. Challenge (sent on Connection 2) + +| Field | Type | +|-------|------| +| version | uint8 | 255 = challenge for fmsg v1, 254 = v2, etc. | +| header hash | 32 bytes | SHA-256 of message header being challenged. | + +## 7. Challenge Response + +| Field | Type | +|-------|------| +| msg hash | 32 bytes | SHA-256 of entire message. | + +## 8. Response Codes + +Single-value codes (sent as first/only byte): + +| Code | Name | Meaning | +|-----:|------|---------| +| 1 | invalid | Message header fails validation. | +| 2 | unsupported version | Version not supported. | +| 3 | undisclosed | No reason given. | +| 4 | too big | Exceeds MAX_SIZE. | +| 5 | insufficient resources | e.g. disk full. | +| 6 | parent not found | pid references unknown message. | +| 7 | too old | Timestamp too far in past. | +| 8 | future time | Timestamp too far in future. | +| 9 | time travel | Timestamp before parent's timestamp. | +| 10 | duplicate | Already received for all recipients. | +| 11 | accept add to | Add-to accepted; parent already stored; no _add to_ recipients on this host. Stop. | +| 64 | continue | Header accepted; send data. | +| 65 | skip data | Add-to accepted; parent already stored; _add to_ recipients on this host. Skip data, per-recipient codes follow. | + +Per-recipient codes (one byte per recipient on this host, in message order): + +| Code | Name | Meaning | +|-----:|------|---------| +| 100 | user unknown | Address not known. | +| 101 | user full | Recipient quota exceeded. | +| 102 | user not accepting | Recipient not accepting messages. | +| 103 | user duplicate | Already received for this recipient. | +| 105 | user undisclosed | No reason disclosed. MAY replace any 100–103. | +| 200 | accept | Message stored for this recipient. | + +## 9. Domain Resolution + +Resolve `_fmsg.` for A/AAAA records. The sender's domain is: +- The domain of _add to from_ when _has add to_ is set. +- The domain of _from_ otherwise. + +The Receiving Host MUST verify the incoming connection IP is in the resolved set for the sender's domain. DNSSEC SHOULD be validated; on failure TERMINATE. + +## 10. Protocol Flow + +One message per connection. Two TCP connections used: Connection 1 (message transfer) and Connection 2 (optional challenge). Host A = Sending Host, Host B = Receiving Host. + +### 10.1 Host Configuration + +| Variable | Description | +|----------|-------------| +| MAX_SIZE | Max total bytes of data + attachment data. | +| MAX_MESSAGE_AGE | Max seconds a message time may be in the past. | +| MAX_TIME_SKEW | Max seconds a message time may be in the future. | + +### 10.2 Sending (Host A perspective) + +Host A delivers iff _from_ or _add to from_ belongs to Host A's domain. For each unique recipient domain: + +1. Resolve recipient domain IPs via `_fmsg.`. Connect to first responsive IP (Connection 1). Retry with backoff if unreachable. +2. Register the message header hash and Host B's IP in an outgoing record (for matching challenges). +3. Transmit the message header on Connection 1. +4. Wait for response. During this wait, be ready to handle a CHALLENGE on Connection 2 (see §10.5). +5. Read one byte from Connection 1: + - **1–10**: Rejected for all recipients. Record and close. + - **11**: Accept add-to (only valid when _has add to_ set). Record and close. + - **64**: Continue — transmit data + attachment data (exact declared sizes). + - **65**: Skip data (only valid when _has add to_ set). Do NOT transmit data. + - **Other**: TERMINATE. +6. After transmitting data (code 64) or immediately (code 65), read one byte per recipient on this host (in message field order: _to_ then _add to_). Each byte is a per-recipient response code. +7. Record results. Remove Host B's IP from outgoing record; remove entry if no IPs remain. Close Connection 1. + +### 10.3 Receiving — Header Exchange (Host B perspective) + +1. Read first byte on Connection 1: + - 1–127 and supported → message version, continue. + - 129–255 and (256 − value) supported → incoming CHALLENGE, handle per §10.5. + - Otherwise → respond code 2 (unsupported version), close. +2. Parse remaining header. If unparseable → TERMINATE. +3. Validate (all must pass, else respond code 1 invalid and close): + - _to_ has ≥ 1 distinct address. + - If _has add to_: _add to from_ exists and is in _from_ or _to_; _add to_ has ≥ 1 distinct address. + - ≥ 1 recipient in _to_ or _add to_ belongs to Host B's domain. + - Common type IDs (message and attachment) are mapped. +4. DNS-verify sender IP: resolve `_fmsg.`, check Connection 1 source IP is in result set. Fail → TERMINATE. +5. If _size_ + attachment sizes > MAX_SIZE → respond code 4, close. +6. Compute DELTA = now − _time_: + - DELTA > MAX_MESSAGE_AGE → respond code 7, close. + - DELTA < −MAX_TIME_SKEW → respond code 8, close. +7. Evaluate pid / add-to: + - **No pid, no add-to** (new thread): respond 64 (continue). + - **pid set, no add-to** (reply): + - Verify parent stored (§11). Not found → respond code 6, close. + - Parent time − MAX_TIME_SKEW must be before incoming time. Fail → respond code 9, close. + - _from_ must be a participant of the parent. Fail → respond code 1, close. + - Respond 64 (continue). + - **add-to set** (adding recipients): + - pid MUST also be set. Fail → respond code 1, close. + - Check if parent stored (§11): + - **Stored**: check time travel (code 9 if fail). + - If any _add to_ recipient belongs to Host B's domain → respond 65 (skip data), then per-recipient codes per §10.4. + - Otherwise → record add-to fields, respond 11 (accept add to), close. + - **Not stored**: respond 64 (continue) — treat as full message delivery. + +### 10.4 Receiving — Data Download and Per-Recipient Response + +1. If challenge was completed, use the message hash from the challenge response to check for duplicates across all recipients on Host B. If duplicate for all → respond code 10, close. +2. If code 65 was sent, skip to step 4 (data already stored). Otherwise download data + attachments (exactly declared sizes). +3. If challenge was completed, verify computed message hash matches the challenge response hash. For code 65, compute from received header + stored data. Mismatch → TERMINATE. +4. For each recipient on Host B's domain (in _to_ order, then _add to_ order), send one response byte: + - Already received → 103 (or 105). + - Unknown address → 100 (or 105). + - Quota exceeded → 101 (or 105). + - Not accepting → 102 (or 105). + - Otherwise → 200 (accept). +5. Close Connection 1. + +### 10.5 Challenge Flow + +The challenge is optional (Receiving Host's discretion). It runs on a separate Connection 2 while Connection 1 is paused after the header exchange. + +**Receiving Host (Host B) initiates:** +1. Open Connection 2 to Host A's IP (the source IP of Connection 1). DNS verification of sender IP must already have passed. +2. Send CHALLENGE: version byte + 32-byte message header hash. + +**Sending Host (Host A) handles:** +1. Read first byte on incoming connection: + - 1–127 → incoming message, handle normally. + - 129–255 → CHALLENGE, continue. + - Other → TERMINATE connection. +2. Read 32-byte header hash. Match against outgoing record by header hash AND challenger's IP. No match → TERMINATE. +3. Send CHALLENGE RESPONSE: 32-byte SHA-256 of entire message. + +**Host B receives** the 32-byte message hash from Host A. Both close Connection 2. Exchange continues on Connection 1. + +Host A MUST maintain a record of outgoing messages keyed by message header hash, including each Receiving Host's IP. Used to match challenges and verify the challenger's IP. Create before transmission; remove a host's IP on completion/abort; remove the entry when no IPs remain. + +## 11. Verifying Message Stored + +A message is verified as stored iff: +- A SHA-256 digest matches a previously accepted message (code 200 or 11). +- That message currently exists and is retrievable. + +For accept-add-to (code 11) messages, the hash is computed by combining the add-to message header with the original message's data and attachment data. + +Each add-to batch produces a distinct hash. Only the exact batch that had an accepted response (200 or 11) matches. + +## 12. Adding Recipients + +An add-to message is a duplicate of the original message with these differences: +- Flag bit 1 (_has add to_) set. +- _pid_ = hash of the message being added to. +- _add to from_ = participant initiating the add (must be in original _from_ or _to_). +- _add to_ = new recipient addresses. +- _time_ = new timestamp. +- _topic_ is NOT present (pid is set). + +## 13. Security Requirements + +- Enforce MAX_SIZE before downloading data. +- Enforce per-connection and per-IP rate limits. +- Apply idle/slow-connection timeouts. +- Verify sender IP via DNS BEFORE issuing any challenge. +- Rate-limit outgoing challenge connections. +- Use DNSSEC where supported. Fail → TERMINATE. +- Track accepted message hashes to reject duplicates. +- Support per-user storage quotas. +- Use code 105 (user undisclosed) to prevent sender enumeration. +- Log: timestamp, source IP, sender domain, response codes, challenge outcomes, termination reasons. diff --git a/TODO.md b/TODO.md deleted file mode 100644 index e700774..0000000 --- a/TODO.md +++ /dev/null @@ -1,191 +0,0 @@ -# TODO - -Items ordered by perceived priority — foundational / blocking items first, then -correctness issues, then enhancements. - ---- - -## P0 — Foundational (blocks most other work) - - -### 4. Add `Attachments` field to `FMsgHeader` -**File:** `defs.go` struct -Add `Attachments []FMsgAttachmentHeader` to store parsed attachment headers -(flags, type, filename, size). Required before attachment parsing, encoding, -size checks, or hash computation can be correct. - -### 5. Add `ChallengeCompleted` sentinel to `FMsgHeader` -**File:** `defs.go` struct -Add a `ChallengeCompleted bool` to distinguish "challenge was completed and -`ChallengeHash` is valid" from "challenge was not performed." Without this the -hash verification check in `downloadMessage` erroneously fails when the -challenge was skipped. - ---- - -## P1 — Receiving path (host.go) correctness - -### 7. DNS-verify sender IP during header exchange, not just in challenge -**File:** `host.go` `readHeader()` -Spec 1.4.ii: Host B MUST resolve `_fmsg.` and verify the incoming -connection's IP is authorised. If not → TERMINATE (no reject code). Currently -this only happens inside `challenge()` and is skipped when skip-challenge is -allowed. - - -### 9. Make topic conditional on pid absence -**File:** `host.go` `readHeader()` -Topic field is only present when pid is NOT set. Currently read unconditionally. - -### 10. Handle common-type flag for type field -**File:** `host.go` `readHeader()` -When common-type flag (bit 2) is set, type is a single uint8 index into the -Common Media Types table. If value has no mapping → reject code 1. - -### 11. Parse attachment headers -**File:** `host.go` `readHeader()` -Parse each attachment header (flags, type, filename, size). Validate filenames, -common-type mappings, uniqueness. Incorporate attachment sizes into MAX_SIZE -check. Currently rejects any non-zero attachment count. - -### 12. Move pid verification to header exchange -**File:** `host.go` `readHeader()` / `downloadMessage()` -Spec 1.4.v.c: When pid exists (with or without add-to), verify stored per -"Verifying Message Stored" (code 200 or 11, data retrievable). Check parent -time < message time (else code 9). Currently deferred to `downloadMessage`. - -### 13. Pre-download duplicate check using challenge hash -**File:** `host.go` `downloadMessage()` -Spec 3.1: Before downloading data, if challenge was completed, use the message -hash to check for duplicates → code 10. Currently duplicate check is after -download, wasting bandwidth. - -### 14. Guard hash verification behind ChallengeCompleted -**File:** `host.go` `downloadMessage()` -Spec 3.3: Hash verification must only run when challenge was completed. -Currently `ChallengeHash` is zero-valued when skipped, causing false mismatch. - -### 15. Download attachment data -**File:** `host.go` `downloadMessage()` -Spec 3.2: Download sequential attachment byte sequences after message body, -bounded by attachment header sizes. Currently only message body is downloaded. - -### 18. Validate at least one "to" recipient -**File:** `host.go` `readHeader()` -Spec 1.4.i.a: If to count is 0, reject code 1 (invalid). - -### 19. Send reject code 2 for unsupported version -**File:** `host.go` `readHeader()` -Spec 1.3.iii: Send code 2 on the connection before closing when version is -unsupported. Currently just returns an error. - -### 20. Generalise challenge version byte handling -**File:** `host.go` `readHeader()` -Any value > 128 where (256 - value) is a supported version is a challenge. -Currently only v==255 is handled. - ---- - -## P2 — Sending path (sender.go) correctness - -### ~~21. Include add-to recipients in domain recipient list~~ DONE -**File:** `sender.go` `deliverMessage()` -`domainRecips` already iterates both `h.To` and `h.AddTo` in order, with -`isAddTo` flag set appropriately. Implemented as part of the add-to feature. - -### 22. Write attachment headers and attachment bodies -**File:** `sender.go` `deliverMessage()` -Attachment count is hardcoded to 0. Write actual attachment headers (flags, -type, filename, size) and send attachment data after message body. - -### 24. Handle missing per-user codes 102 and 103 -**File:** `sender.go` `deliverMessage()` -Code 102 (user not accepting) may be transient — consider retry. Code 103 (user -undisclosed) should be handled similarly to code 3. - ---- - -## P3 — Challenge path correctness - -### 25. Connection 2 must target same IP as Connection 1 -**File:** `host.go` `challenge()` -Spec 2.1: Dial `conn.RemoteAddr()` IP, not `h.From.Domain`. Dialling the domain -may resolve to a different IP. - -### 26. Replace FlagSkipChallenge with implementation-defined challenge mode -**File:** `host.go` `challenge()` -Spec defines challenge modes (NEVER, ALWAYS, HAS_NOT_PARTICIPATED, -DIFFERENT_DOMAIN) as Host B's choice. `FlagSkipChallenge` is sender-controlled -and not part of the spec. - ---- - -## P4 — Storage layer - -### 28. Store and load attachment metadata -**File:** `store.go` `storeMsgDetail()` / `loadMsg()` -Insert attachment headers into `msg_attachment` and load them back into -`FMsgHeader.Attachments` so the sender can write them on the wire and hashing is -correct. - ---- - -## P5 — Hash computation - -### 30. Include size + attachment headers in header hash -**File:** `defs.go` `Encode()` / `GetHeaderHash()` -Once `Encode()` is fixed (item 2), the hash via `GetHeaderHash()` will -automatically be correct. - -### 31. Include attachment data in message hash -**File:** `defs.go` `GetMessageHash()` -Append sequential attachment byte sequences (bounded by header sizes) after the -message body data in the hash computation. - ---- - -## P6 — Retry / delivery robustness - -### 32. Implement exponential back-off for retries -**File:** `sender.go` `findPendingTargets()` -Spec says "SHOULD apply a back-off strategy." Currently uses a fixed -`RetryInterval`. - -### 33. Add code 101 (user full) to retryable set -**File:** `sender.go` `findPendingTargets()` -Per-user code 101 is analogous to global code 5 (insufficient resources) and is -likely transient. - -### 34. Clarify behaviour for unlocked recipients in per-recipient responses -**File:** `sender.go` `deliverMessage()` -When some recipients on a domain are already delivered or locked by another -sender, do we still get per-recipient codes for them? Need to decide whether -to update response codes for non-locked addresses. - ---- - -## P7 — DNS / network - -### 35. Perform DNSSEC validation -**File:** `dns.go` / `sender.go` -Spec: DNSSEC validation SHOULD be performed. If validation fails → TERMINATE -(no retry). `lookupAuthorisedIPs` does not currently perform or report DNSSEC -status. - ---- - -## P8 — Operational - -### 36. Ping ID URL on startup -**File:** `host.go` startup -Verify the FMSG_ID_URL is up and responding in a timely manner before accepting -connections. - ---- - -## Legacy questions (from original TODO.md) - -- Should `time_delivered` be on the whole msg since either wholly accepted or - not? -- What when one recipient accepted and another did not — resend to domain — but - then will get duplicate? diff --git a/TODO2.md b/TODO2.md deleted file mode 100644 index 54fd073..0000000 --- a/TODO2.md +++ /dev/null @@ -1,240 +0,0 @@ -# TODO2 - -Gap analysis of the codebase against SPEC.md. - ---- - -## P0 — Wire Format / Encoding (defs.go) - -These are foundational: the header hash (used for challenge verification and pid -references) and message hash (used for duplicate detection, challenge response, -and pid) are both wrong until Encode() is complete. - -### ~~1. Encode() must include size and attachment headers~~ DONE -**File:** `defs.go` `Encode()` -Spec defines "message header" as all fields through the attachment headers. -Encode() currently stops after the type field — it omits: - - size (uint32 LE) - - attachment count (uint8) + attachment headers (flags, type, filename, size) - -The header hash (SHA-256 of the encoded header) will be incorrect without these -fields, which breaks challenge verification and pid references. - -### ~~2. Encode() must omit topic when pid is set~~ DONE -**File:** `defs.go` `Encode()` -Spec: "When pid exists the entire topic field MUST NOT be included on the wire." -Encode() always writes the topic. It must only write topic when pid is absent. - -### 3. Encode() must support common-type encoding for type field -**File:** `defs.go` `Encode()` -When common-type flag (bit 2) is set, type on the wire is a single uint8 index, -not a length-prefixed string. Encode() always writes length-prefixed. - -### ~~4. Add Attachments field to FMsgHeader~~ DONE -**File:** `defs.go` struct -Add `Attachments []FMsgAttachmentHeader` to store parsed attachment headers. -Required before Encode() can include attachment headers, before attachment -parsing/validation/download, and before hash computation is correct. - -### ~~5. Complete FMsgAttachmentHeader struct~~ DONE -**File:** `defs.go` struct -Currently only has Filename, Size, Filepath. Missing per-spec fields: - - Flags (uint8) — including per-attachment common-type (bit 0) and deflate (bit 1) - - Type (string) — the attachment's media type - -### 6. Add ChallengeCompleted flag to FMsgHeader -**File:** `defs.go` struct -Add `ChallengeCompleted bool` to distinguish "challenge was completed and -ChallengeHash is valid" from "challenge was not performed." Without this, the -hash check in downloadMessage erroneously fails when the challenge was skipped. - -### ~~7. GetMessageHash() must include attachment data~~ DONE -**File:** `defs.go` `GetMessageHash()` -Spec: message hash is SHA-256 of the entire message — header + data + -attachment data. Currently attachment data (sequential byte sequences following -the message body) is not included. - ---- - -## P1 — Receiving: Header Exchange (host.go readHeader) - -### 8. Generalise first-byte version/challenge detection -**File:** `host.go` `readHeader()` -Spec step 1.3: 1..127 = version, 129..255 = CHALLENGE (version = 256 − value), -0 and 128 are undefined. Currently only v==255 is handled as a challenge. Must -handle any value > 128 where (256 − value) is a supported version. - -### 9. Send reject code 2 for unsupported version -**File:** `host.go` `readHeader()` -Spec 1.3.iii: Send code 2 on the connection before closing. Currently just -returns an error without sending any code. - -### 10. Validate at least one "to" recipient -**File:** `host.go` `readHeader()` -Spec 1.4.i.a: If to count is 0, reject code 1 (invalid). Currently no check. - -### ~~11. Make topic conditional on pid absence~~ DONE -**File:** `host.go` `readHeader()` -Spec: topic field is only present when pid is NOT set. Currently topic is -read unconditionally regardless of pid. - -### 12. Handle common-type flag for type field -**File:** `host.go` `readHeader()` -When common-type flag (bit 2) is set, type is a single uint8 index into the -Common Media Types table. If the value has no mapping → reject code 1. -Currently always reads type as a length-prefixed string. - -### 13. Parse and validate attachment headers -**File:** `host.go` `readHeader()` -Currently rejects any non-zero attachment count. Must: - - Parse each header: flags (uint8), type, filename (uint8 + UTF-8), size (uint32). - - Validate per-attachment common-type flag mapping (reject code 1 if unmapped). - - Validate filenames per spec (Unicode letters/numbers, limited special chars, - no consecutive dots, not at start/end, unique case-insensitive, < 256 bytes). - - Include all attachment sizes in MAX_SIZE check (data size + Σ attachment sizes). - - Store parsed headers on FMsgHeader.Attachments. - -### 14. DNS-verify sender IP during header exchange -**File:** `host.go` `readHeader()` -Spec 1.4.ii / "Sender IP Verification": Host B MUST resolve -`_fmsg.` and verify the incoming connection's IP is in the -authorised set. If not authorised → TERMINATE (no reject code, no challenge). -Currently this check only happens inside challenge() and is skipped when -challenge is not performed. - -### 15. Perform pid verification during header exchange -**File:** `host.go` `readHeader()` -Spec 1.4.v.c: When pid exists and add to does not: - a. pid must be verified stored per "Verifying Message Stored"; else code 6. - b. Parent time must be before message time; else code 9 (time travel). -Currently deferred to downloadMessage(). - -### 16. Distinguish code-200 vs code-11 parents for add-to header-only path -**File:** `host.go` `readHeader()` + `store.go` -Spec 1.4.v.b.a (no add-to recipients for our domain): pid MUST match a message -originally accepted with code 200 (not code 11), preventing add-to chaining. -Currently lookupMsgIdByHash does not distinguish how the message was accepted. - -### 17. Verify sender participated in parent message -**File:** `host.go` `readHeader()` -Spec invariant: "A sender (from) MUST have been a participant in the message -referenced by pid." Not currently checked. - ---- - -## P2 — Receiving: Challenge (host.go challenge) - -### ~~18. Connection 2 must target same IP as Connection 1~~ DONE -**File:** `host.go` `challenge()` -Spec 2.1: Dial conn.RemoteAddr() IP, not h.From.Domain. Dialling the domain -may resolve to a different IP. - -### 19. Make challenge mode configurable -**File:** `host.go` `challenge()` / `handleConn()` -Spec defines challenge modes (NEVER, ALWAYS, HAS_NOT_PARTICIPATED, -DIFFERENT_DOMAIN) as Host B's implementation choice. Currently always challenges. -At minimum, support skipping the challenge so the ChallengeCompleted guard -(item 6) works correctly. - ---- - -## P3 — Receiving: Download and Response (host.go downloadMessage) - -### 20. Pre-download duplicate check via challenge hash -**File:** `host.go` `downloadMessage()` -Spec 3.1: BEFORE downloading data, if challenge was completed, use the message -hash to check for duplicate → code 10. Currently the duplicate check is after -download, wasting bandwidth. - -### 21. Guard hash verification behind ChallengeCompleted -**File:** `host.go` `downloadMessage()` -Spec 3.3: Hash verification must only run when challenge was completed. Currently -ChallengeHash is zero-valued when skipped, causing false mismatch on every -non-challenged message. - -### 22. Download attachment data -**File:** `host.go` `downloadMessage()` -Spec 3.2: Download sequential attachment byte sequences after message body, -bounded by attachment header sizes. Currently only message body is downloaded. - -### 23. Per-recipient user-duplicate check (code 103) -**File:** `host.go` `downloadMessage()` / `validateMsgRecvForAddr()` -Spec 3.4.i ordering: user duplicate (103) → unknown (100) → full (101) → -not accepting (102) → accept (200). Currently there is no check for whether -the message was already received for a specific recipient (code 103). - ---- - -## P4 — Sending (sender.go) - -### 24. Write actual attachment headers -**File:** `sender.go` `deliverMessage()` -Attachment count is hardcoded to 0. Must write attachment count + each -attachment header (flags, type, filename, size) from FMsgHeader.Attachments. - -### 25. Send attachment data after message body -**File:** `sender.go` `deliverMessage()` -Spec: sequential attachment byte sequences following data, bounded by header -sizes. Currently not sent. - -### 26. Implement exponential back-off for retries -**File:** `sender.go` `findPendingTargets()` -Spec says "SHOULD apply a back-off strategy." Currently uses a fixed -RetryInterval. - -### 27. Add code 101 (user full) to retryable set -**File:** `sender.go` `findPendingTargets()` -Per-user code 101 is analogous to global code 5 and is likely transient. -Spec says MAY warrant retry. - ---- - -## P5 — Storage (store.go / dd.sql) - -### 28. Store and load attachment metadata -**Files:** `store.go` `storeMsgDetail()` / `loadMsg()`, `dd.sql` msg_attachment - - dd.sql: msg_attachment is missing `flags` (uint8) and `type` (varchar) - columns needed for wire-format reconstruction and hash computation. - - storeMsgDetail: insert attachment headers into msg_attachment. - - loadMsg: load attachment rows into FMsgHeader.Attachments. - -### 29. Distinguish acceptance mode in stored messages -**File:** `store.go` / `dd.sql` -For item 16 (preventing add-to chaining), need a way to distinguish messages -accepted via code 200 (full message) from code 11 (header-only add-to -notification). Options: boolean column, separate lookup, or filepath="" check. - ---- - -## P6 — Address Validation (host.go) - -### 30. Support Unicode in address recipient part -**File:** `host.go` `isValidUser()` -Spec says recipient may contain Unicode letters/numbers (`\p{L}`, `\p{N}`). -Currently only ASCII a-z, A-Z, 0-9 are accepted. Must use Unicode-aware -checks (e.g. `unicode.IsLetter`, `unicode.IsNumber`). - -### 31. Enforce dot placement rules in address recipient part -**File:** `host.go` `isValidUser()` -Spec says dot `.` must not be consecutive and not at start/end. Currently dots -are allowed anywhere with no positional checks. - ---- - -## P7 — DNS / DNSSEC (dns.go) - -### 32. Perform DNSSEC validation -**Files:** `dns.go` `lookupAuthorisedIPs()`, `sender.go` -Spec: DNSSEC validation SHOULD be performed. If validation fails → connection -MUST terminate (no retry). Currently not performed or reported. - ---- - -## P8 — Handling a Challenge (host.go) - -### 33. Incoming challenge must send reject code 2 for unsupported version -**File:** `host.go` `readHeader()` / `handleChallenge()` -Spec "Handling a Challenge" step 1: if the first byte is not a supported -version and not a valid challenge, send reject code 2 (unsupported version) -and close. Currently handleChallenge is only entered for v==255 and other -unsupported values return an error without sending a code. diff --git a/dd.sql b/dd.sql index 8e05dc9..19325e4 100644 --- a/dd.sql +++ b/dd.sql @@ -15,6 +15,7 @@ create table if not exists msg ( is_deflate boolean not null default false, time_sent double precision, -- time sending host recieved message for sending, message timestamp field, NULL means message not ready for sending i.e. draft from_addr varchar(255) not null, + add_to_from varchar(255), topic varchar(255) not null, type varchar(255) not null, sha256 bytea unique, @@ -31,6 +32,7 @@ create table if not exists msg_to ( time_delivered double precision, -- if sending, time sending host recieved delivery confirmation, if receiving, time successfully received message time_last_attempt double precision, -- only used when sending, time of last delivery attempt if failed; otherwise null response_code smallint, -- only used when sending, response code of last delivery attempt if failed; otherwise null + attempt_count int not null default 0, -- number of failed delivery attempts; used for exponential back-off unique (msg_id, addr) ); create index on msg_to ((lower(addr))); @@ -42,12 +44,16 @@ create table if not exists msg_add_to ( time_delivered double precision, -- if sending, time sending host recieved delivery confirmation, if receiving, time successfully received message time_last_attempt double precision, -- only used when sending, time of last delivery attempt if failed; otherwise null response_code smallint, -- only used when sending, response code of last delivery attempt if failed; otherwise null + attempt_count int not null default 0, -- number of failed delivery attempts; used for exponential back-off unique (msg_id, addr) ); create index on msg_add_to ((lower(addr))); create table if not exists msg_attachment ( msg_id bigint references msg (id), + position smallint not null default 0, + flags smallint not null default 0, + type varchar(255) not null default 'application/octet-stream', filename varchar(255) not null, filesize int not null, filepath text not null, diff --git a/src/defs.go b/src/defs.go index 87e2245..a44733c 100644 --- a/src/defs.go +++ b/src/defs.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "compress/zlib" "crypto/sha256" "encoding/binary" "encoding/hex" @@ -18,6 +19,7 @@ type FMsgAddress struct { type FMsgAttachmentHeader struct { Flags uint8 + TypeID uint8 Type string Filename string Size uint32 @@ -31,8 +33,10 @@ type FMsgHeader struct { Pid []byte From FMsgAddress To []FMsgAddress + AddToFrom *FMsgAddress // Present when has-add-to flag is set AddTo []FMsgAddress Timestamp float64 + TypeID uint8 Topic string Type string @@ -40,17 +44,12 @@ type FMsgHeader struct { Size uint32 Attachments []FMsgAttachmentHeader - HeaderHash []byte - // Hash of message from challenge response - ChallengeHash [32]byte - // TODO [Spec]: Add a ChallengeCompleted bool (or similar sentinel) to - // distinguish "challenge was completed and ChallengeHash is valid" from - // "challenge was not performed and ChallengeHash is zero-valued". - // Absolute filepath set when downloaded - Filepath string - - // Actual hash of message data including any attachments - messageHash []byte + HeaderHash []byte + ChallengeHash [32]byte + ChallengeCompleted bool // True if challenge was initiated and completed + InitialResponseCode uint8 // Protocol response chosen after header validation (64/65) + Filepath string + messageHash []byte } // Returns a string representation of an address in the form @user@example.com @@ -78,6 +77,15 @@ func (h *FMsgHeader) Encode() []byte { b.WriteString(str) } if h.Flags&FlagHasAddTo != 0 { + // add-to-from address (field 6) + addToFrom := h.AddToFrom + if addToFrom == nil { + addToFrom = &h.From + } + str := addToFrom.ToString() + b.WriteByte(byte(len(str))) + b.WriteString(str) + // add-to addresses (field 7) b.WriteByte(byte(len(h.AddTo))) for _, addr := range h.AddTo { str = addr.ToString() @@ -93,8 +101,18 @@ func (h *FMsgHeader) Encode() []byte { b.WriteByte(byte(len(h.Topic))) b.WriteString(h.Topic) } - b.WriteByte(byte(len(h.Type))) - b.WriteString(h.Type) + if h.Flags&FlagCommonType != 0 { + typeID := h.TypeID + if typeID == 0 { + if id, ok := getCommonMediaTypeID(h.Type); ok { + typeID = id + } + } + b.WriteByte(typeID) + } else { + b.WriteByte(byte(len(h.Type))) + b.WriteString(h.Type) + } // size (uint32 LE) if err := binary.Write(&b, binary.LittleEndian, h.Size); err != nil { panic(err) @@ -103,8 +121,18 @@ func (h *FMsgHeader) Encode() []byte { b.WriteByte(byte(len(h.Attachments))) for _, att := range h.Attachments { b.WriteByte(att.Flags) - b.WriteByte(byte(len(att.Type))) - b.WriteString(att.Type) + if att.Flags&1 != 0 { + typeID := att.TypeID + if typeID == 0 { + if id, ok := getCommonMediaTypeID(att.Type); ok { + typeID = id + } + } + b.WriteByte(typeID) + } else { + b.WriteByte(byte(len(att.Type))) + b.WriteString(att.Type) + } b.WriteByte(byte(len(att.Filename))) b.WriteString(att.Filename) if err := binary.Write(&b, binary.LittleEndian, att.Size); err != nil { @@ -152,12 +180,6 @@ func (h *FMsgHeader) GetHeaderHash() []byte { func (h *FMsgHeader) GetMessageHash() ([]byte, error) { if h.messageHash == nil { - f, err := os.Open(h.Filepath) - if err != nil { - return nil, err - } - defer f.Close() - hash := sha256.New() headerBytes := h.Encode() @@ -165,25 +187,45 @@ func (h *FMsgHeader) GetMessageHash() ([]byte, error) { return nil, err } - if _, err := io.Copy(hash, f); err != nil { + if err := hashPayload(hash, h.Filepath, int64(h.Size), h.Flags&FlagDeflate != 0); err != nil { return nil, err } // include attachment data (sequential byte sequences following // the message body, bounded by attachment header sizes) for _, att := range h.Attachments { - af, err := os.Open(att.Filepath) - if err != nil { - return nil, fmt.Errorf("open attachment %s: %w", att.Filename, err) + compressed := att.Flags&(1<<1) != 0 + if err := hashPayload(hash, att.Filepath, int64(att.Size), compressed); err != nil { + return nil, fmt.Errorf("hash attachment %s: %w", att.Filename, err) } - if _, err := io.CopyN(hash, af, int64(att.Size)); err != nil { - af.Close() - return nil, fmt.Errorf("read attachment %s: %w", att.Filename, err) - } - af.Close() } h.messageHash = hash.Sum(nil) } return h.messageHash, nil } + +func hashPayload(dst io.Writer, filepath string, wireSize int64, deflated bool) error { + f, err := os.Open(filepath) + if err != nil { + return err + } + defer f.Close() + + if deflated { + lr := io.LimitReader(f, wireSize) + zr, err := zlib.NewReader(lr) + if err != nil { + return err + } + _, err = io.Copy(dst, zr) + _ = zr.Close() + if err != nil { + return err + } + return nil + } + + _, err = io.CopyN(dst, f, wireSize) + return err +} diff --git a/src/defs_test.go b/src/defs_test.go index 4dca209..3eee685 100644 --- a/src/defs_test.go +++ b/src/defs_test.go @@ -2,9 +2,13 @@ package main import ( "bytes" + "compress/zlib" "crypto/sha256" "encoding/binary" + "io" "math" + "os" + "path/filepath" "testing" ) @@ -193,6 +197,7 @@ func TestEncodeWithAddTo(t *testing.T) { Pid: pid, From: FMsgAddress{User: "a", Domain: "b.com"}, To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + AddToFrom: &FMsgAddress{User: "a", Domain: "b.com"}, AddTo: []FMsgAddress{{User: "e", Domain: "f.com"}}, Timestamp: 0, Topic: "", @@ -222,6 +227,14 @@ func TestEncodeWithAddTo(t *testing.T) { tBuf := make([]byte, tLen) r.Read(tBuf) + // add-to-from + addToFromLen, _ := r.ReadByte() + addToFrom := make([]byte, addToFromLen) + r.Read(addToFrom) + if string(addToFrom) != "@a@b.com" { + t.Fatalf("add-to-from = %q, want %q", string(addToFrom), "@a@b.com") + } + // add to count addToCount, _ := r.ReadByte() if addToCount != 1 { @@ -237,6 +250,38 @@ func TestEncodeWithAddTo(t *testing.T) { } } +func TestEncodeWithAddToDefaultsAddToFromToFromAddress(t *testing.T) { + pid := make([]byte, 32) + h := &FMsgHeader{ + Version: 1, + Flags: FlagHasPid | FlagHasAddTo, + Pid: pid, + From: FMsgAddress{User: "a", Domain: "b.com"}, + To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + AddTo: []FMsgAddress{{User: "e", Domain: "f.com"}}, + Timestamp: 0, + Type: "text/plain", + } + b := h.Encode() + r := bytes.NewReader(b) + + r.ReadByte() // version + r.ReadByte() // flags + r.Read(make([]byte, 32)) + fLen, _ := r.ReadByte() + r.Read(make([]byte, fLen)) + r.ReadByte() // to count + tLen, _ := r.ReadByte() + r.Read(make([]byte, tLen)) + + addToFromLen, _ := r.ReadByte() + addToFrom := make([]byte, addToFromLen) + r.Read(addToFrom) + if string(addToFrom) != "@a@b.com" { + t.Fatalf("default add-to-from = %q, want %q", string(addToFrom), "@a@b.com") + } +} + func TestEncodeNoAddToWhenFlagUnset(t *testing.T) { // When FlagHasAddTo is NOT set, add-to addresses should not appear on the wire // even if the AddTo slice is populated. @@ -296,6 +341,24 @@ func TestGetHeaderHash(t *testing.T) { } } +func TestGetHeaderHashCommonTypeMatchesWireIDEncoding(t *testing.T) { + h := &FMsgHeader{ + Version: 1, + Flags: FlagCommonType, + From: FMsgAddress{User: "alice", Domain: "a.com"}, + To: []FMsgAddress{{User: "bob", Domain: "b.com"}}, + Timestamp: 1700000000, + Topic: "x", + TypeID: 3, + Type: "application/json", + } + expected := sha256.Sum256(h.Encode()) + got := h.GetHeaderHash() + if !bytes.Equal(got, expected[:]) { + t.Fatalf("GetHeaderHash mismatch for common type ID") + } +} + func TestStringOutput(t *testing.T) { h := &FMsgHeader{ Version: 1, @@ -387,7 +450,7 @@ func TestEncodeWithAttachments(t *testing.T) { Size: 100, Attachments: []FMsgAttachmentHeader{ {Flags: 0, Type: "image/png", Filename: "pic.png", Size: 2048}, - {Flags: 1, Type: "a", Filename: "doc.txt", Size: 512}, + {Flags: 1, TypeID: 38, Type: "image/png", Filename: "doc.txt", Size: 512}, }, } b := h.Encode() @@ -454,11 +517,9 @@ func TestEncodeWithAttachments(t *testing.T) { if att1Flags != 1 { t.Fatalf("att[1] flags = %d, want 1", att1Flags) } - att1TypeLen, _ := r.ReadByte() - att1Type := make([]byte, att1TypeLen) - r.Read(att1Type) - if string(att1Type) != "a" { - t.Fatalf("att[1] type = %q, want %q", string(att1Type), "a") + att1TypeID, _ := r.ReadByte() + if att1TypeID != 38 { + t.Fatalf("att[1] type ID = %d, want 38", att1TypeID) } att1FnLen, _ := r.ReadByte() att1Fn := make([]byte, att1FnLen) @@ -476,3 +537,105 @@ func TestEncodeWithAttachments(t *testing.T) { t.Fatalf("unexpected %d trailing bytes", r.Len()) } } + +func TestEncodeWithCommonMessageType(t *testing.T) { + h := &FMsgHeader{ + Version: 1, + Flags: FlagCommonType, + From: FMsgAddress{User: "a", Domain: "b.com"}, + To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + Timestamp: 0, + Topic: "", + TypeID: 3, + Type: "application/json", + } + b := h.Encode() + r := bytes.NewReader(b) + + r.ReadByte() // version + r.ReadByte() // flags + + // skip from + fLen, _ := r.ReadByte() + r.Read(make([]byte, fLen)) + // skip to count + to[0] + r.ReadByte() + tLen, _ := r.ReadByte() + r.Read(make([]byte, tLen)) + // skip timestamp + var ts float64 + binary.Read(r, binary.LittleEndian, &ts) + // skip topic + topicLen, _ := r.ReadByte() + r.Read(make([]byte, topicLen)) + + typeID, _ := r.ReadByte() + if typeID != 3 { + t.Fatalf("type ID = %d, want 3", typeID) + } +} + +func TestGetMessageHashUsesDecompressedPayloads(t *testing.T) { + compress := func(data []byte) []byte { + var b bytes.Buffer + w := zlib.NewWriter(&b) + if _, err := w.Write(data); err != nil { + t.Fatalf("zlib write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("zlib close: %v", err) + } + return b.Bytes() + } + + msgPlain := []byte("hello compressed body") + attPlain := []byte("hello compressed attachment") + msgWire := compress(msgPlain) + attWire := compress(attPlain) + + tmpDir := t.TempDir() + msgPath := filepath.Join(tmpDir, "msg.bin") + if err := os.WriteFile(msgPath, msgWire, 0600); err != nil { + t.Fatalf("write msg file: %v", err) + } + attPath := filepath.Join(tmpDir, "att.bin") + if err := os.WriteFile(attPath, attWire, 0600); err != nil { + t.Fatalf("write attachment file: %v", err) + } + + h := &FMsgHeader{ + Version: 1, + Flags: FlagDeflate, + From: FMsgAddress{User: "alice", Domain: "a.com"}, + To: []FMsgAddress{{User: "bob", Domain: "b.com"}}, + Timestamp: 1700000000, + Topic: "t", + Type: "text/plain", + Size: uint32(len(msgWire)), + Attachments: []FMsgAttachmentHeader{ + {Flags: 1 << 1, Type: "application/octet-stream", Filename: "a.bin", Size: uint32(len(attWire)), Filepath: attPath}, + }, + Filepath: msgPath, + } + + got, err := h.GetMessageHash() + if err != nil { + t.Fatalf("GetMessageHash() error: %v", err) + } + + manual := sha256.New() + if _, err := io.Copy(manual, bytes.NewReader(h.Encode())); err != nil { + t.Fatalf("manual header copy: %v", err) + } + if _, err := manual.Write(msgPlain); err != nil { + t.Fatalf("manual msg write: %v", err) + } + if _, err := manual.Write(attPlain); err != nil { + t.Fatalf("manual att write: %v", err) + } + want := manual.Sum(nil) + + if !bytes.Equal(got, want) { + t.Fatalf("message hash mismatch: got %x want %x", got, want) + } +} diff --git a/src/dns.go b/src/dns.go index e9840bf..8929b53 100644 --- a/src/dns.go +++ b/src/dns.go @@ -9,8 +9,50 @@ import ( "os" "strings" "time" + + "github.com/miekg/dns" ) +func dnssecRequired() bool { + return os.Getenv("FMSG_REQUIRE_DNSSEC") == "true" +} + +func resolverAuthenticatedData(name string, qtype uint16) (bool, error) { + cfg, err := dns.ClientConfigFromFile("/etc/resolv.conf") + if err != nil { + return false, err + } + + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(name), qtype) + msg.SetEdns0(4096, true) + + client := &dns.Client{Timeout: 5 * time.Second} + var lastErr error + for _, server := range cfg.Servers { + addr := net.JoinHostPort(server, cfg.Port) + resp, _, err := client.Exchange(msg, addr) + if err != nil { + lastErr = err + continue + } + if resp == nil { + lastErr = fmt.Errorf("nil DNS response from %s", addr) + continue + } + if resp.Rcode != dns.RcodeSuccess && resp.Rcode != dns.RcodeNameError { + lastErr = fmt.Errorf("dns rcode %d from %s", resp.Rcode, addr) + continue + } + return resp.AuthenticatedData, nil + } + + if lastErr == nil { + lastErr = fmt.Errorf("no DNS resolvers configured") + } + return false, lastErr +} + // lookupAuthorisedIPs resolves _fmsg. for A and AAAA records func lookupAuthorisedIPs(domain string) ([]net.IP, error) { fmsgDomain := "_fmsg." + domain @@ -21,6 +63,18 @@ func lookupAuthorisedIPs(domain string) ([]net.IP, error) { if len(ips) == 0 { return nil, fmt.Errorf("no A/AAAA records found for %s", fmsgDomain) } + + if dnssecRequired() { + adA, errA := resolverAuthenticatedData(fmsgDomain, dns.TypeA) + adAAAA, errAAAA := resolverAuthenticatedData(fmsgDomain, dns.TypeAAAA) + if !adA && !adAAAA { + if errA != nil && errAAAA != nil { + return nil, fmt.Errorf("dnssec validation failed for %s: A=%v AAAA=%v", fmsgDomain, errA, errAAAA) + } + return nil, fmt.Errorf("dnssec validation failed for %s: resolver did not set AD bit", fmsgDomain) + } + } + return ips, nil } diff --git a/src/go.mod b/src/go.mod index c4b2d01..f624234 100644 --- a/src/go.mod +++ b/src/go.mod @@ -10,7 +10,13 @@ require ( ) require ( + github.com/miekg/dns v1.1.68 // indirect github.com/stretchr/testify v1.8.2 // indirect + golang.org/x/mod v0.24.0 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/sync v0.14.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/tools v0.33.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/src/go.sum b/src/go.sum index 92e3a16..0bb8650 100644 --- a/src/go.sum +++ b/src/go.sum @@ -14,6 +14,8 @@ github.com/levenlabs/golib v0.0.0-20180911183212-0f8974794783 h1:ErBsqZpyadTBr2z github.com/levenlabs/golib v0.0.0-20180911183212-0f8974794783/go.mod h1:zw8z7nRRkGDZHexz1aMbZGtwxli5so0CBVZeIa3G+RE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA= +github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -23,6 +25,16 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/src/host.go b/src/host.go index f08a4af..828ec5c 100644 --- a/src/host.go +++ b/src/host.go @@ -3,7 +3,7 @@ package main import ( "bufio" "bytes" - "compress/flate" + "compress/zlib" "encoding/binary" "encoding/hex" "errors" @@ -18,6 +18,7 @@ import ( "path/filepath" "strings" "time" + "unicode" "unicode/utf8" env "github.com/caitlinelfring/go-env-default" @@ -50,13 +51,19 @@ const ( RejectCodeTimeTravel uint8 = 9 RejectCodeDuplicate uint8 = 10 AcceptCodeAddTo uint8 = 11 + AcceptCodeContinue uint8 = 64 + AcceptCodeSkipData uint8 = 65 RejectCodeUserUnknown uint8 = 100 RejectCodeUserFull uint8 = 101 RejectCodeUserNotAccepting uint8 = 102 - RejectCodeUserUndisclosed uint8 = 103 + RejectCodeUserDuplicate uint8 = 103 + RejectCodeUserUndisclosed uint8 = 105 RejectCodeAccept uint8 = 200 + + messageReservedBitsMask uint8 = 0b11000000 + attachmentReservedBitsMask uint8 = 0b11111100 ) // responseCodeName returns the human-friendly name for a response code. @@ -90,6 +97,8 @@ func responseCodeName(code uint8) string { return "user full" case RejectCodeUserNotAccepting: return "user not accepting" + case RejectCodeUserDuplicate: + return "user duplicate" case RejectCodeUserUndisclosed: return "user undisclosed" case RejectCodeAccept: @@ -101,6 +110,45 @@ func responseCodeName(code uint8) string { var ErrProtocolViolation = errors.New("protocol violation") +// commonMediaTypes maps common type IDs to their MIME strings per SPEC.md §4. +// IDs 1–64; unmapped IDs must be rejected with code 1 (invalid). +var commonMediaTypes = map[uint8]string{ + 1: "application/epub+zip", 2: "application/gzip", 3: "application/json", 4: "application/msword", + 5: "application/octet-stream", 6: "application/pdf", 7: "application/rtf", 8: "application/vnd.amazon.ebook", + 9: "application/vnd.ms-excel", 10: "application/vnd.ms-powerpoint", + 11: "application/vnd.oasis.opendocument.presentation", 12: "application/vnd.oasis.opendocument.spreadsheet", + 13: "application/vnd.oasis.opendocument.text", + 14: "application/vnd.openxmlformats-officedocument.presentationml.presentation", + 15: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + 16: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + 17: "application/x-tar", 18: "application/xhtml+xml", 19: "application/xml", 20: "application/zip", + 21: "audio/aac", 22: "audio/midi", 23: "audio/mpeg", 24: "audio/ogg", 25: "audio/opus", 26: "audio/vnd.wave", 27: "audio/webm", + 28: "font/otf", 29: "font/ttf", 30: "font/woff", 31: "font/woff2", + 32: "image/apng", 33: "image/avif", 34: "image/bmp", 35: "image/gif", 36: "image/heic", 37: "image/jpeg", 38: "image/png", + 39: "image/svg+xml", 40: "image/tiff", 41: "image/webp", + 42: "model/3mf", 43: "model/gltf-binary", 44: "model/obj", 45: "model/step", 46: "model/stl", 47: "model/vnd.usdz+zip", + 48: "text/calendar", 49: "text/css", 50: "text/csv", 51: "text/html", 52: "text/javascript", 53: "text/markdown", + 54: "text/plain;charset=US-ASCII", 55: "text/plain;charset=UTF-16", 56: "text/plain;charset=UTF-8", 57: "text/vcard", + 58: "video/H264", 59: "video/H265", 60: "video/H266", 61: "video/ogg", 62: "video/VP8", 63: "video/VP9", 64: "video/webm", +} + +// getCommonMediaType returns the MIME type string for a common type ID, or +// empty string + false if the ID is not mapped (should be rejected per spec). +func getCommonMediaType(id uint8) (string, bool) { + s, ok := commonMediaTypes[id] + return s, ok +} + +// getCommonMediaTypeID returns the common type ID for a MIME string. +func getCommonMediaTypeID(mediaType string) (uint8, bool) { + for id, mime := range commonMediaTypes { + if mime == mediaType { + return id, true + } + } + return 0, false +} + var Port = 4930 // The only reason RemotePort would ever be different from Port is when running two fmsg hosts on the same machine so the same port is unavaliable. @@ -189,17 +237,128 @@ func calcNetIODuration(sizeInBytes int, bytesPerSecond float64) time.Duration { } func isValidUser(s string) bool { - if len(s) == 0 || len(s) > 64 { + if !utf8.ValidString(s) || len(s) == 0 || len(s) > 64 { + return false + } + + isSpecial := func(r rune) bool { + return r == '-' || r == '_' || r == '.' + } + + runes := []rune(s) + if isSpecial(runes[0]) || isSpecial(runes[len(runes)-1]) { + return false + } + + lastWasSpecial := false + for _, c := range runes { + if unicode.IsLetter(c) || unicode.IsNumber(c) { + lastWasSpecial = false + continue + } + if !isSpecial(c) { + return false + } + if lastWasSpecial { + return false + } + lastWasSpecial = true + } + return true +} + +func isASCIIBytes(b []byte) bool { + for _, c := range b { + if c > 127 { + return false + } + } + return true +} + +func isValidAttachmentFilename(name string) bool { + if !utf8.ValidString(name) || len(name) == 0 || len(name) >= 256 { + return false + } + + isSpecial := func(r rune) bool { + return r == '-' || r == '_' || r == ' ' || r == '.' + } + + runes := []rune(name) + if isSpecial(runes[0]) || isSpecial(runes[len(runes)-1]) { return false } - for _, c := range s { - if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.') { + + lastWasSpecial := false + for _, r := range runes { + if unicode.IsLetter(r) || unicode.IsNumber(r) { + lastWasSpecial = false + continue + } + if !isSpecial(r) { + return false + } + if lastWasSpecial { return false } + lastWasSpecial = true } + return true } +func isMessageRetrievable(msg *FMsgHeader) bool { + if msg == nil { + return false + } + if msg.Filepath != "" { + st, err := os.Stat(msg.Filepath) + if err == nil && !st.IsDir() { + return true + } + } + if len(msg.Pid) == 0 { + return false + } + parentID, err := lookupMsgIdByHash(msg.Pid) + if err != nil || parentID == 0 { + return false + } + parentMsg, err := getMsgByID(parentID) + if err != nil { + return false + } + if parentMsg == nil { + return false + } + return isMessageRetrievable(parentMsg) +} + +func isParentParticipant(parent *FMsgHeader, addr *FMsgAddress) bool { + if parent == nil || addr == nil { + return false + } + target := strings.ToLower(addr.ToString()) + if strings.ToLower(parent.From.ToString()) == target { + return true + } + for i := range parent.To { + if strings.ToLower(parent.To[i].ToString()) == target { + return true + } + } + if parent.AddToFrom != nil && strings.ToLower(parent.AddToFrom.ToString()) == target { + return true + } + for i := range parent.AddTo { + if strings.ToLower(parent.AddTo[i].ToString()) == target { + return true + } + } + return false +} + func isValidDomain(s string) bool { if len(s) == 0 || len(s) > 253 { return false @@ -275,11 +434,15 @@ func handleChallenge(c net.Conn, r *bufio.Reader) error { if err != nil { return err } - hash := *(*[32]byte)(hashSlice) // get the underlying array (alternatively we could use hex strings..) + hash := *(*[32]byte)(hashSlice) log.Printf("INFO: CHALLENGE <-- %s", hex.EncodeToString(hashSlice)) - header, exists := lookupOutgoing(hash) + + // Verify the challenger's IP is the Host-B IP registered for this message + // (§10.5 step 2). An unrecognised hash OR a mismatched IP both → TERMINATE. + remoteIP, _, _ := net.SplitHostPort(c.RemoteAddr().String()) + header, exists := lookupOutgoing(hash, remoteIP) if !exists { - return fmt.Errorf("challenge for unknown message: %s, from: %s\n", hex.EncodeToString(hashSlice), c.RemoteAddr().String()) + return fmt.Errorf("challenge for unknown message: %s, from: %s", hex.EncodeToString(hashSlice), c.RemoteAddr().String()) } msgHash, err := header.GetMessageHash() if err != nil { @@ -296,310 +459,580 @@ func rejectAccept(c net.Conn, codes []byte) error { return err } -func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { - r := bufio.NewReaderSize(c, ReadBufferSize) - var h = &FMsgHeader{} +func sendCode(c net.Conn, code uint8) error { + return rejectAccept(c, []byte{code}) +} - d := calcNetIODuration(66000, MinDownloadRate) // max possible header size - c.SetReadDeadline(time.Now().Add(d)) +func validateMessageFlags(c net.Conn, flags uint8) error { + if flags&messageReservedBitsMask != 0 { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err + } + return fmt.Errorf("reserved message flag bits set: %#08b", flags) + } + return nil +} - // read version - v, err := r.ReadByte() +func validateAttachmentFlags(c net.Conn, flags uint8) error { + if flags&attachmentReservedBitsMask != 0 { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err + } + return fmt.Errorf("reserved attachment flag bits set: %#08b", flags) + } + return nil +} + +func hasDomainRecipient(addrs []FMsgAddress, domain string) bool { + for _, addr := range addrs { + if strings.EqualFold(addr.Domain, domain) { + return true + } + } + return false +} + +func determineSenderDomain(h *FMsgHeader) string { + if len(h.AddTo) > 0 && h.AddToFrom != nil { + return h.AddToFrom.Domain + } + return h.From.Domain +} + +func verifySenderIP(c net.Conn, senderDomain string) error { + if SkipAuthorisedIPs { + return nil + } + + remoteHost, _, err := net.SplitHostPort(c.RemoteAddr().String()) if err != nil { - return h, r, err + log.Printf("WARN: failed to parse remote address for DNS check: %s", err) + return fmt.Errorf("DNS verification failed") } - // TODO [Spec 1.3]: Per spec step 1.3, the first byte determines the message type: - // - 1..127: fmsg version (currently only version 1 supported). - // - 128..255: incoming CHALLENGE; the fmsg version is (256 - value). - // E.g. 255 = challenge for version 1, 254 = challenge for version 2, etc. - // Currently only v==255 is handled as a challenge. This should be generalised - // to handle any value > 128 where (256 - value) is a supported version. - // For unsupported versions or invalid first-byte values, Host B MUST send - // REJECT code 2 (unsupported version) on the connection before closing. - if v == 255 { - return nil, r, handleChallenge(c, r) + + remoteIP := net.ParseIP(remoteHost) + if remoteIP == nil { + log.Printf("WARN: failed to parse remote IP: %s", remoteHost) + return fmt.Errorf("DNS verification failed") } - if v != 1 { - // TODO [Spec 1.3.iii]: Send REJECT code 2 (unsupported version) on the - // connection before returning/closing, per spec step 1.3.iii. - return h, r, fmt.Errorf("unsupported version: %d", v) + + authorisedIPs, err := lookupAuthorisedIPs(senderDomain) + if err != nil { + log.Printf("WARN: DNS lookup failed for _fmsg.%s: %s", senderDomain, err) + return fmt.Errorf("DNS verification failed") } - h.Version = v - // read flags - flags, err := r.ReadByte() + for _, ip := range authorisedIPs { + if remoteIP.Equal(ip) { + return nil + } + } + + log.Printf("WARN: remote IP %s not in authorised IPs for _fmsg.%s", remoteIP.String(), senderDomain) + return fmt.Errorf("DNS verification failed") +} + +func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) { + if len(h.AddTo) == 0 { + return h, nil + } + + addToHasOurDomain := hasDomainRecipient(h.AddTo, Domain) + + parentID, err := lookupMsgIdByHash(h.Pid) if err != nil { - return h, r, err + return h, err } - h.Flags = flags - // read pid if any - if flags&FlagHasPid == 1 { - pid, err := io.ReadAll(io.LimitReader(r, 32)) - if err != nil { - return h, r, err + if parentID == 0 { + h.InitialResponseCode = AcceptCodeContinue + return h, nil + } + + parentMsg, err := getMsgByID(parentID) + if err != nil { + return h, err + } + if parentMsg == nil || !isMessageRetrievable(parentMsg) { + h.InitialResponseCode = AcceptCodeContinue + return h, nil + } + + if parentMsg.Timestamp-FutureTimeDelta > h.Timestamp { + if err := sendCode(c, RejectCodeTimeTravel); err != nil { + return h, err } - h.Pid = make([]byte, 32) - copy(h.Pid, pid) - // TODO [Spec 1.4.v]: pid verification depends on the add-to field and must - // be performed here (during header exchange), not deferred to downloadMessage. - // See step 1.4.v for the full decision tree. + return h, fmt.Errorf("add-to: time travel detected (parent time %f, current %f)", parentMsg.Timestamp, h.Timestamp) } - // read from address - from, err := readAddress(r) + if addToHasOurDomain { + h.InitialResponseCode = AcceptCodeSkipData + return h, nil + } + + h.Filepath = parentMsg.Filepath + for i := range h.Attachments { + if i < len(parentMsg.Attachments) { + h.Attachments[i].Filepath = parentMsg.Attachments[i].Filepath + } + } + if err := storeMsgHeaderOnly(h); err != nil { + if err2 := sendCode(c, RejectCodeUndisclosed); err2 != nil { + return h, err2 + } + return h, fmt.Errorf("add-to notification: storing header: %w", err) + } + if err := sendCode(c, AcceptCodeAddTo); err != nil { + return h, err + } + log.Printf("INFO: additional recipients received (code 11) for pid %s", hex.EncodeToString(h.Pid)) + return nil, nil +} + +func validatePidReplyPath(c net.Conn, h *FMsgHeader) error { + if len(h.AddTo) != 0 || h.Flags&FlagHasPid == 0 { + return nil + } + + parentID, err := lookupMsgIdByHash(h.Pid) if err != nil { - return h, r, err + return err + } + if parentID == 0 { + if err := sendCode(c, RejectCodeParentNotFound); err != nil { + return err + } + return fmt.Errorf("pid reply: parent not found for pid %s", hex.EncodeToString(h.Pid)) } - h.From = *from + parentMsg, err := getMsgByID(parentID) + if err != nil { + return err + } + if parentMsg == nil { + if err := sendCode(c, RejectCodeParentNotFound); err != nil { + return err + } + return fmt.Errorf("pid reply: parent message not found by ID %d", parentID) + } + if !isMessageRetrievable(parentMsg) { + if err := sendCode(c, RejectCodeParentNotFound); err != nil { + return err + } + return fmt.Errorf("pid reply: parent is not retrievable for msg %d", parentID) + } + + if parentMsg.Timestamp-FutureTimeDelta > h.Timestamp { + if err := sendCode(c, RejectCodeTimeTravel); err != nil { + return err + } + return fmt.Errorf("pid reply: time travel detected (parent time %f, current %f)", parentMsg.Timestamp, h.Timestamp) + } + if !isParentParticipant(parentMsg, &h.From) { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err + } + return fmt.Errorf("pid reply: sender %s was not a participant of parent", h.From.ToString()) + } + + return nil +} + +func readVersionOrChallenge(c net.Conn, r *bufio.Reader, h *FMsgHeader) (bool, error) { + v, err := r.ReadByte() + if err != nil { + return false, err + } + if v >= 129 { + challengeVersion := 256 - int(v) + if challengeVersion == 1 { + return true, handleChallenge(c, r) + } + if err := sendCode(c, RejectCodeUnsupportedVersion); err != nil { + log.Printf("WARN: failed to send unsupported version response: %s", err) + } + return false, fmt.Errorf("unsupported challenge version: %d", challengeVersion) + } + if v != 1 { + if err := sendCode(c, RejectCodeUnsupportedVersion); err != nil { + log.Printf("WARN: failed to send unsupported version response: %s", err) + } + return false, fmt.Errorf("unsupported message version: %d", v) + } + h.Version = v + return false, nil +} - // read to addresses +func readToRecipients(c net.Conn, r *bufio.Reader, h *FMsgHeader) (map[string]bool, error) { num, err := r.ReadByte() if err != nil { - return h, r, err + return nil, err + } + if num == 0 { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return nil, err + } + return nil, fmt.Errorf("to count must be >= 1") } - // TODO [Spec 1.4.i.a]: Spec requires at least one address in "to". If num == 0, - // reject with code 1 (invalid). seen := make(map[string]bool) for num > 0 { addr, err := readAddress(r) if err != nil { - return h, r, err + return nil, err } key := strings.ToLower(addr.ToString()) if seen[key] { - return h, r, fmt.Errorf("duplicate recipient address: %s", addr.ToString()) + return nil, fmt.Errorf("duplicate recipient address: %s", addr.ToString()) } seen[key] = true h.To = append(h.To, *addr) num-- } + return seen, nil +} + +func readAddToRecipients(c net.Conn, r *bufio.Reader, h *FMsgHeader, seen map[string]bool) error { + if h.Flags&FlagHasAddTo == 0 { + return nil + } + if h.Flags&FlagHasPid == 0 { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err + } + return fmt.Errorf("add to exists but pid does not") + } + + addToFrom, err := readAddress(r) + if err != nil { + if err2 := sendCode(c, RejectCodeInvalid); err2 != nil { + return err2 + } + return fmt.Errorf("reading add-to-from address: %w", err) + } - // parse "add to" field when has-add-to flag (bit 1) is set - if flags&FlagHasAddTo != 0 { - // add to requires pid per spec 1.4.v.b.a - if flags&FlagHasPid == 0 { - codes := []byte{RejectCodeInvalid} - if err := rejectAccept(c, codes); err != nil { - return h, r, err + addToFromKey := strings.ToLower(addToFrom.ToString()) + fromKey := strings.ToLower(h.From.ToString()) + inFromOrTo := fromKey == addToFromKey + if !inFromOrTo { + for _, toAddr := range h.To { + if strings.ToLower(toAddr.ToString()) == addToFromKey { + inFromOrTo = true + break } - return h, r, fmt.Errorf("add to exists but pid does not") } + } + if !inFromOrTo { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err + } + return fmt.Errorf("add-to-from (%s) not in from or to", addToFrom.ToString()) + } + h.AddToFrom = addToFrom + + addToCount, err := r.ReadByte() + if err != nil { + return err + } + if addToCount == 0 { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err + } + return fmt.Errorf("add to flag set but count is 0") + } - addToCount, err := r.ReadByte() + addToSeen := make(map[string]bool) + for addToCount > 0 { + addr, err := readAddress(r) if err != nil { - return h, r, err + return err } - if addToCount == 0 { - codes := []byte{RejectCodeInvalid} - if err := rejectAccept(c, codes); err != nil { - return h, r, err + key := strings.ToLower(addr.ToString()) + if addToSeen[key] { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err } - return h, r, fmt.Errorf("add to flag set but count is 0") + return fmt.Errorf("duplicate recipient address in add to: %s", addr.ToString()) } - addToSeen := make(map[string]bool) - for addToCount > 0 { - addr, err := readAddress(r) - if err != nil { - return h, r, err - } - key := strings.ToLower(addr.ToString()) - if addToSeen[key] { - codes := []byte{RejectCodeInvalid} - if err := rejectAccept(c, codes); err != nil { - return h, r, err - } - return h, r, fmt.Errorf("duplicate recipient address in add to: %s", addr.ToString()) + addToSeen[key] = true + if seen[key] { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err } - addToSeen[key] = true - h.AddTo = append(h.AddTo, *addr) - addToCount-- + return fmt.Errorf("add-to address already in to: %s", addr.ToString()) } + h.AddTo = append(h.AddTo, *addr) + addToCount-- } - // read timestamp + return nil +} + +func readAndValidateTimestamp(c net.Conn, r *bufio.Reader, h *FMsgHeader) error { if err := binary.Read(r, binary.LittleEndian, &h.Timestamp); err != nil { - return h, r, err + return err } now := timeutil.TimestampNow().Float64() delta := now - h.Timestamp if PastTimeDelta > 0 && delta > PastTimeDelta { - codes := []byte{RejectCodePastTime} - if err := rejectAccept(c, codes); err != nil { - return h, r, err + if err := sendCode(c, RejectCodePastTime); err != nil { + return err } - return h, r, fmt.Errorf("message timestamp: %f too far in past, delta: %fs", h.Timestamp, delta) + return fmt.Errorf("message timestamp: %f too far in past, delta: %fs", h.Timestamp, delta) } if FutureTimeDelta > 0 && delta < 0 && math.Abs(delta) > FutureTimeDelta { - codes := []byte{RejectCodeFutureTime} - if err := rejectAccept(c, codes); err != nil { - return h, r, err + if err := sendCode(c, RejectCodeFutureTime); err != nil { + return err } - return h, r, fmt.Errorf("message timestamp: %f too far in future, delta: %fs", h.Timestamp, delta) + return fmt.Errorf("message timestamp: %f too far in future, delta: %fs", h.Timestamp, delta) } + return nil +} - // read topic — only present when pid is NOT set (first message in a thread) - if flags&FlagHasPid == 0 { - topic, err := ReadUInt8Slice(r) +func readMessageType(c net.Conn, r *bufio.Reader, h *FMsgHeader) error { + if h.Flags&FlagCommonType != 0 { + typeID, err := r.ReadByte() if err != nil { - return h, r, err + return err } - h.Topic = string(topic) + mtype, ok := getCommonMediaType(typeID) + if !ok { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err + } + return fmt.Errorf("unmapped common type ID: %d", typeID) + } + h.TypeID = typeID + h.Type = mtype + return nil } - // TODO [Spec 1.4.i.b / flag bit 2]: When the "common type" flag (bit 2) is - // set, the type field is a single uint8 that maps to a predefined Media Type - // string per the Common Media Types table. If the value has no mapping, reject - // with code 1 (invalid). Currently the code always reads type as uint8-prefixed - // string, ignoring the common type flag entirely. - - // read type mime, err := ReadUInt8Slice(r) if err != nil { - return h, r, err + return err + } + if !isASCIIBytes(mime) { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err + } + return fmt.Errorf("message media type must be US-ASCII") } h.Type = string(mime) + return nil +} - // read message size - if err := binary.Read(r, binary.LittleEndian, &h.Size); err != nil { - return h, r, err +func readAttachmentType(c net.Conn, r *bufio.Reader, flags uint8) (string, uint8, error) { + if flags&(1<<0) != 0 { + typeID, err := r.ReadByte() + if err != nil { + return "", 0, err + } + mtype, ok := getCommonMediaType(typeID) + if !ok { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return "", 0, err + } + return "", 0, fmt.Errorf("unmapped attachment common type ID: %d", typeID) + } + return mtype, typeID, nil } - // TODO [Spec 1.4.iii]: Size check must be total of data size PLUS all - // attachment sizes, not just data size alone. Currently only h.Size (data) - // is checked against MaxMessageSize. - if h.Size > MaxMessageSize { - codes := []byte{RejectCodeTooBig} - if err := rejectAccept(c, codes); err != nil { - return h, r, err + + typeBytes, err := ReadUInt8Slice(r) + if err != nil { + return "", 0, err + } + if !isASCIIBytes(typeBytes) { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return "", 0, err } - return h, r, fmt.Errorf("message size: %d exceeds max: %d", h.Size, MaxMessageSize) + return "", 0, fmt.Errorf("attachment media type must be US-ASCII") } + return string(typeBytes), 0, nil +} - // read attachment count +func readAttachmentHeaders(c net.Conn, r *bufio.Reader, h *FMsgHeader) error { var attachCount uint8 if err := binary.Read(r, binary.LittleEndian, &attachCount); err != nil { - return h, r, err - } - // TODO [Spec 1.4.i.c / attachments]: Parse attachment headers (flags, type, - // filename, size) for each attachment. Validate: - // - Each attachment's type when its own common-type flag is set exists in - // Common Media Type mapping; else reject code 1 (invalid). - // - Filename is valid UTF-8 per spec (Unicode letters/numbers, limited - // special chars, unique case-insensitive, < 256 bytes). - // - Incorporate attachment sizes into the MAX_SIZE check above. - // - Store parsed attachment headers on FMsgHeader. - if attachCount > 0 { - return h, r, fmt.Errorf("attachments not yet supported (count: %d)", attachCount) + return err } - log.Printf("INFO: <-- MSG\n%s", h) + totalSize := h.Size + filenameSeen := make(map[string]bool) + for i := uint8(0); i < attachCount; i++ { + attFlags, err := r.ReadByte() + if err != nil { + return err + } + if err := validateAttachmentFlags(c, attFlags); err != nil { + return err + } - // TODO [Spec 1.4.ii]: DNS verification of sender's IP address MUST be - // performed here during header exchange (not deferred to the challenge - // step). Host B must resolve _fmsg. and verify the incoming - // connection's IP is in the authorised set. If not authorised, Host B MUST - // TERMINATE the message exchange (abort connection, no reject code sent). - // Currently this check only happens inside challenge() and is skipped - // entirely when skip-challenge is allowed. - - // add-to pid decision tree per spec 1.4.v.b - if len(h.AddTo) > 0 { - // check if any add-to recipients are for our domain - addToHasOurDomain := false - for _, addr := range h.AddTo { - if strings.EqualFold(addr.Domain, Domain) { - addToHasOurDomain = true - break - } + attType, attTypeID, err := readAttachmentType(c, r, attFlags) + if err != nil { + return err } - if !addToHasOurDomain { - // none of the add-to recipients are for our domain — record - // the additional recipients and respond code 11. - // pid must refer to a stored message per "Verifying Message Stored" - parentID, err := lookupMsgIdByHash(h.Pid) - if err != nil { - return h, r, err + filenameBytes, err := ReadUInt8Slice(r) + if err != nil { + return err + } + filename := string(filenameBytes) + if !isValidAttachmentFilename(filename) { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err } - if parentID == 0 { - codes := []byte{RejectCodeParentNotFound} - if err := rejectAccept(c, codes); err != nil { - return h, r, err - } - return h, r, fmt.Errorf("add-to notification: parent not found for pid %s", hex.EncodeToString(h.Pid)) + return fmt.Errorf("invalid attachment filename: %s", filename) + } + filenameKey := strings.ToLower(filename) + if filenameSeen[filenameKey] { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return err } + return fmt.Errorf("duplicate attachment filename: %s", filename) + } + filenameSeen[filenameKey] = true - // record add-to recipients for future hash reconstruction, respond code 11 - if err := storeMsgHeaderOnly(h); err != nil { - codes := []byte{RejectCodeUndisclosed} - if err2 := rejectAccept(c, codes); err2 != nil { - return h, r, err2 - } - return h, r, fmt.Errorf("add-to notification: storing header: %w", err) - } - codes := []byte{AcceptCodeAddTo} - if err := rejectAccept(c, codes); err != nil { - return h, r, err - } - log.Printf("INFO: additional recipients received (code 11) for pid %s", hex.EncodeToString(h.Pid)) - return nil, r, nil // nil header signals handled (like challenge) + var attSize uint32 + if err := binary.Read(r, binary.LittleEndian, &attSize); err != nil { + return err } - // else: add-to recipients are for our domain, continue normally - // (pid message does NOT have to be already stored per spec 1.4.v.b.b) + + h.Attachments = append(h.Attachments, FMsgAttachmentHeader{ + Flags: attFlags, + TypeID: attTypeID, + Type: attType, + Filename: filename, + Size: attSize, + }) + totalSize += attSize } - // TODO [Spec 1.4.v.c]: When pid exists and add-to does not: - // a. The message or message header pid refers to MUST be verified as stored - // per "Verifying Message Stored"; else reject code 6 (parent not found). - // "Verifying Message Stored" means: the digest matches a previously - // accepted message (code 200) or accepted header (code 11), AND the - // corresponding data currently exists on the host. - // b. The stored parent's time MUST be before the incoming message's time; - // else reject code 9 (time travel). - // Both checks should be here (step 1), not deferred to downloadMessage. + if totalSize > MaxMessageSize { + if err := sendCode(c, RejectCodeTooBig); err != nil { + return err + } + return fmt.Errorf("total message size %d exceeds max %d", totalSize, MaxMessageSize) + } - return h, r, nil + return nil } -// Sends CHALLENGE request to sender domain first checking if domain is indeed located -// at address in connection supplied by verifying the remote IP is in the -// sender's _fmsg. authorised IP set. -// TODO [Spec step 2]: The spec defines challenge modes (NEVER, ALWAYS, -// HAS_NOT_PARTICIPATED, DIFFERENT_DOMAIN) as implementation choices. -// Currently defaults to ALWAYS. Implement configurable challenge mode. -func challenge(conn net.Conn, h *FMsgHeader) error { +func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) { + r := bufio.NewReaderSize(c, ReadBufferSize) + var h = &FMsgHeader{InitialResponseCode: AcceptCodeContinue} - // verify remote IP is authorised by sender's _fmsg DNS record - if SkipAuthorisedIPs { - log.Println("WARN: skipping authorised IP check (FMSG_SKIP_AUTHORISED_IPS=true)") - } else { - remoteHost, _, err := net.SplitHostPort(conn.RemoteAddr().String()) - if err != nil { - return fmt.Errorf("failed to parse remote address: %w", err) - } - remoteIP := net.ParseIP(remoteHost) - if remoteIP == nil { - return fmt.Errorf("failed to parse remote IP: %s", remoteHost) + d := calcNetIODuration(66000, MinDownloadRate) // max possible header size + c.SetReadDeadline(time.Now().Add(d)) + + handled, err := readVersionOrChallenge(c, r, h) + if err != nil { + if handled { + return nil, r, err } + return h, r, err + } + if handled { + return nil, r, nil + } + + // read flags + flags, err := r.ReadByte() + if err != nil { + return h, r, err + } + h.Flags = flags + if err := validateMessageFlags(c, flags); err != nil { + return h, r, err + } - authorisedIPs, err := lookupAuthorisedIPs(h.From.Domain) + // read pid if any + if flags&FlagHasPid == 1 { + pid, err := io.ReadAll(io.LimitReader(r, 32)) if err != nil { - return err + return h, r, err } + h.Pid = make([]byte, 32) + copy(h.Pid, pid) + } - found := false - for _, ip := range authorisedIPs { - if remoteIP.Equal(ip) { - found = true - break - } + // read from address + from, err := readAddress(r) + if err != nil { + return h, r, err + } + + h.From = *from + + seen, err := readToRecipients(c, r, h) + if err != nil { + return h, r, err + } + + if err := readAddToRecipients(c, r, h, seen); err != nil { + return h, r, err + } + + if err := readAndValidateTimestamp(c, r, h); err != nil { + return h, r, err + } + + // read topic — only present when pid is NOT set (first message in a thread) + if flags&FlagHasPid == 0 { + topic, err := ReadUInt8Slice(r) + if err != nil { + return h, r, err } - if !found { - return fmt.Errorf("remote address %s not in _fmsg.%s authorised IPs", remoteIP.String(), h.From.Domain) + h.Topic = string(topic) + } + + if err := readMessageType(c, r, h); err != nil { + return h, r, err + } + + // read message size + if err := binary.Read(r, binary.LittleEndian, &h.Size); err != nil { + return h, r, err + } + // Size check is deferred until attachment headers are parsed (see below) + + if err := readAttachmentHeaders(c, r, h); err != nil { + return h, r, err + } + + log.Printf("INFO: <-- MSG\n%s", h) + + if !hasDomainRecipient(h.To, Domain) && !hasDomainRecipient(h.AddTo, Domain) { + if err := sendCode(c, RejectCodeInvalid); err != nil { + return h, r, err } + return h, r, fmt.Errorf("no recipients for domain %s", Domain) + } + + if err := verifySenderIP(c, determineSenderDomain(h)); err != nil { + return nil, r, err } + h, err = handleAddToPath(c, h) + if err != nil { + return h, r, err + } + if h == nil { + return nil, r, nil + } + + if err := validatePidReplyPath(c, h); err != nil { + return h, r, err + } + + return h, r, nil +} + +// Sends CHALLENGE request to sender, receiving and storing the challenge hash. +// DNS verification of the remote IP is performed during header exchange (readHeader). +// TODO [Spec step 2]: The spec defines challenge modes (NEVER, ALWAYS, +// HAS_NOT_PARTICIPATED, DIFFERENT_DOMAIN) as implementation choices. +// Currently defaults to ALWAYS. Implement configurable challenge mode. +func challenge(conn net.Conn, h *FMsgHeader) error { + // Connection 2 MUST target the same IP as Connection 1 (spec 2.1). remoteHost, _, err := net.SplitHostPort(conn.RemoteAddr().String()) if err != nil { @@ -624,7 +1057,11 @@ func challenge(conn net.Conn, h *FMsgHeader) error { if err != nil { return err } + if len(resp) != 32 { + return fmt.Errorf("challenge response size %d, expected 32", len(resp)) + } copy(h.ChallengeHash[:], resp) + h.ChallengeCompleted = true log.Printf("INFO: <-- CHALLENGE RESP\t%s\n", hex.EncodeToString(resp)) // gracefully close 2nd connection @@ -635,10 +1072,18 @@ func challenge(conn net.Conn, h *FMsgHeader) error { return nil } -func validateMsgRecvForAddr(h *FMsgHeader, addr *FMsgAddress) (code uint8, err error) { +func validateMsgRecvForAddr(h *FMsgHeader, addr *FMsgAddress, msgHash []byte) (code uint8, err error) { + duplicate, err := hasAddrReceivedMsgHash(msgHash, addr) + if err != nil { + return RejectCodeUserUndisclosed, err + } + if duplicate { + return RejectCodeUserDuplicate, nil + } + detail, err := getAddressDetail(addr) if err != nil { - return RejectCodeUndisclosed, err + return RejectCodeUserUndisclosed, err } if detail == nil { return RejectCodeUserUnknown, nil @@ -682,17 +1127,8 @@ func uniqueFilepath(dir string, timestamp uint32, ext string) string { } } -func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { - - // TODO [Spec 3.1]: Per spec step 3.1, BEFORE downloading the remaining - // message data, if the CHALLENGE/CHALLENGE-RESP exchange was completed the - // message hash from the CHALLENGE-RESP SHOULD be used to check if the - // message is already stored (duplicate check). If found, respond REJECT - // code 10 (duplicate) and close. Currently the duplicate check happens - // AFTER downloading all data, wasting bandwidth on duplicates. - - // filter to our domain recipients in "to" then "add to" order per spec 3.4.i - addrs := []FMsgAddress{} +func localRecipients(h *FMsgHeader) []FMsgAddress { + addrs := make([]FMsgAddress, 0, len(h.To)+len(h.AddTo)) for _, addr := range h.To { if strings.EqualFold(addr.Domain, Domain) { addrs = append(addrs, addr) @@ -703,79 +1139,258 @@ func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { addrs = append(addrs, addr) } } + return addrs +} + +func allLocalRecipientsHaveMessageHash(msgHash []byte, addrs []FMsgAddress) (bool, error) { if len(addrs) == 0 { - return fmt.Errorf("%w our domain: %s, not in recipient list", ErrProtocolViolation, Domain) + return false, nil } - codes := make([]byte, len(addrs)) + for i := range addrs { + duplicate, err := hasAddrReceivedMsgHash(msgHash, &addrs[i]) + if err != nil { + return false, err + } + if !duplicate { + return false, nil + } + } + return true, nil +} + +func markAllCodes(codes []byte, code uint8) { + for i := range codes { + codes[i] = code + } +} + +func prepareMessageData(r io.Reader, h *FMsgHeader, skipData bool) ([]string, error) { + if skipData { + parentID, err := lookupMsgIdByHash(h.Pid) + if err != nil { + return nil, err + } + if parentID == 0 { + return nil, fmt.Errorf("%w code 65 requires stored parent for pid %s", ErrProtocolViolation, hex.EncodeToString(h.Pid)) + } + parentMsg, err := getMsgByID(parentID) + if err != nil { + return nil, err + } + if parentMsg == nil || parentMsg.Filepath == "" { + return nil, fmt.Errorf("%w code 65 parent data unavailable for msg %d", ErrProtocolViolation, parentID) + } + h.Filepath = parentMsg.Filepath + return nil, nil + } + + createdPaths := make([]string, 0, 1+len(h.Attachments)) - // download to temp file fd, err := os.CreateTemp("", "fmsg-download-*") if err != nil { + return nil, err + } + + if _, err := io.CopyN(fd, r, int64(h.Size)); err != nil { + fd.Close() + _ = os.Remove(fd.Name()) + return nil, err + } + if err := fd.Close(); err != nil { + _ = os.Remove(fd.Name()) + return nil, err + } + + h.Filepath = fd.Name() + createdPaths = append(createdPaths, fd.Name()) + + for i := range h.Attachments { + afd, err := os.CreateTemp("", "fmsg-attachment-*") + if err != nil { + for _, path := range createdPaths { + _ = os.Remove(path) + } + return nil, err + } + + if _, err := io.CopyN(afd, r, int64(h.Attachments[i].Size)); err != nil { + afd.Close() + _ = os.Remove(afd.Name()) + for _, path := range createdPaths { + _ = os.Remove(path) + } + return nil, err + } + if err := afd.Close(); err != nil { + _ = os.Remove(afd.Name()) + for _, path := range createdPaths { + _ = os.Remove(path) + } + return nil, err + } + h.Attachments[i].Filepath = afd.Name() + createdPaths = append(createdPaths, afd.Name()) + } + + return createdPaths, nil +} + +func cleanupFiles(paths []string) { + for _, path := range paths { + if path == "" { + continue + } + _ = os.Remove(path) + } +} + +func copyMessagePayload(src *os.File, dstPath string, compressed bool, wireSize uint32) error { + if _, err := src.Seek(0, io.SeekStart); err != nil { return err } - defer os.Remove(fd.Name()) - defer fd.Close() - _, err = io.CopyN(fd, r, int64(h.Size)) + fd2, err := os.Create(dstPath) if err != nil { return err } - // TODO [Spec 3.2]: Also download attachment data (sequential byte sequences - // whose boundaries are defined by attachment header sizes). Currently only - // message body data is downloaded; attachment bodies are not handled. - - // verify hash matches challenge response - // TODO [Spec 3.3]: This verification MUST only be performed when the - // CHALLENGE/CHALLENGE-RESP exchange was actually completed. If the challenge - // was skipped (or not issued), ChallengeHash will be zero-valued and this - // comparison will erroneously fail for every message. Guard this check behind - // a flag or sentinel indicating whether the challenge exchange occurred. - h.Filepath = fd.Name() - msgHash, err := h.GetMessageHash() - if err != nil { + var copyErr error + if compressed { + lr := io.LimitReader(src, int64(wireSize)) + zr, err := zlib.NewReader(lr) + if err != nil { + fd2.Close() + _ = os.Remove(dstPath) + return err + } + _, copyErr = io.Copy(fd2, zr) + _ = zr.Close() + } else { + _, copyErr = io.CopyN(fd2, src, int64(wireSize)) + } + if err := fd2.Close(); err != nil { return err } - if !bytes.Equal(h.ChallengeHash[:], msgHash) { - challengeHashStr := hex.EncodeToString(h.ChallengeHash[:]) - actualHashStr := hex.EncodeToString(msgHash) - return fmt.Errorf("%w actual hash: %s mismatch challenge response: %s", ErrProtocolViolation, actualHashStr, challengeHashStr) + + if copyErr != nil { + _ = os.Remove(dstPath) + return copyErr } + return nil +} - // check for duplicate message - dupID, err := lookupMsgIdByHash(msgHash) - if err != nil { - return err +func uniqueAttachmentPath(dir string, timestamp uint32, idx int, filename string) string { + ext := filepath.Ext(filename) + base := fmt.Sprintf("%d_att_%d", timestamp, idx) + p := filepath.Join(dir, base+ext) + if _, err := os.Stat(p); os.IsNotExist(err) { + return p + } + for n := 1; ; n++ { + p = filepath.Join(dir, fmt.Sprintf("%s_%d%s", base, n, ext)) + if _, err := os.Stat(p); os.IsNotExist(err) { + return p + } } - if dupID > 0 { +} + +func persistAttachmentPayloads(h *FMsgHeader, dirpath string) error { + for i := range h.Attachments { + a := &h.Attachments[i] + src, err := os.Open(a.Filepath) + if err != nil { + return err + } + dstPath := uniqueAttachmentPath(dirpath, uint32(h.Timestamp), i, a.Filename) + compressed := a.Flags&(1<<1) != 0 + err = copyMessagePayload(src, dstPath, compressed, a.Size) + src.Close() + if err != nil { + return err + } + a.Filepath = dstPath + } + return nil +} + +func storeAcceptedMessage(h *FMsgHeader, codes []byte, acceptedTo []FMsgAddress, acceptedAddTo []FMsgAddress, primaryFilepath string) bool { + if len(acceptedTo) == 0 && len(acceptedAddTo) == 0 { + return false + } + + origTo := h.To + origAddTo := h.AddTo + h.To = acceptedTo + h.AddTo = acceptedAddTo + h.Filepath = primaryFilepath + if err := storeMsgDetail(h); err != nil { + log.Printf("ERROR: storing message: %s", err) + h.To = origTo + h.AddTo = origAddTo for i := range codes { - codes[i] = RejectCodeDuplicate + if codes[i] == RejectCodeAccept { + codes[i] = RejectCodeUndisclosed + } } - return rejectAccept(c, codes) + return false } - // verify parent exists if pid is set - // TODO [Spec 1.4.v.c]: This pid verification should happen during header - // exchange (readHeader / step 1.4.v), not after downloading the message. - // Additionally, the spec requires checking the parent's timestamp is before - // the current message's timestamp (reject code 9 = time travel), and the - // verification must follow the "Verifying Message Stored" semantics: - // - The digest must match a previously accepted message (code 200) or - // accepted add-to (code 11). - // - The message/header must currently exist and be retrievable. - if h.Flags&FlagHasPid != 0 { - parentID, err := lookupMsgIdByHash(h.Pid) + h.To = origTo + h.AddTo = origAddTo + allAccepted := append(acceptedTo, acceptedAddTo...) + for i := range allAccepted { + if err := postMsgStatRecv(&allAccepted[i], h.Timestamp, int(h.Size)); err != nil { + log.Printf("WARN: Failed to post msg recv stat: %s", err) + } + } + return true +} + +func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader, skipData bool) error { + addrs := localRecipients(h) + if len(addrs) == 0 { + return fmt.Errorf("%w our domain: %s, not in recipient list", ErrProtocolViolation, Domain) + } + codes := make([]byte, len(addrs)) + + if h.ChallengeCompleted { + allDup, err := allLocalRecipientsHaveMessageHash(h.ChallengeHash[:], addrs) if err != nil { return err } - if parentID == 0 { - for i := range codes { - codes[i] = RejectCodeParentNotFound - } - return rejectAccept(c, codes) + handled, err := respondGlobalDuplicateIfNeeded(c, h.ChallengeCompleted, allDup) + if err != nil { + return err + } + if handled { + return nil } } + createdPaths, err := prepareMessageData(r, h, skipData) + if err != nil { + return err + } + cleanupOnReturn := !skipData + defer func() { + if cleanupOnReturn { + cleanupFiles(createdPaths) + } + }() + + // verify hash matches challenge response when challenge was completed + msgHash, err := h.GetMessageHash() + if err != nil { + return err + } + if h.ChallengeCompleted && !bytes.Equal(h.ChallengeHash[:], msgHash) { + challengeHashStr := hex.EncodeToString(h.ChallengeHash[:]) + actualHashStr := hex.EncodeToString(msgHash) + return fmt.Errorf("%w actual hash: %s mismatch challenge response: %s", ErrProtocolViolation, actualHashStr, challengeHashStr) + } + + // pid/add-to validation is handled during header exchange in readHeader(). + // determine file extension from MIME type exts, _ := mime.ExtensionsByType(h.Type) var ext string @@ -785,6 +1400,12 @@ func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { ext = exts[0] } + src, err := os.Open(h.Filepath) + if err != nil { + return err + } + defer src.Close() + // validate each recipient and copy message for accepted ones // Build a set of add-to addresses for later classification addToSet := make(map[string]bool) @@ -795,7 +1416,7 @@ func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { acceptedAddTo := []FMsgAddress{} var primaryFilepath string for i, addr := range addrs { - code, err := validateMsgRecvForAddr(h, &addr) + code, err := validateMsgRecvForAddr(h, &addr, msgHash) if err != nil { return err } @@ -812,28 +1433,8 @@ func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { } fp := uniqueFilepath(dirpath, uint32(h.Timestamp), ext) - if _, err := fd.Seek(0, io.SeekStart); err != nil { - return err - } - - fd2, err := os.Create(fp) - if err != nil { - return err - } - - var copyErr error - if h.Flags&FlagDeflate != 0 { - fr := flate.NewReader(fd) - _, copyErr = io.Copy(fd2, fr) - fr.Close() - } else { - _, copyErr = io.Copy(fd2, fd) - } - fd2.Close() - - if copyErr != nil { - log.Printf("ERROR: copying downloaded message from: %s, to: %s", fd.Name(), fp) - os.Remove(fp) + if err := copyMessagePayload(src, fp, h.Flags&FlagDeflate != 0, h.Size); err != nil { + log.Printf("ERROR: copying downloaded message from: %s, to: %s", h.Filepath, fp) codes[i] = RejectCodeUndisclosed continue } @@ -846,40 +1447,35 @@ func downloadMessage(c net.Conn, r io.Reader, h *FMsgHeader) error { } if primaryFilepath == "" { primaryFilepath = fp + if err := persistAttachmentPayloads(h, filepath.Dir(primaryFilepath)); err != nil { + log.Printf("ERROR: copying attachment payloads for message storage: %s", err) + codes[i] = RejectCodeUndisclosed + primaryFilepath = "" + acceptedTo = acceptedTo[:0] + acceptedAddTo = acceptedAddTo[:0] + continue + } } } - // store message details once for all accepted recipients - if len(acceptedTo) > 0 || len(acceptedAddTo) > 0 { - origTo := h.To - origAddTo := h.AddTo - h.To = acceptedTo - h.AddTo = acceptedAddTo - h.Filepath = primaryFilepath - if err := storeMsgDetail(h); err != nil { - log.Printf("ERROR: storing message: %s", err) - h.To = origTo - h.AddTo = origAddTo - for i := range codes { - if codes[i] == RejectCodeAccept { - codes[i] = RejectCodeUndisclosed - } - } - } else { - h.To = origTo - h.AddTo = origAddTo - allAccepted := append(acceptedTo, acceptedAddTo...) - for i := range allAccepted { - if err := postMsgStatRecv(&allAccepted[i], h.Timestamp, int(h.Size)); err != nil { - log.Printf("WARN: Failed to post msg recv stat: %s", err) - } - } - } + stored := storeAcceptedMessage(h, codes, acceptedTo, acceptedAddTo, primaryFilepath) + if stored { + cleanupOnReturn = false } return rejectAccept(c, codes) } +func respondGlobalDuplicateIfNeeded(c net.Conn, challengeCompleted, allDup bool) (bool, error) { + if !challengeCompleted || !allDup { + return false, nil + } + if err := sendCode(c, RejectCodeDuplicate); err != nil { + return false, err + } + return true, nil +} + func abortConn(c net.Conn) { if tcp, ok := c.(*net.TCPConn); ok { tcp.SetLinger(0) @@ -910,25 +1506,40 @@ func handleConn(c net.Conn) { return } - // challenge - err = challenge(c, header) - if err != nil { + if err := challenge(c, header); err != nil { log.Printf("ERROR: Challenge failed to, %s: %s", c.RemoteAddr().String(), err) abortConn(c) return } + // Send post-header response code (64 continue / 65 skip data). + if err := rejectAccept(c, []byte{header.InitialResponseCode}); err != nil { + log.Printf("ERROR: failed sending initial response to %s: %s", c.RemoteAddr().String(), err) + abortConn(c) + return + } + + skipData := header.InitialResponseCode == AcceptCodeSkipData + + if skipData { + log.Printf("INFO: sent code 65 (skip data) to %s", c.RemoteAddr().String()) + } else { + log.Printf("INFO: sent code 64 (continue) to %s", c.RemoteAddr().String()) + } + // store message - d := calcNetIODuration(int(header.Size), MinDownloadRate) - c.SetReadDeadline(time.Now().Add(d)) - err = downloadMessage(c, r, header) - if err != nil { + deadlineBytes := int(header.Size) + if skipData { + deadlineBytes = 1 + } + c.SetReadDeadline(time.Now().Add(calcNetIODuration(deadlineBytes, MinDownloadRate))) + if err := downloadMessage(c, r, header, skipData); err != nil { // if error was a protocal violation, abort; otherise let sender know there was an internal error log.Printf("ERROR: Download failed from, %s: %s", c.RemoteAddr().String(), err) if errors.Is(err, ErrProtocolViolation) { return } else { - rejectAccept(c, []byte{RejectCodeUndisclosed}) + _ = sendCode(c, RejectCodeUndisclosed) } } diff --git a/src/host_test.go b/src/host_test.go index 2f8a2c8..bfb63d1 100644 --- a/src/host_test.go +++ b/src/host_test.go @@ -1,20 +1,41 @@ package main import ( + "bufio" "bytes" + "encoding/binary" + "net" "testing" "time" ) +type testAddr string + +func (a testAddr) Network() string { return "tcp" } +func (a testAddr) String() string { return string(a) } + +type testConn struct { + bytes.Buffer +} + +func (c *testConn) Read(b []byte) (int, error) { return 0, nil } +func (c *testConn) Write(b []byte) (int, error) { return c.Buffer.Write(b) } +func (c *testConn) Close() error { return nil } +func (c *testConn) LocalAddr() net.Addr { return testAddr("127.0.0.1:1000") } +func (c *testConn) RemoteAddr() net.Addr { return testAddr("127.0.0.1:2000") } +func (c *testConn) SetDeadline(t time.Time) error { return nil } +func (c *testConn) SetReadDeadline(t time.Time) error { return nil } +func (c *testConn) SetWriteDeadline(t time.Time) error { return nil } + func TestIsValidUser(t *testing.T) { - valid := []string{"alice", "Bob", "a-b", "a_b", "a.b", "user123", "A"} + valid := []string{"alice", "Bob", "a-b", "a_b", "a.b", "user123", "A", "u\u00f1icode", "\u7528\u62371"} for _, u := range valid { if !isValidUser(u) { t.Errorf("isValidUser(%q) = false, want true", u) } } - invalid := []string{"", " ", "a b", "a@b", "a/b", string(make([]byte, 65))} + invalid := []string{"", " ", "a b", "a@b", "a/b", string(make([]byte, 65)), "-alice", "alice-", "a..b", "a-_b"} for _, u := range invalid { if isValidUser(u) { t.Errorf("isValidUser(%q) = true, want false", u) @@ -153,6 +174,7 @@ func TestResponseCodeName(t *testing.T) { {RejectCodeUserUnknown, "user unknown"}, {RejectCodeUserFull, "user full"}, {RejectCodeUserNotAccepting, "user not accepting"}, + {RejectCodeUserDuplicate, "user duplicate"}, {RejectCodeUserUndisclosed, "user undisclosed"}, {RejectCodeAccept, "accept"}, {99, "unknown(99)"}, @@ -165,6 +187,15 @@ func TestResponseCodeName(t *testing.T) { } } +func TestPerRecipientDuplicateAndUndisclosedCodeValues(t *testing.T) { + if RejectCodeUserDuplicate != 103 { + t.Fatalf("RejectCodeUserDuplicate = %d, want 103", RejectCodeUserDuplicate) + } + if RejectCodeUserUndisclosed != 105 { + t.Fatalf("RejectCodeUserUndisclosed = %d, want 105", RejectCodeUserUndisclosed) + } +} + func TestFlagConstants(t *testing.T) { // Verify flag bit assignments match SPEC.md if FlagHasPid != 1 { @@ -186,3 +217,322 @@ func TestFlagConstants(t *testing.T) { t.Errorf("FlagDeflate = %d, want 32 (bit 5)", FlagDeflate) } } + +func encodeUInt8String(t *testing.T, s string) []byte { + t.Helper() + if len(s) > 255 { + t.Fatalf("string too long for uint8 prefix: %d", len(s)) + } + b := []byte{byte(len(s))} + b = append(b, []byte(s)...) + return b +} + +func TestHasDomainRecipient(t *testing.T) { + addrs := []FMsgAddress{ + {User: "alice", Domain: "example.com"}, + {User: "bob", Domain: "other.org"}, + } + if !hasDomainRecipient(addrs, "EXAMPLE.COM") { + t.Fatalf("expected domain match") + } + if hasDomainRecipient(addrs, "missing.test") { + t.Fatalf("did not expect domain match") + } +} + +func TestDetermineSenderDomain(t *testing.T) { + h := &FMsgHeader{ + From: FMsgAddress{User: "alice", Domain: "from.example"}, + } + if got := determineSenderDomain(h); got != "from.example" { + t.Fatalf("determineSenderDomain() = %q, want %q", got, "from.example") + } + + h.AddTo = []FMsgAddress{{User: "new", Domain: "to.example"}} + h.AddToFrom = &FMsgAddress{User: "bob", Domain: "sender.example"} + if got := determineSenderDomain(h); got != "sender.example" { + t.Fatalf("determineSenderDomain() = %q, want %q", got, "sender.example") + } +} + +func TestReadToRecipients(t *testing.T) { + b := []byte{2} + b = append(b, encodeUInt8String(t, "@alice@example.com")...) + b = append(b, encodeUInt8String(t, "@bob@example.com")...) + + h := &FMsgHeader{} + seen, err := readToRecipients(nil, bufio.NewReader(bytes.NewReader(b)), h) + if err != nil { + t.Fatalf("readToRecipients returned error: %v", err) + } + if len(h.To) != 2 { + t.Fatalf("len(h.To) = %d, want 2", len(h.To)) + } + if !seen["@alice@example.com"] || !seen["@bob@example.com"] { + t.Fatalf("seen map missing expected recipients: %#v", seen) + } +} + +func TestReadAddToRecipients(t *testing.T) { + h := &FMsgHeader{ + Flags: FlagHasPid | FlagHasAddTo, + From: FMsgAddress{User: "alice", Domain: "example.com"}, + To: []FMsgAddress{{User: "bob", Domain: "example.com"}}, + } + seen := map[string]bool{"@bob@example.com": true} + + b := []byte{} + b = append(b, encodeUInt8String(t, "@alice@example.com")...) // add-to-from + b = append(b, 1) // add-to count + b = append(b, encodeUInt8String(t, "@carol@example.com")...) + + err := readAddToRecipients(nil, bufio.NewReader(bytes.NewReader(b)), h, seen) + if err != nil { + t.Fatalf("readAddToRecipients returned error: %v", err) + } + if h.AddToFrom == nil || h.AddToFrom.ToString() != "@alice@example.com" { + t.Fatalf("unexpected AddToFrom: %+v", h.AddToFrom) + } + if len(h.AddTo) != 1 || h.AddTo[0].ToString() != "@carol@example.com" { + t.Fatalf("unexpected AddTo: %+v", h.AddTo) + } +} + +func TestReadMessageType(t *testing.T) { + hCommon := &FMsgHeader{Flags: FlagCommonType} + if err := readMessageType(nil, bufio.NewReader(bytes.NewReader([]byte{3})), hCommon); err != nil { + t.Fatalf("readMessageType(common) error: %v", err) + } + if hCommon.TypeID != 3 { + t.Fatalf("common type ID = %d, want 3", hCommon.TypeID) + } + if hCommon.Type != "application/json" { + t.Fatalf("common type = %q, want %q", hCommon.Type, "application/json") + } + + hText := &FMsgHeader{Flags: 0} + b := encodeUInt8String(t, "text/plain") + if err := readMessageType(nil, bufio.NewReader(bytes.NewReader(b)), hText); err != nil { + t.Fatalf("readMessageType(string) error: %v", err) + } + if hText.Type != "text/plain" { + t.Fatalf("string type = %q, want %q", hText.Type, "text/plain") + } +} + +func TestReadAttachmentHeaders(t *testing.T) { + origMax := MaxMessageSize + MaxMessageSize = 1024 + t.Cleanup(func() { + MaxMessageSize = origMax + }) + + h := &FMsgHeader{Size: 10} + b := []byte{1} // attachment count + b = append(b, 0) // attachment flags (no common type) + b = append(b, encodeUInt8String(t, "text/plain")...) + b = append(b, encodeUInt8String(t, "file.txt")...) + + var sz [4]byte + binary.LittleEndian.PutUint32(sz[:], 12) + b = append(b, sz[:]...) + + err := readAttachmentHeaders(nil, bufio.NewReader(bytes.NewReader(b)), h) + if err != nil { + t.Fatalf("readAttachmentHeaders returned error: %v", err) + } + if len(h.Attachments) != 1 { + t.Fatalf("len(h.Attachments) = %d, want 1", len(h.Attachments)) + } + att := h.Attachments[0] + if att.TypeID != 0 { + t.Fatalf("attachment type ID = %d, want 0 for non-common", att.TypeID) + } + if att.Type != "text/plain" || att.Filename != "file.txt" || att.Size != 12 { + t.Fatalf("unexpected attachment parsed: %+v", att) + } +} + +func TestReadAddToRecipientsRejectsWhenPidMissing(t *testing.T) { + h := &FMsgHeader{Flags: FlagHasAddTo} + c := &testConn{} + + err := readAddToRecipients(c, bufio.NewReader(bytes.NewReader(nil)), h, map[string]bool{}) + if err == nil { + t.Fatalf("expected error when add-to flag is set without pid") + } + if got := c.Bytes(); len(got) != 1 || got[0] != RejectCodeInvalid { + t.Fatalf("expected reject code %d, got %v", RejectCodeInvalid, got) + } +} + +func TestReadAddToRecipientsRejectsDuplicateAddTo(t *testing.T) { + h := &FMsgHeader{ + Flags: FlagHasPid | FlagHasAddTo, + From: FMsgAddress{User: "alice", Domain: "example.com"}, + To: []FMsgAddress{{User: "bob", Domain: "example.com"}}, + } + c := &testConn{} + seen := map[string]bool{"@bob@example.com": true} + + b := []byte{} + b = append(b, encodeUInt8String(t, "@alice@example.com")...) // add-to-from + b = append(b, 2) // add-to count + b = append(b, encodeUInt8String(t, "@carol@example.com")...) + b = append(b, encodeUInt8String(t, "@carol@example.com")...) + + err := readAddToRecipients(c, bufio.NewReader(bytes.NewReader(b)), h, seen) + if err == nil { + t.Fatalf("expected duplicate add-to error") + } + if got := c.Bytes(); len(got) != 1 || got[0] != RejectCodeInvalid { + t.Fatalf("expected reject code %d, got %v", RejectCodeInvalid, got) + } +} + +func TestReadMessageTypeRejectsUnknownCommonType(t *testing.T) { + h := &FMsgHeader{Flags: FlagCommonType} + c := &testConn{} + + err := readMessageType(c, bufio.NewReader(bytes.NewReader([]byte{200})), h) + if err == nil { + t.Fatalf("expected error for unknown common type") + } + if got := c.Bytes(); len(got) != 1 || got[0] != RejectCodeInvalid { + t.Fatalf("expected reject code %d, got %v", RejectCodeInvalid, got) + } +} + +func TestReadMessageTypeRejectsNonASCIIStringType(t *testing.T) { + h := &FMsgHeader{Flags: 0} + c := &testConn{} + + b := encodeUInt8String(t, "text/\u03c0lain") + err := readMessageType(c, bufio.NewReader(bytes.NewReader(b)), h) + if err == nil { + t.Fatalf("expected error for non-ASCII message type") + } + if got := c.Bytes(); len(got) != 1 || got[0] != RejectCodeInvalid { + t.Fatalf("expected reject code %d, got %v", RejectCodeInvalid, got) + } +} + +func TestReadAttachmentTypeRejectsNonASCIIStringType(t *testing.T) { + c := &testConn{} + b := encodeUInt8String(t, "text/\u03c0lain") + + _, _, err := readAttachmentType(c, bufio.NewReader(bytes.NewReader(b)), 0) + if err == nil { + t.Fatalf("expected error for non-ASCII attachment type") + } + if got := c.Bytes(); len(got) != 1 || got[0] != RejectCodeInvalid { + t.Fatalf("expected reject code %d, got %v", RejectCodeInvalid, got) + } +} + +func TestReadAttachmentHeadersRejectsInvalidFilename(t *testing.T) { + origMax := MaxMessageSize + MaxMessageSize = 1024 + t.Cleanup(func() { + MaxMessageSize = origMax + }) + + h := &FMsgHeader{Size: 10} + c := &testConn{} + b := []byte{1} + b = append(b, 0) + b = append(b, encodeUInt8String(t, "text/plain")...) + b = append(b, encodeUInt8String(t, "bad..name")...) // invalid: consecutive special chars + + var sz [4]byte + binary.LittleEndian.PutUint32(sz[:], 12) + b = append(b, sz[:]...) + + err := readAttachmentHeaders(c, bufio.NewReader(bytes.NewReader(b)), h) + if err == nil { + t.Fatalf("expected error for invalid attachment filename") + } + if got := c.Bytes(); len(got) != 1 || got[0] != RejectCodeInvalid { + t.Fatalf("expected reject code %d, got %v", RejectCodeInvalid, got) + } +} + +func TestReadAttachmentHeadersRejectsTooBig(t *testing.T) { + origMax := MaxMessageSize + MaxMessageSize = 20 + t.Cleanup(func() { + MaxMessageSize = origMax + }) + + h := &FMsgHeader{Size: 15} + c := &testConn{} + b := []byte{1} + b = append(b, 0) + b = append(b, encodeUInt8String(t, "text/plain")...) + b = append(b, encodeUInt8String(t, "file.txt")...) + + var sz [4]byte + binary.LittleEndian.PutUint32(sz[:], 10) + b = append(b, sz[:]...) + + err := readAttachmentHeaders(c, bufio.NewReader(bytes.NewReader(b)), h) + if err == nil { + t.Fatalf("expected size overflow error") + } + if got := c.Bytes(); len(got) != 1 || got[0] != RejectCodeTooBig { + t.Fatalf("expected reject code %d, got %v", RejectCodeTooBig, got) + } +} + +func TestValidateMessageFlagsRejectsReservedBits(t *testing.T) { + c := &testConn{} + err := validateMessageFlags(c, 1<<6) + if err == nil { + t.Fatalf("expected error for reserved message flag bit") + } + if got := c.Bytes(); len(got) != 1 || got[0] != RejectCodeInvalid { + t.Fatalf("expected reject code %d, got %v", RejectCodeInvalid, got) + } +} + +func TestReadAttachmentHeadersRejectsReservedAttachmentBits(t *testing.T) { + h := &FMsgHeader{Size: 0} + c := &testConn{} + + // attachment count=1, then attachment flags with reserved bit 2 set + b := []byte{1, 1 << 2} + err := readAttachmentHeaders(c, bufio.NewReader(bytes.NewReader(b)), h) + if err == nil { + t.Fatalf("expected error for reserved attachment flag bits") + } + if got := c.Bytes(); len(got) != 1 || got[0] != RejectCodeInvalid { + t.Fatalf("expected reject code %d, got %v", RejectCodeInvalid, got) + } +} + +func TestRespondGlobalDuplicateIfNeeded(t *testing.T) { + c := &testConn{} + handled, err := respondGlobalDuplicateIfNeeded(c, true, true) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !handled { + t.Fatalf("expected handled=true") + } + if got := c.Bytes(); len(got) != 1 || got[0] != RejectCodeDuplicate { + t.Fatalf("expected duplicate code %d, got %v", RejectCodeDuplicate, got) + } + + c2 := &testConn{} + handled, err = respondGlobalDuplicateIfNeeded(c2, true, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if handled { + t.Fatalf("expected handled=false") + } + if len(c2.Bytes()) != 0 { + t.Fatalf("expected no bytes written, got %v", c2.Bytes()) + } +} diff --git a/src/outgoing.go b/src/outgoing.go index 6592886..242811f 100644 --- a/src/outgoing.go +++ b/src/outgoing.go @@ -2,34 +2,65 @@ package main import "sync" -// outgoing message headers keyed on header hash. +// outgoingEntry tracks an in-flight outgoing message header together with +// the set of Host-B IPs currently being serviced for that message. +// The IP set is used to validate incoming challenges (§10.5 step 2). +type outgoingEntry struct { + header *FMsgHeader + ips map[string]struct{} +} + +// outgoingMap indexes in-flight message headers by their header hash. // All access is synchronised via outgoingMu. -var outgoingMap map[[32]byte]*FMsgHeader +var outgoingMap map[[32]byte]*outgoingEntry var outgoingMu sync.RWMutex func initOutgoing() { - outgoingMap = make(map[[32]byte]*FMsgHeader) + outgoingMap = make(map[[32]byte]*outgoingEntry) } -// registerOutgoing stores a header in the outgoing map so challenge handlers -// running on other goroutines can look it up. -func registerOutgoing(hash [32]byte, h *FMsgHeader) { +// registerOutgoing records hash → (header, ip) so challenge handlers can look +// it up. Multiple IPs may be registered for the same hash when the same message +// is being concurrently delivered to different domains (§10.2 step 2). +func registerOutgoing(hash [32]byte, h *FMsgHeader, ip string) { outgoingMu.Lock() - outgoingMap[hash] = h + e, ok := outgoingMap[hash] + if !ok { + e = &outgoingEntry{header: h, ips: make(map[string]struct{})} + outgoingMap[hash] = e + } + e.ips[ip] = struct{}{} outgoingMu.Unlock() } -// lookupOutgoing retrieves an outgoing header by its hash. -func lookupOutgoing(hash [32]byte) (*FMsgHeader, bool) { +// lookupOutgoing returns the header for hash iff ip is a registered Host-B IP +// for that entry. Returns (nil, false) if the hash is unknown or ip is not in +// the registered set (§10.5 step 2). +func lookupOutgoing(hash [32]byte, ip string) (*FMsgHeader, bool) { outgoingMu.RLock() - h, ok := outgoingMap[hash] + e, ok := outgoingMap[hash] + if !ok { + outgoingMu.RUnlock() + return nil, false + } + _, ipOK := e.ips[ip] + h := e.header outgoingMu.RUnlock() - return h, ok + if !ipOK { + return nil, false + } + return h, true } -// deleteOutgoing removes an outgoing header after delivery completes. -func deleteOutgoing(hash [32]byte) { +// removeOutgoingIP removes ip from the entry's IP set. When the set becomes +// empty the map entry is deleted entirely (§10.2 step 7). +func removeOutgoingIP(hash [32]byte, ip string) { outgoingMu.Lock() - delete(outgoingMap, hash) + if e, ok := outgoingMap[hash]; ok { + delete(e.ips, ip) + if len(e.ips) == 0 { + delete(outgoingMap, hash) + } + } outgoingMu.Unlock() } diff --git a/src/outgoing_test.go b/src/outgoing_test.go index 6585d48..af75412 100644 --- a/src/outgoing_test.go +++ b/src/outgoing_test.go @@ -19,17 +19,19 @@ func TestOutgoingMapOperations(t *testing.T) { var hash [32]byte copy(hash[:], h.GetHeaderHash()) + const ip = "1.2.3.4" + // Lookup before register should fail - _, ok := lookupOutgoing(hash) + _, ok := lookupOutgoing(hash, ip) if ok { t.Fatal("lookupOutgoing found entry before register") } // Register - registerOutgoing(hash, h) + registerOutgoing(hash, h, ip) - // Lookup after register should succeed - got, ok := lookupOutgoing(hash) + // Lookup with correct IP should succeed + got, ok := lookupOutgoing(hash, ip) if !ok { t.Fatal("lookupOutgoing failed after register") } @@ -37,12 +39,58 @@ func TestOutgoingMapOperations(t *testing.T) { t.Fatal("lookupOutgoing returned different pointer") } - // Delete - deleteOutgoing(hash) + // Lookup with wrong IP should fail + _, ok = lookupOutgoing(hash, "9.9.9.9") + if ok { + t.Fatal("lookupOutgoing succeeded with wrong IP") + } + + // Remove IP — entry should be gone + removeOutgoingIP(hash, ip) - _, ok = lookupOutgoing(hash) + _, ok = lookupOutgoing(hash, ip) if ok { - t.Fatal("lookupOutgoing found entry after delete") + t.Fatal("lookupOutgoing found entry after removeOutgoingIP") + } +} + +func TestOutgoingMapMultipleIPs(t *testing.T) { + initOutgoing() + + h := &FMsgHeader{ + Version: 1, + From: FMsgAddress{User: "a", Domain: "b.com"}, + To: []FMsgAddress{{User: "c", Domain: "d.com"}}, + Timestamp: 1.0, + Type: "text/plain", + } + + var hash [32]byte + copy(hash[:], h.GetHeaderHash()) + + registerOutgoing(hash, h, "1.1.1.1") + registerOutgoing(hash, h, "2.2.2.2") + + // Both IPs should resolve + for _, ip := range []string{"1.1.1.1", "2.2.2.2"} { + if _, ok := lookupOutgoing(hash, ip); !ok { + t.Errorf("expected lookup to succeed for IP %s", ip) + } + } + + // Removing first IP still leaves entry for second + removeOutgoingIP(hash, "1.1.1.1") + if _, ok := lookupOutgoing(hash, "1.1.1.1"); ok { + t.Error("1.1.1.1 still present after remove") + } + if _, ok := lookupOutgoing(hash, "2.2.2.2"); !ok { + t.Error("2.2.2.2 missing after removing 1.1.1.1") + } + + // Removing last IP deletes the entry + removeOutgoingIP(hash, "2.2.2.2") + if _, ok := lookupOutgoing(hash, "2.2.2.2"); ok { + t.Error("entry still present after removing last IP") } } @@ -68,11 +116,11 @@ func TestOutgoingMapMultipleEntries(t *testing.T) { copy(hash1[:], h1.GetHeaderHash()) copy(hash2[:], h2.GetHeaderHash()) - registerOutgoing(hash1, h1) - registerOutgoing(hash2, h2) + registerOutgoing(hash1, h1, "1.1.1.1") + registerOutgoing(hash2, h2, "2.2.2.2") - got1, ok1 := lookupOutgoing(hash1) - got2, ok2 := lookupOutgoing(hash2) + got1, ok1 := lookupOutgoing(hash1, "1.1.1.1") + got2, ok2 := lookupOutgoing(hash2, "2.2.2.2") if !ok1 || got1 != h1 { t.Error("failed to look up h1") @@ -81,14 +129,14 @@ func TestOutgoingMapMultipleEntries(t *testing.T) { t.Error("failed to look up h2") } - // Delete one, other should remain - deleteOutgoing(hash1) - _, ok1 = lookupOutgoing(hash1) - _, ok2 = lookupOutgoing(hash2) + // Remove one, other should remain + removeOutgoingIP(hash1, "1.1.1.1") + _, ok1 = lookupOutgoing(hash1, "1.1.1.1") + _, ok2 = lookupOutgoing(hash2, "2.2.2.2") if ok1 { - t.Error("h1 still present after delete") + t.Error("h1 still present after remove") } if !ok2 { - t.Error("h2 missing after deleting h1") + t.Error("h2 missing after removing h1") } } diff --git a/src/sender.go b/src/sender.go index 6f36137..1db7148 100644 --- a/src/sender.go +++ b/src/sender.go @@ -37,14 +37,6 @@ type pendingTarget struct { // findPendingTargets discovers (msg_id, domain) pairs with undelivered, // retryable recipients. This is a lightweight read-only query — row-level // locks are acquired per-delivery in deliverMessage. -// -// TODO [Spec]: Spec says to retry "with back-off" (e.g. exponential back-off). -// Currently uses a fixed RetryInterval. Implement an exponential back-off -// strategy — e.g. double the wait after each failed attempt per (msg, domain). -// -// TODO [Spec]: Per-user code 101 (user full = "insufficient resources for -// specific recipient") is analogous to global code 5 and is likely transient. -// Consider adding 101 to the retryable set alongside codes 3 and 5. func findPendingTargets() ([]pendingTarget, error) { db, err := sql.Open("postgres", "") if err != nil { @@ -61,8 +53,8 @@ func findPendingTargets() ([]pendingTarget, error) { INNER JOIN msg m ON m.id = mt.msg_id WHERE mt.time_delivered IS NULL AND m.time_sent IS NOT NULL - AND (mt.response_code IS NULL OR mt.response_code IN (3, 5)) - AND (mt.time_last_attempt IS NULL OR ($1 - mt.time_last_attempt) > $2) + AND (mt.response_code IS NULL OR mt.response_code IN (3, 5, 101)) + AND (mt.time_last_attempt IS NULL OR ($1 - mt.time_last_attempt) > LEAST($2 * POWER(2.0, GREATEST(mt.attempt_count - 1, 0)::float), $3)) AND ($1 - m.time_sent) < $3 UNION ALL SELECT mat.msg_id, mat.addr @@ -70,8 +62,8 @@ func findPendingTargets() ([]pendingTarget, error) { INNER JOIN msg m ON m.id = mat.msg_id WHERE mat.time_delivered IS NULL AND m.time_sent IS NOT NULL - AND (mat.response_code IS NULL OR mat.response_code IN (3, 5)) - AND (mat.time_last_attempt IS NULL OR ($1 - mat.time_last_attempt) > $2) + AND (mat.response_code IS NULL OR mat.response_code IN (3, 5, 101)) + AND (mat.time_last_attempt IS NULL OR ($1 - mat.time_last_attempt) > LEAST($2 * POWER(2.0, GREATEST(mat.attempt_count - 1, 0)::float), $3)) AND ($1 - m.time_sent) < $3 `, now, RetryInterval, RetryMaxAge) if err != nil { @@ -109,6 +101,73 @@ func findPendingTargets() ([]pendingTarget, error) { return targets, rows.Err() } +// sendMsgData transmits the message body then all attachment payloads on conn. +func sendMsgData(conn net.Conn, h *FMsgHeader) error { + fd, err := os.Open(h.Filepath) + if err != nil { + return fmt.Errorf("opening data file %s: %w", h.Filepath, err) + } + defer fd.Close() + + conn.SetWriteDeadline(time.Now().Add(calcNetIODuration(int(h.Size), MinUploadRate))) + if _, err := io.CopyN(conn, fd, int64(h.Size)); err != nil { + return fmt.Errorf("sending data: %w", err) + } + for _, att := range h.Attachments { + af, err := os.Open(att.Filepath) + if err != nil { + return fmt.Errorf("opening attachment %s: %w", att.Filename, err) + } + _, copyErr := io.CopyN(conn, af, int64(att.Size)) + af.Close() + if copyErr != nil { + return fmt.Errorf("sending attachment %s: %w", att.Filename, copyErr) + } + } + return nil +} + +// updateRecipient records a delivery outcome for one address in table. +// Deliveries set time_delivered; failures set time_last_attempt and increment +// attempt_count to drive exponential back-off on subsequent retries. +func updateRecipient(tx *sql.Tx, table, addr string, msgID int64, now float64, code int, delivered bool) { + var err error + if delivered { + _, err = tx.Exec(fmt.Sprintf(` + UPDATE %s SET time_delivered = $1, response_code = $2 + WHERE msg_id = $3 AND addr = $4 + `, table), now, code, msgID, addr) + } else { + _, err = tx.Exec(fmt.Sprintf(` + UPDATE %s SET time_last_attempt = $1, response_code = $2, + attempt_count = attempt_count + 1 + WHERE msg_id = $3 AND addr = $4 + `, table), now, code, msgID, addr) + } + if err != nil { + log.Printf("ERROR: sender: update recipient %s: %s", addr, err) + } +} + +// updateAllLocked applies the same outcome to every locked to and add-to address. +func updateAllLocked(tx *sql.Tx, lockedAddrs, lockedAddToAddrs []string, msgID int64, now float64, code int, delivered bool) { + for _, a := range lockedAddrs { + updateRecipient(tx, "msg_to", a, msgID, now, code, delivered) + } + for _, a := range lockedAddToAddrs { + updateRecipient(tx, "msg_add_to", a, msgID, now, code, delivered) + } +} + +// commitOrLog commits the transaction and marks it as committed. +func commitOrLog(tx *sql.Tx, committed *bool, msgID int64) { + if err := tx.Commit(); err != nil { + log.Printf("ERROR: sender: commit tx for msg %d: %s", msgID, err) + } else { + *committed = true + } +} + // deliverMessage handles delivery of a single message to a single remote domain. // // It manages its own database transaction with the following lifecycle: @@ -168,8 +227,8 @@ func deliverMessage(target pendingTarget) { WHERE mt.msg_id = $1 AND mt.time_delivered IS NULL AND m.time_sent IS NOT NULL - AND (mt.response_code IS NULL OR mt.response_code IN (3, 5)) - AND (mt.time_last_attempt IS NULL OR ($2 - mt.time_last_attempt) > $3) + AND (mt.response_code IS NULL OR mt.response_code IN (3, 5, 101)) + AND (mt.time_last_attempt IS NULL OR ($2 - mt.time_last_attempt) > LEAST($3 * POWER(2.0, GREATEST(mt.attempt_count - 1, 0)::float), $4)) AND ($2 - m.time_sent) < $4 FOR UPDATE OF mt SKIP LOCKED `, target.MsgID, now, RetryInterval, RetryMaxAge) @@ -205,8 +264,8 @@ func deliverMessage(target pendingTarget) { WHERE mat.msg_id = $1 AND mat.time_delivered IS NULL AND m.time_sent IS NOT NULL - AND (mat.response_code IS NULL OR mat.response_code IN (3, 5)) - AND (mat.time_last_attempt IS NULL OR ($2 - mat.time_last_attempt) > $3) + AND (mat.response_code IS NULL OR mat.response_code IN (3, 5, 101)) + AND (mat.time_last_attempt IS NULL OR ($2 - mat.time_last_attempt) > LEAST($3 * POWER(2.0, GREATEST(mat.attempt_count - 1, 0)::float), $4)) AND ($2 - m.time_sent) < $4 FOR UPDATE OF mat SKIP LOCKED `, target.MsgID, now, RetryInterval, RetryMaxAge) @@ -258,19 +317,13 @@ func deliverMessage(target pendingTarget) { return } - // Register in outgoing map so challenge handler can look up this message + // Compute header hash now; registerOutgoing with Host B's IP happens after + // the connection is established (IP needed for challenge validation §10.5). hash := h.GetHeaderHash() hashArr := *(*[32]byte)(hash) - log.Printf("INFO: sender: registering outgoing message %s", hex.EncodeToString(hash[:])) - registerOutgoing(hashArr, h) - defer deleteOutgoing(hashArr) - - // Build the list of recipients on the target domain (in order) and - // note which ones we locked (i.e. are pending delivery this round). Per spec, - // response codes arrive per-recipient in to then add-to order excluding other domains. - // TODO check retry logic when some recipients on this domain are not locked - // (e.g. already delivered or locked by another sender) — do we still get - // per-recipient codes for the locked ones? + + // Build the list of recipients on the target domain in to then add-to order. + // Per spec, per-recipient response codes follow the same order. lockedSet := make(map[string]bool) for _, a := range lockedAddrs { lockedSet[strings.ToLower(a)] = true @@ -307,13 +360,10 @@ func deliverMessage(target pendingTarget) { // --- network delivery --- - // TODO [Spec]: DNSSEC validation SHOULD be performed on the DNS lookup. - // If DNSSEC validation fails the connection MUST terminate (no retry). - // lookupAuthorisedIPs does not currently perform or report DNSSEC validation. targetIPs, err := lookupAuthorisedIPs(target.Domain) if err != nil { log.Printf("ERROR: sender: DNS lookup for _fmsg.%s failed: %s", target.Domain, err) - return // rollback, retry later + return } var conn net.Conn @@ -327,177 +377,101 @@ func deliverMessage(target pendingTarget) { } if conn == nil { log.Printf("ERROR: sender: could not connect to any IP for _fmsg.%s", target.Domain) - return // rollback, retry later + return } defer conn.Close() - // Send header — Encode() writes all fields through attachment headers per spec - headerBytes := h.Encode() - if _, err := conn.Write(headerBytes); err != nil { - log.Printf("ERROR: sender: writing header: %s", err) - return - } + // Register in outgoing map with Host B's IP before sending the header so + // any incoming challenge can be matched by hash AND IP (§10.2 step 2). + connectedIP := conn.RemoteAddr().(*net.TCPAddr).IP.String() + log.Printf("INFO: sender: registering outgoing message %s (%s)", hex.EncodeToString(hashArr[:]), connectedIP) + registerOutgoing(hashArr, h, connectedIP) + defer removeOutgoingIP(hashArr, connectedIP) - // message data - fd, err := os.Open(h.Filepath) - if err != nil { - log.Printf("ERROR: sender: opening file %s: %s", h.Filepath, err) + // Step 3: Transmit message header. + if _, err := conn.Write(h.Encode()); err != nil { + log.Printf("ERROR: sender: writing header for msg %d: %s", target.MsgID, err) return } - defer fd.Close() - conn.SetWriteDeadline(time.Now().Add(calcNetIODuration(int(h.Size), MinUploadRate))) - n, err := io.CopyN(conn, fd, int64(h.Size)) - if n != int64(h.Size) { - log.Printf("ERROR: sender: file size mismatch for msg %d: expected %d, got %d", target.MsgID, h.Size, n) - return - } - if err != nil { - log.Printf("ERROR: sender: sending data (%d/%d bytes): %s", n, h.Size, err) + // Step 5: Read the initial response byte before sending any data (§10.2 step 5). + // The challenge handler may fire on a separate goroutine during this wait. + conn.SetReadDeadline(time.Now().Add(30 * time.Second)) + initCode := make([]byte, 1) + if _, err := io.ReadFull(conn, initCode); err != nil { + log.Printf("ERROR: sender: reading initial response for msg %d: %s", target.MsgID, err) return } + now = timeutil.TimestampNow().Float64() + isAddToMsg := h.Flags&FlagHasAddTo != 0 - // attachment data — sequential byte sequences bounded by header sizes - for _, att := range h.Attachments { - af, err := os.Open(att.Filepath) - if err != nil { - log.Printf("ERROR: sender: opening attachment %s: %s", att.Filename, err) + switch initCode[0] { + case AcceptCodeContinue: // 64 — send data + attachments, then per-recipient codes + if err := sendMsgData(conn, h); err != nil { + log.Printf("ERROR: sender: %s (msg %d)", err, target.MsgID) return } - an, err := io.CopyN(conn, af, int64(att.Size)) - af.Close() - if an != int64(att.Size) { - log.Printf("ERROR: sender: attachment %s size mismatch: expected %d, got %d", att.Filename, att.Size, an) + case AcceptCodeSkipData: // 65 — add-to, parent stored, recipients on this host; skip data + if !isAddToMsg { + log.Printf("WARN: sender: msg %d received protocol-invalid code 65 from %s for non-add-to message, terminating", + target.MsgID, target.Domain) return } - if err != nil { - log.Printf("ERROR: sender: sending attachment %s: %s", att.Filename, err) - return - } - } - - // --- read response --- - // A code < 100 is a global rejection (single byte for all recipients). - // Otherwise one code per recipient on this domain, in To-field order. - conn.SetReadDeadline(time.Now().Add(30 * time.Second)) - - firstByte := make([]byte, 1) - if _, err := io.ReadFull(conn, firstByte); err != nil { - log.Printf("ERROR: sender: reading response: %s", err) - return // rollback, retry later - } - - code := firstByte[0] - now = timeutil.TimestampNow().Float64() - - if code < 100 { - // Code 11 (accept add to) — additional recipients received. - if code == AcceptCodeAddTo { - log.Printf("INFO: sender: msg %d additional recipients received by %s (code 11)", target.MsgID, target.Domain) - for _, a := range lockedAddrs { - if _, err := tx.Exec(` - UPDATE msg_to SET time_delivered = $1, response_code = $2 - WHERE msg_id = $3 AND addr = $4 - `, now, int(code), target.MsgID, a); err != nil { - log.Printf("ERROR: sender: update delivered for %s: %s", a, err) - } - } - for _, a := range lockedAddToAddrs { - if _, err := tx.Exec(` - UPDATE msg_add_to SET time_delivered = $1, response_code = $2 - WHERE msg_id = $3 AND addr = $4 - `, now, int(code), target.MsgID, a); err != nil { - log.Printf("ERROR: sender: update delivered add-to for %s: %s", a, err) - } - } - if err := tx.Commit(); err != nil { - log.Printf("ERROR: sender: commit tx for msg %d: %s", target.MsgID, err) - } else { - committed = true - } + // do not transmit data; per-recipient codes follow below + case AcceptCodeAddTo: // 11 — add-to accepted, no recipients on this host + if !isAddToMsg { + log.Printf("WARN: sender: msg %d received protocol-invalid code 11 from %s for non-add-to message, terminating", + target.MsgID, target.Domain) return } - - // global rejection — update all locked recipients - // - // TODO [Spec]: Permanent failures (1 invalid, 2 unsupported version, - // 4 too big, 10 duplicate) should NOT be retried. Currently all global - // codes are stored identically; findPendingTargets only retries codes - // 3 and 5, which is correct for global codes. But ensure code 10 - // (duplicate) is explicitly recognised as permanent and not retried. - log.Printf("WARN: sender: msg %d rejected by %s: %s (%d)", - target.MsgID, target.Domain, responseCodeName(code), code) - for _, a := range lockedAddrs { - if _, err := tx.Exec(` - UPDATE msg_to SET time_last_attempt = $1, response_code = $2 - WHERE msg_id = $3 AND addr = $4 - `, now, int(code), target.MsgID, a); err != nil { - log.Printf("ERROR: sender: update last attempt for %s: %s", a, err) - } - } - for _, a := range lockedAddToAddrs { - if _, err := tx.Exec(` - UPDATE msg_add_to SET time_last_attempt = $1, response_code = $2 - WHERE msg_id = $3 AND addr = $4 - `, now, int(code), target.MsgID, a); err != nil { - log.Printf("ERROR: sender: update last attempt add-to for %s: %s", a, err) - } - } - if err := tx.Commit(); err != nil { - log.Printf("ERROR: sender: commit tx for msg %d: %s", target.MsgID, err) + log.Printf("INFO: sender: msg %d add-to accepted by %s (code 11)", target.MsgID, target.Domain) + updateAllLocked(tx, lockedAddrs, lockedAddToAddrs, target.MsgID, now, int(initCode[0]), true) + commitOrLog(tx, &committed, target.MsgID) + return + default: + if initCode[0] >= 1 && initCode[0] <= 10 { + // global rejection + log.Printf("WARN: sender: msg %d rejected by %s: %s (%d)", + target.MsgID, target.Domain, responseCodeName(initCode[0]), initCode[0]) + updateAllLocked(tx, lockedAddrs, lockedAddToAddrs, target.MsgID, now, int(initCode[0]), false) + commitOrLog(tx, &committed, target.MsgID) } else { - committed = true + // unexpected code — TERMINATE + log.Printf("WARN: sender: msg %d unexpected response code %d from %s, terminating", + target.MsgID, initCode[0], target.Domain) } return } - // per-recipient codes + // Step 6: Read one per-recipient code per recipient on this host, in + // to-field order then add-to order (§10.2 step 6). + conn.SetReadDeadline(time.Now().Add(30 * time.Second)) codes := make([]byte, len(domainRecips)) - codes[0] = code - if len(domainRecips) > 1 { - rest := make([]byte, len(domainRecips)-1) - if _, err := io.ReadFull(conn, rest); err != nil { - log.Printf("ERROR: sender: reading remaining response codes: %s", err) - return // rollback, retry later - } - copy(codes[1:], rest) + if _, err := io.ReadFull(conn, codes); err != nil { + log.Printf("ERROR: sender: reading per-recipient codes for msg %d: %s", target.MsgID, err) + return } + now = timeutil.TimestampNow().Float64() for i, dr := range domainRecips { if !dr.isLocked { - continue // not our responsibility this round - // TODO well receiving host still attempted delivery to this recipient — do we update response code for it? Spec is not clear on this scenario. + continue } c := codes[i] table := "msg_to" if dr.isAddTo { table = "msg_add_to" } - if c == RejectCodeAccept { + delivered := c == RejectCodeAccept + if delivered { log.Printf("INFO: sender: delivered msg %d to %s", target.MsgID, dr.addr) - if _, err := tx.Exec(fmt.Sprintf(` - UPDATE %s SET time_delivered = $1, response_code = 200 - WHERE msg_id = $2 AND addr = $3 - `, table), now, target.MsgID, dr.addr); err != nil { - log.Printf("ERROR: sender: update delivered for %s: %s", dr.addr, err) - } } else { - log.Printf("WARN: sender: msg %d to %s rejected: %s (%d)", - target.MsgID, dr.addr, responseCodeName(c), c) - if _, err := tx.Exec(fmt.Sprintf(` - UPDATE %s SET time_last_attempt = $1, response_code = $2 - WHERE msg_id = $3 AND addr = $4 - `, table), now, int(c), target.MsgID, dr.addr); err != nil { - log.Printf("ERROR: sender: update last attempt for %s: %s", dr.addr, err) - } + log.Printf("WARN: sender: msg %d to %s: %s (%d)", target.MsgID, dr.addr, responseCodeName(c), c) } + updateRecipient(tx, table, dr.addr, target.MsgID, now, int(c), delivered) } - if err := tx.Commit(); err != nil { - log.Printf("ERROR: sender: commit tx for msg %d: %s", target.MsgID, err) - } else { - committed = true - } + commitOrLog(tx, &committed, target.MsgID) } // processPendingMessages finds messages needing delivery and dispatches a diff --git a/src/store.go b/src/store.go index bbc65d5..19775d3 100644 --- a/src/store.go +++ b/src/store.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "log" + "strings" "github.com/levenlabs/golib/timeutil" _ "github.com/lib/pq" @@ -61,6 +62,74 @@ func lookupMsgIdByHash(hash []byte) (int64, error) { return id, err } +// hasAddrReceivedMsgHash reports whether addr has already received a stored +// message identified by hash. +func hasAddrReceivedMsgHash(hash []byte, addr *FMsgAddress) (bool, error) { + if addr == nil || len(hash) == 0 { + return false, nil + } + + db, err := sql.Open("postgres", "") + if err != nil { + return false, err + } + defer db.Close() + + addrStr := strings.ToLower(addr.ToString()) + + var exists bool + err = db.QueryRow(` + SELECT EXISTS ( + SELECT 1 + FROM msg m + JOIN msg_to mt ON mt.msg_id = m.id + WHERE m.sha256 = $1 + AND lower(mt.addr) = $2 + AND mt.time_delivered IS NOT NULL + UNION ALL + SELECT 1 + FROM msg m + JOIN msg_add_to mat ON mat.msg_id = m.id + WHERE m.sha256 = $1 + AND lower(mat.addr) = $2 + AND mat.time_delivered IS NOT NULL + ) + `, hash, addrStr).Scan(&exists) + if err != nil { + return false, err + } + + return exists, nil +} + +// getMsgByID loads a message and all its recipients from the database by msg ID. +// Returns the full FMsgHeader or nil if the message doesn't exist. +func getMsgByID(msgID int64) (*FMsgHeader, error) { + db, err := sql.Open("postgres", "") + if err != nil { + return nil, err + } + defer db.Close() + + tx, err := db.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + + h, err := loadMsg(tx, msgID) + if err != nil { + // If the message doesn't exist, loadMsg will return an error, + // but we want to distinguish "not found" from other errors + if err.Error() == "no rows in result set" || err == sql.ErrNoRows { + return nil, nil + } + return nil, err + } + + return h, nil +} + func storeMsgDetail(msg *FMsgHeader) error { db, err := sql.Open("postgres", "") @@ -80,6 +149,11 @@ func storeMsgDetail(msg *FMsgHeader) error { return err } + var addToFrom interface{} + if msg.AddToFrom != nil { + addToFrom = msg.AddToFrom.ToString() + } + var msgID int64 err = tx.QueryRow(`insert into msg (version , no_reply @@ -87,13 +161,14 @@ func storeMsgDetail(msg *FMsgHeader) error { , is_deflate , time_sent , from_addr + , add_to_from , topic , type , sha256 , psha256 , size , filepath) -values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) +values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) returning id`, msg.Version, msg.Flags&FlagNoReply != 0, @@ -101,6 +176,7 @@ returning id`, msg.Flags&FlagDeflate != 0, msg.Timestamp, msg.From.ToString(), + addToFrom, msg.Topic, msg.Type, msgHash, @@ -150,6 +226,22 @@ values ($1, $2, $3)`) } } + if len(msg.Attachments) > 0 { + attStmt, err := tx.Prepare(`insert into msg_attachment (msg_id, position, flags, type, filename, filesize, filepath) +values ($1, $2, $3, $4, $5, $6, $7)`) + if err != nil { + return err + } + defer attStmt.Close() + + for i := range msg.Attachments { + att := msg.Attachments[i] + if _, err := attStmt.Exec(msgID, i, int(att.Flags), att.Type, att.Filename, int(att.Size), att.Filepath); err != nil { + return err + } + } + } + // resolve pid from psha256 (parent message hash) if len(msg.Pid) > 0 { var parentID sql.NullInt64 @@ -184,7 +276,15 @@ func storeMsgHeaderOnly(msg *FMsgHeader) error { } defer tx.Rollback() - headerHash := msg.GetHeaderHash() + msgHash, err := msg.GetMessageHash() + if err != nil { + return err + } + + var addToFrom interface{} + if msg.AddToFrom != nil { + addToFrom = msg.AddToFrom.ToString() + } var msgID int64 err = tx.QueryRow(`insert into msg (version @@ -193,13 +293,14 @@ func storeMsgHeaderOnly(msg *FMsgHeader) error { , is_deflate , time_sent , from_addr + , add_to_from , topic , type , sha256 , psha256 , size , filepath) -values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) +values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) returning id`, msg.Version, msg.Flags&FlagNoReply != 0, @@ -207,9 +308,10 @@ returning id`, msg.Flags&FlagDeflate != 0, msg.Timestamp, msg.From.ToString(), + addToFrom, msg.Topic, msg.Type, - headerHash, + msgHash, msg.Pid, int(msg.Size), "").Scan(&msgID) @@ -243,6 +345,22 @@ returning id`, } } + if len(msg.Attachments) > 0 { + attStmt, err := tx.Prepare(`insert into msg_attachment (msg_id, position, flags, type, filename, filesize, filepath) +values ($1, $2, $3, $4, $5, $6, $7)`) + if err != nil { + return err + } + defer attStmt.Close() + + for i := range msg.Attachments { + att := msg.Attachments[i] + if _, err := attStmt.Exec(msgID, i, int(att.Flags), att.Type, att.Filename, int(att.Size), att.Filepath); err != nil { + return err + } + } + } + // resolve pid from psha256 if len(msg.Pid) > 0 { var parentID sql.NullInt64 @@ -262,21 +380,17 @@ returning id`, // loadMsg loads a message and all its recipients from the database within the // given transaction and returns a fully populated FMsgHeader. -// -// TODO [Spec]: Attachment headers are not loaded. Once the msg_attachment table -// stores attachment metadata (flags, type, filename, size, filepath), loadMsg -// should populate FMsgHeader.Attachments so the sender can write attachment -// headers and data on the wire and compute a correct header/message hash. func loadMsg(tx *sql.Tx, msgID int64) (*FMsgHeader, error) { var version, size int var noReply, isImportant, isDeflate bool var pid, msgHash []byte var fromAddr, topic, typ, filepath string + var addToFromAddr sql.NullString var timeSent float64 err := tx.QueryRow(` - SELECT version, no_reply, is_important, is_deflate, psha256, sha256, from_addr, topic, type, time_sent, size, filepath + SELECT version, no_reply, is_important, is_deflate, psha256, sha256, from_addr, add_to_from, topic, type, time_sent, size, filepath FROM msg WHERE id = $1 - `, msgID).Scan(&version, &noReply, &isImportant, &isDeflate, &pid, &msgHash, &fromAddr, &topic, &typ, &timeSent, &size, &filepath) + `, msgID).Scan(&version, &noReply, &isImportant, &isDeflate, &pid, &msgHash, &fromAddr, &addToFromAddr, &topic, &typ, &timeSent, &size, &filepath) if err != nil { return nil, fmt.Errorf("load msg %d: %w", msgID, err) } @@ -336,6 +450,36 @@ func loadMsg(tx *sql.Tx, msgID int64) (*FMsgHeader, error) { return nil, fmt.Errorf("add-to recipients query err for msg %d: %w", msgID, err) } + attRows, err := tx.Query(` + SELECT flags, type, filename, filesize, filepath + FROM msg_attachment + WHERE msg_id = $1 + ORDER BY position, filename + `, msgID) + if err != nil { + return nil, fmt.Errorf("load attachments for msg %d: %w", msgID, err) + } + attachments := []FMsgAttachmentHeader{} + for attRows.Next() { + var flags, filesize int + var typ, filename, filepath string + if err := attRows.Scan(&flags, &typ, &filename, &filesize, &filepath); err != nil { + attRows.Close() + return nil, fmt.Errorf("scan attachment row: %w", err) + } + attachments = append(attachments, FMsgAttachmentHeader{ + Flags: uint8(flags), + Type: typ, + Filename: filename, + Size: uint32(filesize), + Filepath: filepath, + }) + } + attRows.Close() + if err := attRows.Err(); err != nil { + return nil, fmt.Errorf("attachments query err for msg %d: %w", msgID, err) + } + // Compute flags bitfield from stored booleans and loaded data. // has_pid and has_add_to are derived from actual data rather than stored, // so add-to recipients added after the original message are included. @@ -347,6 +491,21 @@ func loadMsg(tx *sql.Tx, msgID int64) (*FMsgHeader, error) { if len(allAddTo) > 0 && len(pid) == 0 { pid = msgHash } + + var addToFrom *FMsgAddress + if addToFromAddr.Valid && addToFromAddr.String != "" { + addr, err := parseAddress([]byte(addToFromAddr.String)) + if err != nil { + return nil, fmt.Errorf("invalid add_to_from address %s: %w", addToFromAddr.String, err) + } + addToFrom = addr + } + if len(allAddTo) > 0 && addToFrom == nil { + // Backward-compatibility for older rows before add_to_from existed. + fallback := *from + addToFrom = &fallback + } + var flags uint8 if len(pid) > 0 { flags |= FlagHasPid @@ -365,16 +524,18 @@ func loadMsg(tx *sql.Tx, msgID int64) (*FMsgHeader, error) { } return &FMsgHeader{ - Version: uint8(version), - Flags: flags, - Pid: pid, - From: *from, - To: allTo, - AddTo: allAddTo, - Timestamp: timeSent, - Topic: topic, - Type: typ, - Size: uint32(size), - Filepath: filepath, + Version: uint8(version), + Flags: flags, + Pid: pid, + From: *from, + To: allTo, + AddToFrom: addToFrom, + AddTo: allAddTo, + Timestamp: timeSent, + Topic: topic, + Type: typ, + Size: uint32(size), + Attachments: attachments, + Filepath: filepath, }, nil }