add to

README.md

@@ -40,7 +40,7 @@ Tested with Go 1.25 on Linux and Windows, AMD64 and ARM
 
 ## Running
 
-An up and running [fmsg Id API](https://github.com/markmnl/fmsgid) needs to be reachable by fmsgd to know users and their quotas for this fmsgd service.  
+An up and running [fmsg Id API](https://github.com/markmnl/fmsgid) needs to be reachable by fmsgd to know users and their quotas for this fmsgd service. See also [fmsg-docker](https://github.com/markmnl/fmsg-docker) - a docker compose stack for a fmsg host including fmsgid, fmsg-webpi and fmsgd.
 
 IP address to bind to and listen on is the only argument, `127.0.0.1` is used if argument not supplied. e.g. on Linux:
 

TODO.md

@@ -8,20 +8,6 @@ correctness issues, then enhancements.
 ## P0 — Foundational (blocks most other work)
 
 
-### 2. Fix `Encode()` to produce the full message header per spec
-**File:** `defs.go` `Encode()`  
-Currently encodes only version through type. Missing: add-to field, size
-(uint32), attachment headers (uint8 count + headers). Topic is always encoded
-but must be absent when pid is set. Type is always length-prefixed but must be a
-single uint8 index when the common-type flag (bit 2) is set. The header hash
-(SHA-256 of encoded header) is wrong without these fields, breaking challenge
-verification and pid references.
-
-### 3. Add `AddTo` field to `FMsgHeader`
-**File:** `defs.go` struct  
-Add `AddTo []FMsgAddress` field. Required before any add-to parsing, encoding,
-storage, or per-recipient response ordering can work.
-
 ### 4. Add `Attachments` field to `FMsgHeader`
 **File:** `defs.go` struct  
 Add `Attachments []FMsgAttachmentHeader` to store parsed attachment headers
@@ -35,14 +21,6 @@ Add a `ChallengeCompleted bool` to distinguish "challenge was completed and
 hash verification check in `downloadMessage` erroneously fails when the
 challenge was skipped.
 
-### ~~6. Fix response code constants~~
-**File:** `host.go` constants  
-~~`RejectCodeMustChallenge` (11) and `RejectCodeCannotChallenge` (12) do not
-exist in the spec. Code 11 = "accept header" (add-to notification success).
-Add missing per-user codes: 102 (user not accepting), 103 (user undisclosed).~~
-**DONE:** Replaced with `AcceptCodeHeader` (11), added `RejectCodeUserNotAccepting` (102)
-and `RejectCodeUserUndisclosed` (103).
-
 ---
 
 ## P1 — Receiving path (host.go) correctness
@@ -54,11 +32,6 @@ connection's IP is authorised. If not → TERMINATE (no reject code). Currently
 this only happens inside `challenge()` and is skipped when skip-challenge is
 allowed.
 
-### 8. Parse "add to" field when has-add-to flag is set
-**File:** `host.go` `readHeader()`  
-Spec 1.4.v.b: Read uint8 count + addresses. Verify distinct from each other and
-from "to" (case-insensitive). Implement the pid/add-to decision tree (add-to
-requires pid; if no add-to recipients on our domain → accept header code 11).
 
 ### 9. Make topic conditional on pid absence
 **File:** `host.go` `readHeader()`  
@@ -125,11 +98,10 @@ Currently only v==255 is handled.
 
 ## P2 — Sending path (sender.go) correctness
 
-### 21. Include add-to recipients in domain recipient list
+### ~~21. Include add-to recipients in domain recipient list~~ DONE
 **File:** `sender.go` `deliverMessage()`  
-`domainRecips` only iterates `h.To`. Per spec, per-recipient codes arrive in
-"to then add to" order. Append add-to recipients on the target domain after to
-recipients.
+`domainRecips` already iterates both `h.To` and `h.AddTo` in order, with
+`isAddTo` flag set appropriately. Implemented as part of the add-to feature.
 
 ### 22. Write attachment headers and attachment bodies
 **File:** `sender.go` `deliverMessage()`  

dd.sql

@@ -10,12 +10,14 @@ create table if not exists msg (
     id            	bigserial       	primary key,
 	version			int					not null,
     pid           	bigint          	references msg (id),
-	flags		  	int					not null,
+	no_reply		boolean				not null default false,
+	is_important	boolean				not null default false,
+	is_deflate		boolean				not null default false,
     time_sent     	double precision,             -- time sending host recieved message for sending, message timestamp field, NULL means message not ready for sending i.e. draft
     from_addr     	varchar(255)    	not null,
     topic         	varchar(255)    	not null, 
     type          	varchar(255)    	not null,
-    sha256        	bytea           	unique not null,
+    sha256        	bytea           	unique,
     psha256       	bytea,
 	size			int					not null, -- spec allows uint32 but we don't enforced by FMSG_MAX_MSG_SIZE
     filepath      	text            	not null
@@ -33,6 +35,17 @@ create table if not exists msg_to (
 );
 create index on msg_to ((lower(addr)));
 
+create table if not exists msg_add_to (
+	id				bigserial			primary key,
+	msg_id			bigint				not null references msg (id),
+	addr			varchar(255)		not null,
+    time_delivered  double precision,   -- if sending, time sending host recieved delivery confirmation, if receiving, time successfully received message
+    time_last_attempt double precision, -- only used when sending, time of last delivery attempt if failed; otherwise null
+    response_code   smallint,		    -- only used when sending, response code of last delivery attempt if failed; otherwise null
+	unique (msg_id, addr)
+);
+create index on msg_add_to ((lower(addr)));
+
 create table if not exists msg_attachment (
     msg_id        	bigint          references msg (id),
     filename      	varchar(255)    not null,
@@ -57,3 +70,10 @@ drop trigger if exists trg_msg_to_insert on msg_to;
 create trigger trg_msg_to_insert
     after insert on msg_to
     for each row execute function notify_msg_to_insert();
+
+-- notify when a new msg_add_to row is inserted with null time_delivered so the
+-- sender can pick it up immediately instead of waiting for the next poll.
+drop trigger if exists trg_msg_add_to_insert on msg_add_to;
+create trigger trg_msg_add_to_insert
+    after insert on msg_add_to
+    for each row execute function notify_msg_to_insert();

src/defs.go

@@ -24,13 +24,12 @@ type FMsgAttachmentHeader struct {
 }
 
 type FMsgHeader struct {
-	Version uint8
-	Flags   uint8
-	Pid     []byte
-	From    FMsgAddress
-	To      []FMsgAddress
-	// TODO [Spec]: Add AddTo []FMsgAddress field for the "add to" recipients
-	// (present when has-add-to flag bit 1 is set).
+	Version   uint8
+	Flags     uint8
+	Pid       []byte
+	From      FMsgAddress
+	To        []FMsgAddress
+	AddTo     []FMsgAddress
 	Timestamp float64
 	Topic     string
 	Type      string
@@ -63,7 +62,6 @@ func (addr *FMsgAddress) ToString() string {
 // instead of returning one.
 // TODO [Spec]: The spec defines "message header" as all fields up to and
 // including the attachment headers field. This Encode() is missing:
-//   - The "add to" field (uint8 count + addresses, when has-add-to flag set).
 //   - The "size" field (uint32).
 //   - The "attachment headers" field (uint8 count + list of attachment headers).
 //
@@ -86,6 +84,14 @@ func (h *FMsgHeader) Encode() []byte {
 		b.WriteByte(byte(len(str)))
 		b.WriteString(str)
 	}
+	if h.Flags&FlagHasAddTo != 0 {
+		b.WriteByte(byte(len(h.AddTo)))
+		for _, addr := range h.AddTo {
+			str = addr.ToString()
+			b.WriteByte(byte(len(str)))
+			b.WriteString(str)
+		}
+	}
 	if err := binary.Write(&b, binary.LittleEndian, h.Timestamp); err != nil {
 		panic(err)
 	}
@@ -111,6 +117,13 @@ func (h *FMsgHeader) String() string {
 			fmt.Fprintf(&b, "\n\t%s", addr.ToString())
 		}
 	}
+	for i, addr := range h.AddTo {
+		if i == 0 {
+			fmt.Fprintf(&b, "\nadd to:\t%s", addr.ToString())
+		} else {
+			fmt.Fprintf(&b, "\n\t%s", addr.ToString())
+		}
+	}
 	fmt.Fprintf(&b, "\ntopic:\t%s", h.Topic)
 	fmt.Fprintf(&b, "\ntype:\t%s", h.Type)
 	fmt.Fprintf(&b, "\nsize:\t%d", h.Size)