Common Media Types and add_to batch table and logic

.env.example

@@ -0,0 +1,29 @@
+# fmsgd Environment Variables
+
+# Required
+FMSG_DATA_DIR=/opt/fmsg/data
+FMSG_DOMAIN=example.com
+FMSG_ID_URL=http://127.0.0.1:8080
+
+# Optional
+FMSG_MAX_MSG_SIZE=10240
+FMSG_PORT=4930
+FMSG_MAX_PAST_TIME_DELTA=604800
+FMSG_MAX_FUTURE_TIME_DELTA=300
+FMSG_MIN_DOWNLOAD_RATE=5000
+FMSG_MIN_UPLOAD_RATE=5000
+FMSG_READ_BUFFER_SIZE=1600
+FMSG_RETRY_INTERVAL=20
+FMSG_RETRY_MAX_AGE=86400
+FMSG_POLL_INTERVAL=10
+FMSG_MAX_CONCURRENT_SEND=1024
+FMSG_SKIP_DOMAIN_IP_CHECK=false
+FMSG_SKIP_AUTHORISED_IPS=false # WARNING: skipping authorised IP check is a security risk and should only be used for testing
+
+# PostgreSQL connection variables (see https://www.postgresql.org/docs/current/libpq-envars.html)
+PGHOST=
+PGPORT=
+PGUSER=
+PGPASSWORD=
+PGDATABASE=
+PGSSLMODE=disable

TODO2.md

@@ -10,51 +10,15 @@ These are foundational: the header hash (used for challenge verification and pid
 references) and message hash (used for duplicate detection, challenge response,
 and pid) are both wrong until Encode() is complete.
 
-### ~~1. Encode() must include size and attachment headers~~ DONE
-**File:** `defs.go` `Encode()`
-Spec defines "message header" as all fields through the attachment headers.
-Encode() currently stops after the type field — it omits:
-  - size (uint32 LE)
-  - attachment count (uint8) + attachment headers (flags, type, filename, size)
-
 The header hash (SHA-256 of the encoded header) will be incorrect without these
 fields, which breaks challenge verification and pid references.
 
-### ~~2. Encode() must omit topic when pid is set~~ DONE
-**File:** `defs.go` `Encode()`
-Spec: "When pid exists the entire topic field MUST NOT be included on the wire."
-Encode() always writes the topic. It must only write topic when pid is absent.
-
-### 3. Encode() must support common-type encoding for type field
-**File:** `defs.go` `Encode()`
-When common-type flag (bit 2) is set, type on the wire is a single uint8 index,
-not a length-prefixed string. Encode() always writes length-prefixed.
-
-### ~~4. Add Attachments field to FMsgHeader~~ DONE
-**File:** `defs.go` struct
-Add `Attachments []FMsgAttachmentHeader` to store parsed attachment headers.
-Required before Encode() can include attachment headers, before attachment
-parsing/validation/download, and before hash computation is correct.
-
-### ~~5. Complete FMsgAttachmentHeader struct~~ DONE
-**File:** `defs.go` struct
-Currently only has Filename, Size, Filepath. Missing per-spec fields:
-  - Flags (uint8) — including per-attachment common-type (bit 0) and deflate (bit 1)
-  - Type (string) — the attachment's media type
-
 ### 6. Add ChallengeCompleted flag to FMsgHeader
 **File:** `defs.go` struct
 Add `ChallengeCompleted bool` to distinguish "challenge was completed and
 ChallengeHash is valid" from "challenge was not performed." Without this, the
 hash check in downloadMessage erroneously fails when the challenge was skipped.
 
-### ~~7. GetMessageHash() must include attachment data~~ DONE
-**File:** `defs.go` `GetMessageHash()`
-Spec: message hash is SHA-256 of the entire message — header + data +
-attachment data. Currently attachment data (sequential byte sequences following
-the message body) is not included.
-
----
 
 ## P1 — Receiving: Header Exchange (host.go readHeader)
 
@@ -72,18 +36,7 @@ returns an error without sending any code.
 ### 10. Validate at least one "to" recipient
 **File:** `host.go` `readHeader()`
 Spec 1.4.i.a: If to count is 0, reject code 1 (invalid). Currently no check.
-
-### ~~11. Make topic conditional on pid absence~~ DONE
-**File:** `host.go` `readHeader()`
-Spec: topic field is only present when pid is NOT set. Currently topic is
-read unconditionally regardless of pid.
-
-### 12. Handle common-type flag for type field
-**File:** `host.go` `readHeader()`
-When common-type flag (bit 2) is set, type is a single uint8 index into the
-Common Media Types table. If the value has no mapping → reject code 1.
-Currently always reads type as a length-prefixed string.
-
+\
 ### 13. Parse and validate attachment headers
 **File:** `host.go` `readHeader()`
 Currently rejects any non-zero attachment count. Must:
@@ -124,11 +77,6 @@ referenced by pid." Not currently checked.
 
 ## P2 — Receiving: Challenge (host.go challenge)
 
-### ~~18. Connection 2 must target same IP as Connection 1~~ DONE
-**File:** `host.go` `challenge()`
-Spec 2.1: Dial conn.RemoteAddr() IP, not h.From.Domain. Dialling the domain
-may resolve to a different IP.
-
 ### 19. Make challenge mode configurable
 **File:** `host.go` `challenge()` / `handleConn()`
 Spec defines challenge modes (NEVER, ALWAYS, HAS_NOT_PARTICIPATED,

dd.sql

@@ -16,7 +16,8 @@ create table if not exists msg (
     time_sent     	double precision,             -- time sending host recieved message for sending, message timestamp field, NULL means message not ready for sending i.e. draft
     from_addr     	varchar(255)    	not null,
     topic         	varchar(255)    	not null, 
-    type          	varchar(255)    	not null,
+    type          	varchar(255),								-- NULL when common_type is set (full string not needed on wire)
+	common_type		smallint,							-- common media type number when common type flag was set on received message, NULL otherwise; needed to faithfully reconstruct wire bytes for hash computation
     sha256        	bytea           	unique,
     psha256       	bytea,
 	size			int					not null, -- spec allows uint32 but we don't enforced by FMSG_MAX_MSG_SIZE
@@ -35,14 +36,21 @@ create table if not exists msg_to (
 );
 create index on msg_to ((lower(addr)));
 
-create table if not exists msg_add_to (
+create table if not exists msg_add_to_batch (
 	id				bigserial			primary key,
 	msg_id			bigint				not null references msg (id),
+	batch_no		int					not null,
+	sha256			bytea				not null,   -- hash of message bytes with this batch's add_to recipients
+	unique (msg_id, batch_no)
+);
+
+create table if not exists msg_add_to (
+	id				bigserial			primary key,
+	batch_id		bigint				not null references msg_add_to_batch (id),
 	addr			varchar(255)		not null,
     time_delivered  double precision,   -- if sending, time sending host recieved delivery confirmation, if receiving, time successfully received message
     time_last_attempt double precision, -- only used when sending, time of last delivery attempt if failed; otherwise null
-    response_code   smallint,		    -- only used when sending, response code of last delivery attempt if failed; otherwise null
-	unique (msg_id, addr)
+    response_code   smallint		    -- only used when sending, response code of last delivery attempt if failed; otherwise null
 );
 create index on msg_add_to ((lower(addr)));
 
@@ -71,9 +79,40 @@ create trigger trg_msg_to_insert
     after insert on msg_to
     for each row execute function notify_msg_to_insert();
 
+-- notify when a new msg_to row is inserted with time_delivered set so that
+-- listeners can be notified a new message has arrived.
+create or replace function notify_msg_to_received() returns trigger as $$
+begin
+    if NEW.time_delivered is not null then
+        perform pg_notify('new_msg_from', NEW.msg_id::text || ',' || NEW.addr);
+    end if;
+    return NEW;
+end;
+$$ language plpgsql;
+
+drop trigger if exists trg_msg_to_received on msg_to;
+create trigger trg_msg_to_received
+    after insert on msg_to
+    for each row execute function notify_msg_to_received();
+
 -- notify when a new msg_add_to row is inserted with null time_delivered so the
 -- sender can pick it up immediately instead of waiting for the next poll.
+-- Must resolve msg_id through the batch table since msg_add_to references
+-- msg_add_to_batch, not msg directly.
+create or replace function notify_msg_add_to_insert() returns trigger as $$
+declare
+    v_msg_id bigint;
+begin
+    if NEW.time_delivered is null then
+        select msg_id into v_msg_id from msg_add_to_batch where id = NEW.batch_id;
+        perform pg_notify('new_msg_to', v_msg_id::text || ',' || NEW.addr)
+        from msg where id = v_msg_id and time_sent is not null;
+    end if;
+    return NEW;
+end;
+$$ language plpgsql;
+
 drop trigger if exists trg_msg_add_to_insert on msg_add_to;
 create trigger trg_msg_add_to_insert
     after insert on msg_add_to
-    for each row execute function notify_msg_to_insert();
+    for each row execute function notify_msg_add_to_insert();

src/defs.go

@@ -93,8 +93,18 @@ func (h *FMsgHeader) Encode() []byte {
 		b.WriteByte(byte(len(h.Topic)))
 		b.WriteString(h.Topic)
 	}
-	b.WriteByte(byte(len(h.Type)))
-	b.WriteString(h.Type)
+	// type: when common-type flag is set, write a single uint8 index;
+	// otherwise write uint8 length + ASCII string.
+	if h.Flags&FlagCommonType != 0 {
+		num, ok := mediaTypeToNumber[h.Type]
+		if !ok {
+			panic(fmt.Sprintf("common type flag set but %q has no mapping", h.Type))
+		}
+		b.WriteByte(num)
+	} else {
+		b.WriteByte(byte(len(h.Type)))
+		b.WriteString(h.Type)
+	}
 	// size (uint32 LE)
 	if err := binary.Write(&b, binary.LittleEndian, h.Size); err != nil {
 		panic(err)
@@ -103,8 +113,17 @@ func (h *FMsgHeader) Encode() []byte {
 	b.WriteByte(byte(len(h.Attachments)))
 	for _, att := range h.Attachments {
 		b.WriteByte(att.Flags)
-		b.WriteByte(byte(len(att.Type)))
-		b.WriteString(att.Type)
+		// attachment type: common-type flag is bit 0 of attachment flags
+		if att.Flags&AttachmentFlagCommonType != 0 {
+			num, ok := mediaTypeToNumber[att.Type]
+			if !ok {
+				panic(fmt.Sprintf("attachment common type flag set but %q has no mapping", att.Type))
+			}
+			b.WriteByte(num)
+		} else {
+			b.WriteByte(byte(len(att.Type)))
+			b.WriteString(att.Type)
+		}
 		b.WriteByte(byte(len(att.Filename)))
 		b.WriteString(att.Filename)
 		if err := binary.Write(&b, binary.LittleEndian, att.Size); err != nil {

src/defs_test.go

@@ -387,7 +387,7 @@ func TestEncodeWithAttachments(t *testing.T) {
 		Size:      100,
 		Attachments: []FMsgAttachmentHeader{
 			{Flags: 0, Type: "image/png", Filename: "pic.png", Size: 2048},
-			{Flags: 1, Type: "a", Filename: "doc.txt", Size: 512},
+			{Flags: AttachmentFlagCommonType, Type: "application/pdf", Filename: "doc.txt", Size: 512},
 		},
 	}
 	b := h.Encode()
@@ -449,16 +449,14 @@ func TestEncodeWithAttachments(t *testing.T) {
 		t.Fatalf("att[0] size = %d, want 2048", att0Size)
 	}
 
-	// attachment 1
+	// attachment 1 (common type: application/pdf = 6)
 	att1Flags, _ := r.ReadByte()
-	if att1Flags != 1 {
-		t.Fatalf("att[1] flags = %d, want 1", att1Flags)
+	if att1Flags != AttachmentFlagCommonType {
+		t.Fatalf("att[1] flags = %d, want %d", att1Flags, AttachmentFlagCommonType)
 	}
-	att1TypeLen, _ := r.ReadByte()
-	att1Type := make([]byte, att1TypeLen)
-	r.Read(att1Type)
-	if string(att1Type) != "a" {
-		t.Fatalf("att[1] type = %q, want %q", string(att1Type), "a")
+	att1TypeNum, _ := r.ReadByte()
+	if att1TypeNum != 6 {
+		t.Fatalf("att[1] common type number = %d, want 6 (application/pdf)", att1TypeNum)
 	}
 	att1FnLen, _ := r.ReadByte()
 	att1Fn := make([]byte, att1FnLen)
@@ -476,3 +474,70 @@ func TestEncodeWithAttachments(t *testing.T) {
 		t.Fatalf("unexpected %d trailing bytes", r.Len())
 	}
 }
+
+func TestEncodeCommonType(t *testing.T) {
+	// When FlagCommonType is set, the type field should be a single uint8
+	// index (56 = "text/plain;charset=UTF-8") instead of length+string.
+	h := &FMsgHeader{
+		Version:   1,
+		Flags:     FlagCommonType,
+		From:      FMsgAddress{User: "a", Domain: "b.com"},
+		To:        []FMsgAddress{{User: "c", Domain: "d.com"}},
+		Timestamp: 0,
+		Topic:     "",
+		Type:      "text/plain;charset=UTF-8",
+	}
+	b := h.Encode()
+	r := bytes.NewReader(b)
+
+	r.ReadByte() // version
+	r.ReadByte() // flags
+
+	// skip from
+	fLen, _ := r.ReadByte()
+	r.Read(make([]byte, fLen))
+	// skip to count + to[0]
+	r.ReadByte()
+	tLen, _ := r.ReadByte()
+	r.Read(make([]byte, tLen))
+	// skip timestamp
+	var ts float64
+	binary.Read(r, binary.LittleEndian, &ts)
+	// skip topic (present because no pid)
+	topicLen, _ := r.ReadByte()
+	r.Read(make([]byte, topicLen))
+
+	// type: should be single byte = 56
+	typeNum, _ := r.ReadByte()
+	if typeNum != 56 {
+		t.Fatalf("common type number = %d, want 56 (text/plain;charset=UTF-8)", typeNum)
+	}
+
+	// size + attachment count
+	var size uint32
+	binary.Read(r, binary.LittleEndian, &size)
+	attachCount, _ := r.ReadByte()
+	if attachCount != 0 {
+		t.Fatalf("attach count = %d, want 0", attachCount)
+	}
+
+	if r.Len() != 0 {
+		t.Fatalf("unexpected %d trailing bytes", r.Len())
+	}
+
+	// Encoding with common type should be shorter than without
+	h2 := &FMsgHeader{
+		Version:   1,
+		Flags:     0,
+		From:      FMsgAddress{User: "a", Domain: "b.com"},
+		To:        []FMsgAddress{{User: "c", Domain: "d.com"}},
+		Timestamp: 0,
+		Topic:     "",
+		Type:      "text/plain;charset=UTF-8",
+	}
+	b2 := h2.Encode()
+	// common type: 1 byte vs length-prefixed: 1 + 24 = 25 bytes → 24 bytes shorter
+	if len(b) >= len(b2) {
+		t.Fatalf("common type encoding (%d bytes) should be shorter than length-prefixed (%d bytes)", len(b), len(b2))
+	}
+}