Behave better around DM dedupe/storage. Closes #77.

app/AGENTS.md

@@ -126,7 +126,7 @@ app/
 ### Echo/repeat dedup
 
 - Message uniqueness: `(type, conversation_key, text, sender_timestamp)`.
-- Duplicate insert is treated as an echo/repeat: the new path (if any) is appended, and the ACK count is incremented only for outgoing channel messages. Incoming repeats and direct-message duplicates may still add path data, but DM delivery state advances only from real ACK events.
+- Duplicate insert is treated as an echo/repeat: the new path (if any) is appended, and the ACK count is incremented only for outgoing channel messages. Incoming direct messages with the same conversation/text/sender timestamp also collapse onto one stored row, with later observations merging path data instead of creating a second DM.
 
 ### Raw packet dedup policy
 
@@ -364,7 +364,7 @@ tests/
 
 The MeshCore radio protocol encodes `sender_timestamp` as a 4-byte little-endian integer (Unix seconds). This is a firmware-level wire format — the radio, the Python library (`commands/messaging.py`), and the decoder (`decoder.py`) all read/write exactly 4 bytes. Millisecond Unix timestamps would overflow 4 bytes, so higher resolution is not possible without a firmware change.
 
-**Consequence:** Channel-message dedup still operates at 1-second granularity because the radio protocol only provides second-resolution `sender_timestamp`. Do not attempt to fix this by switching to millisecond timestamps — it will break echo dedup (the echo's 4-byte timestamp won't match the stored value) and overflow `to_bytes(4, "little")`. Direct messages no longer share that channel dedup index; they are deduplicated by raw-packet identity instead so legitimate same-text same-second DMs can coexist.
+**Consequence:** Message dedup still operates at 1-second granularity because the radio protocol only provides second-resolution `sender_timestamp`. Do not attempt to fix this by switching to millisecond timestamps — it will break echo dedup (the echo's 4-byte timestamp won't match the stored value) and overflow `to_bytes(4, "little")`. Incoming DMs now share the same second-resolution content identity tradeoff as channel echoes: same-contact same-text same-second observations collapse onto one stored row.
 
 ### Outgoing DM echoes remain undecrypted
 

app/database.py

@@ -50,10 +50,9 @@ CREATE TABLE IF NOT EXISTS messages (
     acked INTEGER DEFAULT 0,
     sender_name TEXT,
     sender_key TEXT
-    -- Deduplication: channel echoes/repeats use a channel-only unique index on
-    -- identical conversation/text/timestamp. Direct messages are deduplicated
-    -- separately via raw-packet linkage so legitimate same-text same-second DMs
-    -- can coexist.
+    -- Deduplication: channel echoes/repeats use a content/time unique index so
+    -- duplicate observations reconcile onto a single stored row. Legacy
+    -- databases may also gain an incoming-DM content index via migration 44.
     -- Enforced via idx_messages_dedup_null_safe (unique index) rather than a table constraint
     -- to avoid the storage overhead of SQLite's autoindex duplicating every message text.
 );
@@ -97,6 +96,7 @@ CREATE INDEX IF NOT EXISTS idx_raw_packets_message_id ON raw_packets(message_id)
 CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_packets_payload_hash ON raw_packets(payload_hash);
 CREATE INDEX IF NOT EXISTS idx_contacts_on_radio ON contacts(on_radio);
 -- idx_messages_sender_key is created by migration 25 (after adding the sender_key column)
+-- idx_messages_incoming_priv_dedup is created by migration 44 after legacy rows are reconciled
 CREATE INDEX IF NOT EXISTS idx_contact_advert_paths_recent
     ON contact_advert_paths(public_key, last_seen DESC);
 CREATE INDEX IF NOT EXISTS idx_contact_name_history_key

app/migrations.py

@@ -8,6 +8,7 @@ This approach is safe for existing users - their databases have user_version=0,
 so all migrations run in order on first startup after upgrade.
 """
 
+import json
 import logging
 from hashlib import sha256
 
@@ -338,6 +339,13 @@ async def run_migrations(conn: aiosqlite.Connection) -> int:
         await set_version(conn, 43)
         applied += 1
 
+    # Migration 44: Deduplicate incoming direct messages by content/timestamp
+    if version < 44:
+        logger.info("Applying migration 44: dedupe incoming direct messages")
+        await _migrate_044_dedupe_incoming_direct_messages(conn)
+        await set_version(conn, 44)
+        applied += 1
+
     if applied > 0:
         logger.info(
             "Applied %d migration(s), schema now at version %d", applied, await get_version(conn)
@@ -2476,3 +2484,154 @@ async def _migrate_043_split_message_dedup_by_type(conn: aiosqlite.Connection) -
            WHERE type = 'CHAN'"""
     )
     await conn.commit()
+
+
+def _merge_message_paths(paths_json_values: list[str | None]) -> str | None:
+    """Merge multiple message path arrays into one exact-observation list."""
+    merged: list[dict[str, object]] = []
+    seen: set[tuple[object | None, object | None, object | None]] = set()
+
+    for paths_json in paths_json_values:
+        if not paths_json:
+            continue
+        try:
+            parsed = json.loads(paths_json)
+        except (TypeError, json.JSONDecodeError):
+            continue
+        if not isinstance(parsed, list):
+            continue
+        for entry in parsed:
+            if not isinstance(entry, dict):
+                continue
+            key = (
+                entry.get("path"),
+                entry.get("received_at"),
+                entry.get("path_len"),
+            )
+            if key in seen:
+                continue
+            seen.add(key)
+            merged.append(entry)
+
+    return json.dumps(merged) if merged else None
+
+
+async def _migrate_044_dedupe_incoming_direct_messages(conn: aiosqlite.Connection) -> None:
+    """Collapse same-contact same-text same-second incoming DMs into one row."""
+    cursor = await conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='messages'"
+    )
+    if await cursor.fetchone() is None:
+        await conn.commit()
+        return
+
+    cursor = await conn.execute("PRAGMA table_info(messages)")
+    columns = {row[1] for row in await cursor.fetchall()}
+    required_columns = {
+        "id",
+        "type",
+        "conversation_key",
+        "text",
+        "sender_timestamp",
+        "received_at",
+        "paths",
+        "txt_type",
+        "signature",
+        "outgoing",
+        "acked",
+        "sender_name",
+        "sender_key",
+    }
+    if not required_columns.issubset(columns):
+        logger.debug("messages table missing incoming-DM dedup columns, skipping migration 44")
+        await conn.commit()
+        return
+
+    raw_packets_cursor = await conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='raw_packets'"
+    )
+    raw_packets_exists = await raw_packets_cursor.fetchone() is not None
+
+    duplicate_groups_cursor = await conn.execute(
+        """
+        SELECT conversation_key, text,
+               COALESCE(sender_timestamp, 0) AS normalized_sender_timestamp,
+               COUNT(*) AS duplicate_count
+        FROM messages
+        WHERE type = 'PRIV' AND outgoing = 0
+        GROUP BY conversation_key, text, COALESCE(sender_timestamp, 0)
+        HAVING COUNT(*) > 1
+        """
+    )
+    duplicate_groups = await duplicate_groups_cursor.fetchall()
+
+    for group in duplicate_groups:
+        normalized_sender_timestamp = group["normalized_sender_timestamp"]
+        rows_cursor = await conn.execute(
+            """
+            SELECT *
+            FROM messages
+            WHERE type = 'PRIV' AND outgoing = 0
+              AND conversation_key = ? AND text = ?
+              AND COALESCE(sender_timestamp, 0) = ?
+            ORDER BY id ASC
+            """,
+            (
+                group["conversation_key"],
+                group["text"],
+                normalized_sender_timestamp,
+            ),
+        )
+        rows = list(await rows_cursor.fetchall())
+        if len(rows) < 2:
+            continue
+
+        keeper = rows[0]
+        duplicate_ids = [row["id"] for row in rows[1:]]
+        merged_paths = _merge_message_paths([row["paths"] for row in rows])
+        merged_received_at = min(row["received_at"] for row in rows)
+        merged_txt_type = next((row["txt_type"] for row in rows if row["txt_type"] != 0), 0)
+        merged_signature = next((row["signature"] for row in rows if row["signature"]), None)
+        merged_sender_name = next((row["sender_name"] for row in rows if row["sender_name"]), None)
+        merged_sender_key = next((row["sender_key"] for row in rows if row["sender_key"]), None)
+        merged_acked = max(int(row["acked"] or 0) for row in rows)
+
+        await conn.execute(
+            """
+            UPDATE messages
+            SET received_at = ?, paths = ?, txt_type = ?, signature = ?,
+                acked = ?, sender_name = ?, sender_key = ?
+            WHERE id = ?
+            """,
+            (
+                merged_received_at,
+                merged_paths,
+                merged_txt_type,
+                merged_signature,
+                merged_acked,
+                merged_sender_name,
+                merged_sender_key,
+                keeper["id"],
+            ),
+        )
+
+        if raw_packets_exists:
+            for duplicate_id in duplicate_ids:
+                await conn.execute(
+                    "UPDATE raw_packets SET message_id = ? WHERE message_id = ?",
+                    (keeper["id"], duplicate_id),
+                )
+
+        placeholders = ",".join("?" for _ in duplicate_ids)
+        await conn.execute(
+            f"DELETE FROM messages WHERE id IN ({placeholders})",
+            duplicate_ids,
+        )
+
+    await conn.execute("DROP INDEX IF EXISTS idx_messages_incoming_priv_dedup")
+    await conn.execute(
+        """CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_incoming_priv_dedup
+           ON messages(type, conversation_key, text, COALESCE(sender_timestamp, 0))
+           WHERE type = 'PRIV' AND outgoing = 0"""
+    )
+    await conn.commit()

app/repository/messages.py

@@ -66,9 +66,10 @@ class MessageRepository:
     ) -> int | None:
         """Create a message, returning the ID or None if duplicate.
 
-        Uses INSERT OR IGNORE to handle the UNIQUE constraint on
-        (type, conversation_key, text, sender_timestamp). This prevents
-        duplicate messages when the same message arrives via multiple RF paths.
+        Uses INSERT OR IGNORE to handle the message dedup indexes:
+        - channel messages dedupe by content/timestamp for echo reconciliation
+        - incoming direct messages dedupe by conversation/text/timestamp so
+          raw-packet and fallback observations merge onto one row
 
         The path parameter is converted to the paths JSON array format.
         """
@@ -559,16 +560,20 @@ class MessageRepository:
         conversation_key: str,
         text: str,
         sender_timestamp: int | None,
+        outgoing: bool | None = None,
     ) -> "Message | None":
         """Look up a message by its unique content fields."""
-        cursor = await db.conn.execute(
-            """
+        query = """
             SELECT * FROM messages
             WHERE type = ? AND conversation_key = ? AND text = ?
               AND (sender_timestamp = ? OR (sender_timestamp IS NULL AND ? IS NULL))
-            """,
-            (msg_type, conversation_key, text, sender_timestamp, sender_timestamp),
-        )
+        """
+        params: list[Any] = [msg_type, conversation_key, text, sender_timestamp, sender_timestamp]
+        if outgoing is not None:
+            query += " AND outgoing = ?"
+            params.append(1 if outgoing else 0)
+        query += " ORDER BY id ASC"
+        cursor = await db.conn.execute(query, params)
         row = await cursor.fetchone()
         if not row:
             return None

app/services/dm_ingest.py

@@ -152,6 +152,7 @@ async def _store_direct_message(
                 conversation_key=conversation_key,
                 text=text,
                 sender_timestamp=sender_timestamp,
+                outgoing=outgoing,
             )
             if existing_msg is not None:
                 await reconcile_duplicate_message(
@@ -185,6 +186,7 @@ async def _store_direct_message(
                 conversation_key=conversation_key,
                 text=text,
                 sender_timestamp=sender_timestamp,
+                outgoing=outgoing,
                 path=path,
                 received_at=received_at,
                 path_len=path_len,

app/services/messages.py

@@ -171,6 +171,7 @@ async def handle_duplicate_message(
     conversation_key: str,
     text: str,
     sender_timestamp: int,
+    outgoing: bool | None = None,
     path: str | None,
     received_at: int,
     path_len: int | None = None,
@@ -182,6 +183,7 @@ async def handle_duplicate_message(
         conversation_key=conversation_key,
         text=text,
         sender_timestamp=sender_timestamp,
+        outgoing=outgoing,
     )
     if not existing_msg:
         label = "message" if msg_type == "CHAN" else "DM"
@@ -246,6 +248,7 @@ async def create_message_from_decrypted(
             conversation_key=channel_key_normalized,
             text=text,
             sender_timestamp=timestamp,
+            outgoing=None,
             path=path,
             received_at=received,
             path_len=path_len,
@@ -355,6 +358,7 @@ async def create_fallback_channel_message(
             conversation_key=conversation_key_normalized,
             text=text,
             sender_timestamp=sender_timestamp,
+            outgoing=None,
             path=path,
             received_at=received_at,
             path_len=path_len,