Behave better around DM dedupe/storage. Closes #77.

tests/test_echo_dedup.py

@@ -614,6 +614,82 @@ class TestDualPathDedup:
         assert msg.paths is not None
         assert any(p.path == "bbcc" for p in msg.paths)
 
+    @pytest.mark.asyncio
+    async def test_incoming_duplicate_does_not_reconcile_onto_matching_outgoing_dm(
+        self, test_db, captured_broadcasts
+    ):
+        """Incoming DM duplicates must merge onto the incoming row, not a sent row."""
+        from app.event_handlers import on_contact_message
+        from app.packet_processor import create_dm_message_from_decrypted
+
+        await ContactRepository.upsert(
+            {
+                "public_key": CONTACT_PUB.lower(),
+                "name": "TestContact",
+                "type": 1,
+                "last_seen": SENDER_TIMESTAMP,
+                "last_contacted": SENDER_TIMESTAMP,
+                "first_seen": SENDER_TIMESTAMP,
+                "on_radio": False,
+                "out_path_hash_mode": 0,
+            }
+        )
+
+        outgoing_id = await MessageRepository.create(
+            msg_type="PRIV",
+            text="Mirror text",
+            conversation_key=CONTACT_PUB.lower(),
+            sender_timestamp=SENDER_TIMESTAMP,
+            received_at=SENDER_TIMESTAMP - 1,
+            outgoing=True,
+        )
+        assert outgoing_id is not None
+
+        pkt_id, _ = await RawPacketRepository.create(b"incoming_primary", SENDER_TIMESTAMP)
+        decrypted = DecryptedDirectMessage(
+            timestamp=SENDER_TIMESTAMP,
+            flags=0,
+            message="Mirror text",
+            dest_hash="fa",
+            src_hash="a1",
+        )
+
+        broadcasts, mock_broadcast = captured_broadcasts
+
+        with patch("app.packet_processor.broadcast_event", mock_broadcast):
+            incoming_id = await create_dm_message_from_decrypted(
+                packet_id=pkt_id,
+                decrypted=decrypted,
+                their_public_key=CONTACT_PUB,
+                our_public_key=OUR_PUB,
+                received_at=SENDER_TIMESTAMP,
+                outgoing=False,
+            )
+
+        assert incoming_id is not None
+        broadcasts.clear()
+
+        mock_event = MagicMock()
+        mock_event.payload = {
+            "public_key": CONTACT_PUB,
+            "text": "Mirror text",
+            "txt_type": 0,
+            "sender_timestamp": SENDER_TIMESTAMP,
+            "path": "bbcc",
+            "path_len": 2,
+        }
+
+        with patch("app.event_handlers.broadcast_event", mock_broadcast):
+            await on_contact_message(mock_event)
+
+        incoming_msg = await MessageRepository.get_by_id(incoming_id)
+        outgoing_msg = await MessageRepository.get_by_id(outgoing_id)
+        assert incoming_msg is not None
+        assert outgoing_msg is not None
+        assert incoming_msg.paths is not None
+        assert any(p.path == "bbcc" for p in incoming_msg.paths)
+        assert outgoing_msg.paths is None
+
     @pytest.mark.asyncio
     async def test_fallback_path_duplicate_reconciles_path_without_new_row(
         self, test_db, captured_broadcasts

tests/test_key_normalization.py

@@ -78,8 +78,8 @@ async def test_null_sender_timestamp_defaults_to_received_at(test_db):
 
 
 @pytest.mark.asyncio
-async def test_direct_messages_with_same_text_and_timestamp_are_allowed(test_db):
-    """Direct messages no longer share the channel echo dedup index."""
+async def test_incoming_direct_messages_with_same_text_and_timestamp_dedup(test_db):
+    """Incoming direct messages now collapse onto one content-identity row."""
     received_at = 600
     msg_id1 = await MessageRepository.create(
         msg_type="PRIV",
@@ -97,8 +97,7 @@ async def test_direct_messages_with_same_text_and_timestamp_are_allowed(test_db)
         sender_timestamp=received_at,
         received_at=received_at,
     )
-    assert msg_id2 is not None
-    assert msg_id2 != msg_id1
+    assert msg_id2 is None
 
 
 @pytest.mark.asyncio

tests/test_migrations.py

@@ -1,5 +1,7 @@
 """Tests for database migrations."""
 
+import json
+
 import aiosqlite
 import pytest
 
@@ -754,6 +756,121 @@ class TestMigration020:
             await conn.close()
 
 
+class TestMigration044:
+    """Test migration 044: dedupe incoming direct messages."""
+
+    @pytest.mark.asyncio
+    async def test_migration_merges_incoming_dm_duplicates_and_adds_index(self):
+        """Migration 44 collapses duplicate incoming DMs and re-links raw packets."""
+        conn = await aiosqlite.connect(":memory:")
+        conn.row_factory = aiosqlite.Row
+        try:
+            await set_version(conn, 43)
+
+            await conn.execute(
+                """
+                CREATE TABLE messages (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    type TEXT NOT NULL,
+                    conversation_key TEXT NOT NULL,
+                    text TEXT NOT NULL,
+                    sender_timestamp INTEGER,
+                    received_at INTEGER NOT NULL,
+                    paths TEXT,
+                    txt_type INTEGER DEFAULT 0,
+                    signature TEXT,
+                    outgoing INTEGER DEFAULT 0,
+                    acked INTEGER DEFAULT 0,
+                    sender_name TEXT,
+                    sender_key TEXT
+                )
+                """
+            )
+            await conn.execute(
+                """
+                CREATE TABLE raw_packets (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    timestamp INTEGER NOT NULL,
+                    data BLOB NOT NULL,
+                    message_id INTEGER
+                )
+                """
+            )
+            await conn.execute(
+                """
+                INSERT INTO messages
+                    (id, type, conversation_key, text, sender_timestamp, received_at, paths,
+                     txt_type, signature, outgoing, acked, sender_name, sender_key)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (1, "PRIV", "abc123", "hello", 0, 1001, None, 0, None, 0, 0, None, "abc123"),
+            )
+            await conn.execute(
+                """
+                INSERT INTO messages
+                    (id, type, conversation_key, text, sender_timestamp, received_at, paths,
+                     txt_type, signature, outgoing, acked, sender_name, sender_key)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    2,
+                    "PRIV",
+                    "abc123",
+                    "hello",
+                    None,
+                    1002,
+                    json.dumps([{"path": "", "received_at": 1002, "path_len": 0}]),
+                    2,
+                    "abcd",
+                    0,
+                    0,
+                    "Alice",
+                    "abc123",
+                ),
+            )
+            await conn.execute(
+                "INSERT INTO raw_packets (timestamp, data, message_id) VALUES (?, ?, ?)",
+                (1001, b"pkt1", 1),
+            )
+            await conn.execute(
+                "INSERT INTO raw_packets (timestamp, data, message_id) VALUES (?, ?, ?)",
+                (1002, b"pkt2", 2),
+            )
+            await conn.commit()
+
+            await run_migrations(conn)
+
+            cursor = await conn.execute("SELECT * FROM messages")
+            rows = await cursor.fetchall()
+            assert len(rows) == 1
+            assert rows[0]["id"] == 1
+            assert rows[0]["received_at"] == 1001
+            assert rows[0]["signature"] == "abcd"
+            assert rows[0]["txt_type"] == 2
+            assert rows[0]["sender_name"] == "Alice"
+            assert json.loads(rows[0]["paths"]) == [
+                {"path": "", "received_at": 1002, "path_len": 0}
+            ]
+
+            cursor = await conn.execute("SELECT message_id FROM raw_packets ORDER BY id")
+            assert [row["message_id"] for row in await cursor.fetchall()] == [1, 1]
+
+            cursor = await conn.execute(
+                "INSERT OR IGNORE INTO messages (type, conversation_key, text, sender_timestamp, received_at, outgoing) "
+                "VALUES (?, ?, ?, ?, ?, ?)",
+                ("PRIV", "abc123", "hello", 0, 9999, 0),
+            )
+            assert cursor.rowcount == 0
+
+            cursor = await conn.execute(
+                "SELECT sql FROM sqlite_master WHERE name='idx_messages_incoming_priv_dedup'"
+            )
+            index_sql = (await cursor.fetchone())["sql"]
+            assert "WHERE type = 'PRIV' AND outgoing = 0" in index_sql
+        finally:
+            await conn.close()
+
+
 class TestMigration028:
     """Test migration 028: convert payload_hash from TEXT to BLOB."""
 
@@ -1130,8 +1247,8 @@ class TestMigration039:
 
             applied = await run_migrations(conn)
 
-            assert applied == 5
-            assert await get_version(conn) == 43
+            assert applied == 6
+            assert await get_version(conn) == 44
 
             cursor = await conn.execute(
                 """
@@ -1200,8 +1317,8 @@ class TestMigration039:
 
             applied = await run_migrations(conn)
 
-            assert applied == 5
-            assert await get_version(conn) == 43
+            assert applied == 6
+            assert await get_version(conn) == 44
 
             cursor = await conn.execute(
                 """
@@ -1254,8 +1371,8 @@ class TestMigration040:
 
             applied = await run_migrations(conn)
 
-            assert applied == 4
-            assert await get_version(conn) == 43
+            assert applied == 5
+            assert await get_version(conn) == 44
 
             await conn.execute(
                 """
@@ -1316,8 +1433,8 @@ class TestMigration041:
 
             applied = await run_migrations(conn)
 
-            assert applied == 3
-            assert await get_version(conn) == 43
+            assert applied == 4
+            assert await get_version(conn) == 44
 
             await conn.execute(
                 """
@@ -1369,8 +1486,8 @@ class TestMigration042:
 
             applied = await run_migrations(conn)
 
-            assert applied == 2
-            assert await get_version(conn) == 43
+            assert applied == 3
+            assert await get_version(conn) == 44
 
             await conn.execute(
                 """

tests/test_packet_pipeline.py

@@ -944,10 +944,10 @@ class TestCreateDMMessageFromDecrypted:
         assert len(message_broadcasts) == 1
 
     @pytest.mark.asyncio
-    async def test_allows_same_text_same_second_dms_from_distinct_packets(
+    async def test_dedupes_same_text_same_second_incoming_dms_from_distinct_packets(
         self, test_db, captured_broadcasts
     ):
-        """Distinct DM packets with the same text/timestamp both store."""
+        """Distinct incoming DM observations with the same text/timestamp merge."""
         from app.decoder import DecryptedDirectMessage
         from app.packet_processor import create_dm_message_from_decrypted
 
@@ -983,16 +983,15 @@ class TestCreateDMMessageFromDecrypted:
             )
 
         assert msg_id_1 is not None
-        assert msg_id_2 is not None
-        assert msg_id_1 != msg_id_2
+        assert msg_id_2 is None
 
         messages = await MessageRepository.get_all(
             msg_type="PRIV", conversation_key=self.A1B2C3_PUB.lower(), limit=10
         )
-        assert len(messages) == 2
+        assert len(messages) == 1
 
         message_broadcasts = [b for b in broadcasts if b["type"] == "message"]
-        assert len(message_broadcasts) == 2
+        assert len(message_broadcasts) == 1
 
     @pytest.mark.asyncio
     async def test_links_raw_packet_to_dm_message(self, test_db, captured_broadcasts):

tests/test_repository.py

@@ -171,6 +171,47 @@ class TestMessageRepositoryGetByContent:
         assert result.sender_timestamp is None
         assert result.outgoing is True
 
+    @pytest.mark.asyncio
+    async def test_get_by_content_can_filter_incoming_vs_outgoing(self, test_db):
+        """Outgoing filter keeps incoming duplicate reconciliation on the right row."""
+        conversation_key = "abc123abc123abc123abc123abc12300"
+        incoming_id = await _create_message(
+            test_db,
+            msg_type="PRIV",
+            conversation_key=conversation_key,
+            text="Same text",
+            sender_timestamp=1700000000,
+            outgoing=False,
+        )
+        outgoing_id = await _create_message(
+            test_db,
+            msg_type="PRIV",
+            conversation_key=conversation_key,
+            text="Same text",
+            sender_timestamp=1700000000,
+            outgoing=True,
+        )
+
+        incoming = await MessageRepository.get_by_content(
+            msg_type="PRIV",
+            conversation_key=conversation_key,
+            text="Same text",
+            sender_timestamp=1700000000,
+            outgoing=False,
+        )
+        outgoing = await MessageRepository.get_by_content(
+            msg_type="PRIV",
+            conversation_key=conversation_key,
+            text="Same text",
+            sender_timestamp=1700000000,
+            outgoing=True,
+        )
+
+        assert incoming is not None
+        assert outgoing is not None
+        assert incoming.id == incoming_id
+        assert outgoing.id == outgoing_id
+
     @pytest.mark.asyncio
     async def test_get_by_content_distinguishes_by_timestamp(self, test_db):
         """Different sender_timestamps are distinguished correctly."""