Use packet id as message primary key (#58)

* Use packet id as message primary key

* fix query

* fix query

data/mesh.py

@@ -166,6 +166,9 @@ def store_packet_dict(p: dict):
         ch = 0
 
     # timestamps & ids
+    pkt_id = _first(p, "id", "packet_id", "packetId", default=None)
+    if pkt_id is None:
+        return  # ignore packets without an id
     rx_time = int(_first(p, "rxTime", "rx_time", default=time.time()))
     from_id = _first(p, "fromId", "from_id", "from", default=None)
     to_id = _first(p, "toId", "to_id", "to", default=None)
@@ -176,6 +179,7 @@ def store_packet_dict(p: dict):
     hop = _first(p, "hopLimit", "hop_limit", default=None)
 
     row = (
+        int(pkt_id),
         rx_time,
         _iso(rx_time),
         from_id,
@@ -189,9 +193,9 @@ def store_packet_dict(p: dict):
     )
     with DB_LOCK:
         conn.execute(
-            """INSERT INTO messages
-               (rx_time, rx_iso, from_id, to_id, channel, portnum, text, snr, rssi, hop_limit)
-               VALUES (?,?,?,?,?,?,?,?,?,?)""",
+            """INSERT OR IGNORE INTO messages
+               (id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, snr, rssi, hop_limit)
+               VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
             row,
         )
         conn.commit()

data/messages.sql

@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS messages (
-    id        INTEGER PRIMARY KEY AUTOINCREMENT,
+    id        INTEGER PRIMARY KEY,     -- meshtastic packet id
     rx_time   INTEGER NOT NULL,        -- unix seconds when received
     rx_iso    TEXT NOT NULL,           -- ISO8601 UTC timestamp
     from_id   TEXT,                    -- sender node id (string form)

web/app.rb

@@ -145,9 +145,12 @@ def require_token!
 end
 
 def insert_message(db, m)
+  msg_id = m["id"] || m["packet_id"]
+  return unless msg_id
   rx_time = m["rx_time"]&.to_i || Time.now.to_i
   rx_iso = m["rx_iso"] || Time.at(rx_time).utc.iso8601
   row = [
+    msg_id,
     rx_time,
     rx_iso,
     m["from_id"],
@@ -161,8 +164,8 @@ def insert_message(db, m)
     m["raw_json"],
   ]
   db.execute <<~SQL, row
-               INSERT INTO messages(rx_time,rx_iso,from_id,to_id,channel,portnum,text,snr,rssi,hop_limit,raw_json)
-               VALUES (?,?,?,?,?,?,?,?,?,?,?)
+               INSERT OR IGNORE INTO messages(id,rx_time,rx_iso,from_id,to_id,channel,portnum,text,snr,rssi,hop_limit,raw_json)
+               VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
              SQL
 end