Add collector-level event deduplication using content hashes

Replace presentation-layer deduplication with collector-level approach:
- Add event_hash column to messages, advertisements, trace_paths, telemetry tables
- Handlers compute content hashes and skip duplicate events at insertion time
- Use 5-minute time buckets for advertisements and telemetry
- Include Alembic migration for schema changes
This commit is contained in:
Claude
2025-12-06 12:23:14 +00:00
parent c80986fe67
commit 6e3b86a1ad
13 changed files with 207 additions and 237 deletions
@@ -0,0 +1,67 @@
"""Add event_hash column to event tables for deduplication
Revision ID: 003
Revises: 002
Create Date: 2024-12-06
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "003"
down_revision: Union[str, None] = "002"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Add event_hash column to messages table
op.add_column(
"messages",
sa.Column("event_hash", sa.String(32), nullable=True),
)
op.create_index("ix_messages_event_hash", "messages", ["event_hash"])
# Add event_hash column to advertisements table
op.add_column(
"advertisements",
sa.Column("event_hash", sa.String(32), nullable=True),
)
op.create_index("ix_advertisements_event_hash", "advertisements", ["event_hash"])
# Add event_hash column to trace_paths table
op.add_column(
"trace_paths",
sa.Column("event_hash", sa.String(32), nullable=True),
)
op.create_index("ix_trace_paths_event_hash", "trace_paths", ["event_hash"])
# Add event_hash column to telemetry table
op.add_column(
"telemetry",
sa.Column("event_hash", sa.String(32), nullable=True),
)
op.create_index("ix_telemetry_event_hash", "telemetry", ["event_hash"])
def downgrade() -> None:
# Remove event_hash from telemetry
op.drop_index("ix_telemetry_event_hash", table_name="telemetry")
op.drop_column("telemetry", "event_hash")
# Remove event_hash from trace_paths
op.drop_index("ix_trace_paths_event_hash", table_name="trace_paths")
op.drop_column("trace_paths", "event_hash")
# Remove event_hash from advertisements
op.drop_index("ix_advertisements_event_hash", table_name="advertisements")
op.drop_column("advertisements", "event_hash")
# Remove event_hash from messages
op.drop_index("ix_messages_event_hash", table_name="messages")
op.drop_column("messages", "event_hash")
+5 -59
View File
@@ -9,7 +9,6 @@ from sqlalchemy.orm import aliased, selectinload
from meshcore_hub.api.auth import RequireRead
from meshcore_hub.api.dependencies import DbSession
from meshcore_hub.common.hash_utils import compute_advertisement_hash
from meshcore_hub.common.models import Advertisement, Node
from meshcore_hub.common.schemas.messages import AdvertisementList, AdvertisementRead
@@ -36,9 +35,6 @@ async def list_advertisements(
),
since: Optional[datetime] = Query(None, description="Start timestamp"),
until: Optional[datetime] = Query(None, description="End timestamp"),
dedupe: bool = Query(
True, description="Deduplicate advertisements from multiple receivers"
),
limit: int = Query(50, ge=1, le=100, description="Page size"),
offset: int = Query(0, ge=0, description="Page offset"),
) -> AdvertisementList:
@@ -74,37 +70,12 @@ async def list_advertisements(
if until:
query = query.where(Advertisement.received_at <= until)
# When deduplicating, we need to fetch more results and compute distinct count
if dedupe:
# For deduplicated count, count distinct by public_key within time buckets
# We use a 5-minute time bucket for advertisements
distinct_subquery = (
select(
Advertisement.public_key,
Advertisement.name,
Advertisement.adv_type,
Advertisement.flags,
# Use date truncation for time bucketing (5 min = 300 seconds)
(func.strftime("%s", Advertisement.received_at) / 300).label(
"time_bucket"
),
)
.distinct()
.select_from(query.subquery())
)
count_query = select(func.count()).select_from(distinct_subquery.subquery())
total = session.execute(count_query).scalar() or 0
# Get total count
count_query = select(func.count()).select_from(query.subquery())
total = session.execute(count_query).scalar() or 0
# Fetch extra results to account for duplicates
fetch_limit = (limit + offset) * 3
query = query.order_by(Advertisement.received_at.desc()).limit(fetch_limit)
else:
# Standard count and pagination
count_query = select(func.count()).select_from(query.subquery())
total = session.execute(count_query).scalar() or 0
query = (
query.order_by(Advertisement.received_at.desc()).offset(offset).limit(limit)
)
# Apply pagination
query = query.order_by(Advertisement.received_at.desc()).offset(offset).limit(limit)
# Execute
results = session.execute(query).all()
@@ -128,25 +99,8 @@ async def list_advertisements(
# Build response with node details
items = []
seen_hashes: set[str] = set()
for row in results:
adv = row[0]
# Compute hash for deduplication
if dedupe:
adv_hash = compute_advertisement_hash(
public_key=adv.public_key,
name=adv.name,
adv_type=adv.adv_type,
flags=adv.flags,
received_at=adv.received_at,
bucket_minutes=5,
)
if adv_hash in seen_hashes:
continue
seen_hashes.add(adv_hash)
receiver_node = nodes_by_id.get(row.receiver_id) if row.receiver_id else None
source_node = nodes_by_id.get(row.source_id) if row.source_id else None
@@ -165,14 +119,6 @@ async def list_advertisements(
}
items.append(AdvertisementRead(**data))
# Stop once we have enough items (for dedupe mode with pagination)
if dedupe and len(items) >= offset + limit:
break
# Apply offset for dedupe mode (we fetched from beginning)
if dedupe:
items = items[offset : offset + limit]
return AdvertisementList(
items=items,
total=total,
+22 -119
View File
@@ -8,10 +8,6 @@ from sqlalchemy import func, select
from meshcore_hub.api.auth import RequireRead
from meshcore_hub.api.dependencies import DbSession
from meshcore_hub.common.hash_utils import (
compute_advertisement_hash,
compute_message_hash,
)
from meshcore_hub.common.models import Advertisement, Message, Node, NodeTag
from meshcore_hub.common.schemas.messages import (
ChannelMessage,
@@ -47,110 +43,45 @@ async def get_stats(
or 0
)
# Total messages (deduplicated by content hash)
distinct_messages = (
select(
Message.text,
Message.pubkey_prefix,
Message.channel_idx,
Message.sender_timestamp,
Message.txt_type,
)
.distinct()
.subquery()
)
# Total messages
total_messages = (
session.execute(select(func.count()).select_from(distinct_messages)).scalar()
or 0
session.execute(select(func.count()).select_from(Message)).scalar() or 0
)
# Messages today (deduplicated)
distinct_messages_today = (
select(
Message.text,
Message.pubkey_prefix,
Message.channel_idx,
Message.sender_timestamp,
Message.txt_type,
)
.where(Message.received_at >= today_start)
.distinct()
.subquery()
)
# Messages today
messages_today = (
session.execute(
select(func.count()).select_from(distinct_messages_today)
select(func.count())
.select_from(Message)
.where(Message.received_at >= today_start)
).scalar()
or 0
)
# Total advertisements (deduplicated by public_key + 5min time bucket)
distinct_advertisements = (
select(
Advertisement.public_key,
Advertisement.name,
Advertisement.adv_type,
Advertisement.flags,
(func.strftime("%s", Advertisement.received_at) / 300).label("time_bucket"),
)
.distinct()
.subquery()
)
# Total advertisements
total_advertisements = (
session.execute(
select(func.count()).select_from(distinct_advertisements)
).scalar()
or 0
session.execute(select(func.count()).select_from(Advertisement)).scalar() or 0
)
# Advertisements in last 24h (deduplicated)
distinct_advertisements_24h = (
select(
Advertisement.public_key,
Advertisement.name,
Advertisement.adv_type,
Advertisement.flags,
(func.strftime("%s", Advertisement.received_at) / 300).label("time_bucket"),
)
.where(Advertisement.received_at >= yesterday)
.distinct()
.subquery()
)
# Advertisements in last 24h
advertisements_24h = (
session.execute(
select(func.count()).select_from(distinct_advertisements_24h)
select(func.count())
.select_from(Advertisement)
.where(Advertisement.received_at >= yesterday)
).scalar()
or 0
)
# Recent advertisements (last 10, deduplicated)
# Fetch more to ensure we have 10 unique after deduplication
recent_ads_raw = (
# Recent advertisements (last 10)
recent_ads = (
session.execute(
select(Advertisement).order_by(Advertisement.received_at.desc()).limit(30)
select(Advertisement).order_by(Advertisement.received_at.desc()).limit(10)
)
.scalars()
.all()
)
# Deduplicate by hash
seen_ad_hashes: set[str] = set()
recent_ads = []
for ad in recent_ads_raw:
ad_hash = compute_advertisement_hash(
public_key=ad.public_key,
name=ad.name,
adv_type=ad.adv_type,
flags=ad.flags,
received_at=ad.received_at,
bucket_minutes=5,
)
if ad_hash not in seen_ad_hashes:
seen_ad_hashes.add(ad_hash)
recent_ads.append(ad)
if len(recent_ads) >= 10:
break
# Get node names, adv_types, and friendly_name tags for the advertised nodes
ad_public_keys = [ad.public_key for ad in recent_ads]
node_names: dict[str, str] = {}
@@ -188,57 +119,29 @@ async def get_stats(
for ad in recent_ads
]
# Channel message counts (deduplicated)
distinct_channel_messages = (
select(
Message.channel_idx,
Message.text,
Message.pubkey_prefix,
Message.sender_timestamp,
Message.txt_type,
)
# Channel message counts
channel_counts_query = (
select(Message.channel_idx, func.count())
.where(Message.message_type == "channel")
.where(Message.channel_idx.isnot(None))
.distinct()
.subquery()
.group_by(Message.channel_idx)
)
channel_counts_query = select(
distinct_channel_messages.c.channel_idx, func.count()
).group_by(distinct_channel_messages.c.channel_idx)
channel_results = session.execute(channel_counts_query).all()
channel_message_counts = {
int(channel): int(count) for channel, count in channel_results
}
# Get latest 5 messages for each channel that has messages (deduplicated)
# Get latest 5 messages for each channel that has messages
channel_messages: dict[int, list[ChannelMessage]] = {}
for channel_idx, _ in channel_results:
# Fetch more messages to deduplicate
messages_query = (
select(Message)
.where(Message.message_type == "channel")
.where(Message.channel_idx == channel_idx)
.order_by(Message.received_at.desc())
.limit(15)
.limit(5)
)
channel_msgs_raw = session.execute(messages_query).scalars().all()
# Deduplicate
seen_msg_hashes: set[str] = set()
channel_msgs = []
for m in channel_msgs_raw:
msg_hash = compute_message_hash(
text=m.text,
pubkey_prefix=m.pubkey_prefix,
channel_idx=m.channel_idx,
sender_timestamp=m.sender_timestamp,
txt_type=m.txt_type,
)
if msg_hash not in seen_msg_hashes:
seen_msg_hashes.add(msg_hash)
channel_msgs.append(m)
if len(channel_msgs) >= 5:
break
channel_msgs = session.execute(messages_query).scalars().all()
# Look up sender names for these messages
msg_prefixes = [m.pubkey_prefix for m in channel_msgs if m.pubkey_prefix]
+5 -54
View File
@@ -9,7 +9,6 @@ from sqlalchemy.orm import aliased, selectinload
from meshcore_hub.api.auth import RequireRead
from meshcore_hub.api.dependencies import DbSession
from meshcore_hub.common.hash_utils import compute_message_hash
from meshcore_hub.common.models import Message, Node, NodeTag
from meshcore_hub.common.schemas.messages import MessageList, MessageRead
@@ -39,9 +38,6 @@ async def list_messages(
since: Optional[datetime] = Query(None, description="Start timestamp"),
until: Optional[datetime] = Query(None, description="End timestamp"),
search: Optional[str] = Query(None, description="Search in message text"),
dedupe: bool = Query(
True, description="Deduplicate messages from multiple receivers"
),
limit: int = Query(50, ge=1, le=100, description="Page size"),
offset: int = Query(0, ge=0, description="Page offset"),
) -> MessageList:
@@ -78,33 +74,12 @@ async def list_messages(
if search:
query = query.where(Message.text.ilike(f"%{search}%"))
# When deduplicating, we need to fetch more results to ensure we have enough
# after removing duplicates, and compute distinct count differently
if dedupe:
# For deduplicated count, we need to count distinct content
# Use a subquery that groups by content-identifying fields
distinct_subquery = (
select(
Message.text,
Message.pubkey_prefix,
Message.channel_idx,
Message.sender_timestamp,
Message.txt_type,
)
.distinct()
.select_from(query.subquery())
)
count_query = select(func.count()).select_from(distinct_subquery.subquery())
total = session.execute(count_query).scalar() or 0
# Get total count
count_query = select(func.count()).select_from(query.subquery())
total = session.execute(count_query).scalar() or 0
# Fetch extra results to account for duplicates (3x limit + offset)
fetch_limit = (limit + offset) * 3
query = query.order_by(Message.received_at.desc()).limit(fetch_limit)
else:
# Standard count and pagination
count_query = select(func.count()).select_from(query.subquery())
total = session.execute(count_query).scalar() or 0
query = query.order_by(Message.received_at.desc()).offset(offset).limit(limit)
# Apply pagination
query = query.order_by(Message.received_at.desc()).offset(offset).limit(limit)
# Execute
results = session.execute(query).all()
@@ -153,24 +128,8 @@ async def list_messages(
# Build response with sender info and received_by
items = []
seen_hashes: set[str] = set()
for row in results:
m = row[0]
# Compute hash for deduplication
if dedupe:
msg_hash = compute_message_hash(
text=m.text,
pubkey_prefix=m.pubkey_prefix,
channel_idx=m.channel_idx,
sender_timestamp=m.sender_timestamp,
txt_type=m.txt_type,
)
if msg_hash in seen_hashes:
continue
seen_hashes.add(msg_hash)
receiver_pk = row.receiver_pk
receiver_name = row.receiver_name
receiver_node = (
@@ -203,14 +162,6 @@ async def list_messages(
}
items.append(MessageRead(**msg_dict))
# Stop once we have enough items (for dedupe mode with pagination)
if dedupe and len(items) >= offset + limit:
break
# Apply offset for dedupe mode (we fetched from beginning)
if dedupe:
items = items[offset : offset + limit]
return MessageList(
items=items,
total=total,
@@ -7,6 +7,7 @@ from typing import Any
from sqlalchemy import select
from meshcore_hub.common.database import DatabaseManager
from meshcore_hub.common.hash_utils import compute_advertisement_hash
from meshcore_hub.common.models import Advertisement, Node
logger = logging.getLogger(__name__)
@@ -40,7 +41,31 @@ def handle_advertisement(
flags = payload.get("flags")
now = datetime.now(timezone.utc)
# Compute event hash for deduplication (5-minute time bucket)
event_hash = compute_advertisement_hash(
public_key=adv_public_key,
name=name,
adv_type=adv_type,
flags=flags,
received_at=now,
bucket_minutes=5,
)
with db.session_scope() as session:
# Check if advertisement with same hash already exists
existing = session.execute(
select(Advertisement.id).where(Advertisement.event_hash == event_hash)
).scalar_one_or_none()
if existing:
logger.debug(f"Duplicate advertisement skipped (hash={event_hash[:8]}...)")
# Still update node last_seen even for duplicate advertisements
node_query = select(Node).where(Node.public_key == adv_public_key)
node = session.execute(node_query).scalar_one_or_none()
if node:
node.last_seen = now
return
# Find or create receiver node
receiver_node = None
if public_key:
@@ -91,6 +116,7 @@ def handle_advertisement(
adv_type=adv_type,
flags=flags,
received_at=now,
event_hash=event_hash,
)
session.add(advertisement)
@@ -7,6 +7,7 @@ from typing import Any
from sqlalchemy import select
from meshcore_hub.common.database import DatabaseManager
from meshcore_hub.common.hash_utils import compute_message_hash
from meshcore_hub.common.models import Message, Node
logger = logging.getLogger(__name__)
@@ -84,7 +85,25 @@ def _handle_message(
except (ValueError, OSError):
pass
# Compute event hash for deduplication
event_hash = compute_message_hash(
text=text,
pubkey_prefix=pubkey_prefix,
channel_idx=channel_idx,
sender_timestamp=sender_timestamp,
txt_type=txt_type,
)
with db.session_scope() as session:
# Check if message with same hash already exists
existing = session.execute(
select(Message.id).where(Message.event_hash == event_hash)
).scalar_one_or_none()
if existing:
logger.debug(f"Duplicate message skipped (hash={event_hash[:8]}...)")
return
# Find receiver node
receiver_node = None
if public_key:
@@ -115,6 +134,7 @@ def _handle_message(
snr=snr,
sender_timestamp=sender_timestamp,
received_at=now,
event_hash=event_hash,
)
session.add(message)
@@ -7,6 +7,7 @@ from typing import Any
from sqlalchemy import select
from meshcore_hub.common.database import DatabaseManager
from meshcore_hub.common.hash_utils import compute_telemetry_hash
from meshcore_hub.common.models import Node, Telemetry
logger = logging.getLogger(__name__)
@@ -49,7 +50,26 @@ def handle_telemetry(
except ValueError:
lpp_bytes = lpp_data.encode()
# Compute event hash for deduplication (5-minute time bucket)
event_hash = compute_telemetry_hash(
node_public_key=node_public_key,
parsed_data=parsed_data,
received_at=now,
bucket_minutes=5,
)
with db.session_scope() as session:
# Check if telemetry with same hash already exists
existing = session.execute(
select(Telemetry.id).where(Telemetry.event_hash == event_hash)
).scalar_one_or_none()
if existing:
logger.debug(
f"Duplicate telemetry skipped (node={node_public_key[:12]}...)"
)
return
# Find receiver node
receiver_node = None
if public_key:
@@ -92,6 +112,7 @@ def handle_telemetry(
lpp_data=lpp_bytes,
parsed_data=parsed_data,
received_at=now,
event_hash=event_hash,
)
session.add(telemetry)
@@ -7,6 +7,7 @@ from typing import Any
from sqlalchemy import select
from meshcore_hub.common.database import DatabaseManager
from meshcore_hub.common.hash_utils import compute_trace_hash
from meshcore_hub.common.models import Node, TracePath
logger = logging.getLogger(__name__)
@@ -40,7 +41,19 @@ def handle_trace_data(
snr_values = payload.get("snr_values")
hop_count = payload.get("hop_count")
# Compute event hash for deduplication (initiator_tag is unique per trace)
event_hash = compute_trace_hash(initiator_tag=initiator_tag)
with db.session_scope() as session:
# Check if trace with same hash already exists
existing = session.execute(
select(TracePath.id).where(TracePath.event_hash == event_hash)
).scalar_one_or_none()
if existing:
logger.debug(f"Duplicate trace skipped (tag={initiator_tag})")
return
# Find receiver node
receiver_node = None
if public_key:
@@ -69,6 +82,7 @@ def handle_trace_data(
snr_values=snr_values,
hop_count=hop_count,
received_at=now,
event_hash=event_hash,
)
session.add(trace_path)
@@ -58,8 +58,15 @@ class Advertisement(Base, UUIDMixin, TimestampMixin):
default=utc_now,
nullable=False,
)
event_hash: Mapped[Optional[str]] = mapped_column(
String(32),
nullable=True,
)
__table_args__ = (Index("ix_advertisements_received_at", "received_at"),)
__table_args__ = (
Index("ix_advertisements_received_at", "received_at"),
Index("ix_advertisements_event_hash", "event_hash"),
)
def __repr__(self) -> str:
return f"<Advertisement(id={self.id}, public_key={self.public_key[:12]}..., name={self.name})>"
@@ -76,12 +76,17 @@ class Message(Base, UUIDMixin, TimestampMixin):
default=utc_now,
nullable=False,
)
event_hash: Mapped[Optional[str]] = mapped_column(
String(32),
nullable=True,
)
__table_args__ = (
Index("ix_messages_message_type", "message_type"),
Index("ix_messages_pubkey_prefix", "pubkey_prefix"),
Index("ix_messages_channel_idx", "channel_idx"),
Index("ix_messages_received_at", "received_at"),
Index("ix_messages_event_hash", "event_hash"),
)
def __repr__(self) -> str:
+8 -1
View File
@@ -54,8 +54,15 @@ class Telemetry(Base, UUIDMixin, TimestampMixin):
default=utc_now,
nullable=False,
)
event_hash: Mapped[Optional[str]] = mapped_column(
String(32),
nullable=True,
)
__table_args__ = (Index("ix_telemetry_received_at", "received_at"),)
__table_args__ = (
Index("ix_telemetry_received_at", "received_at"),
Index("ix_telemetry_event_hash", "event_hash"),
)
def __repr__(self) -> str:
return (
+6 -1
View File
@@ -3,7 +3,7 @@
from datetime import datetime
from typing import Optional
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, String
from sqlalchemy.dialects.sqlite import JSON
from sqlalchemy.orm import Mapped, mapped_column
@@ -67,10 +67,15 @@ class TracePath(Base, UUIDMixin, TimestampMixin):
default=utc_now,
nullable=False,
)
event_hash: Mapped[Optional[str]] = mapped_column(
String(32),
nullable=True,
)
__table_args__ = (
Index("ix_trace_paths_initiator_tag", "initiator_tag"),
Index("ix_trace_paths_received_at", "received_at"),
Index("ix_trace_paths_event_hash", "event_hash"),
)
def __repr__(self) -> str:
-2
View File
@@ -2,8 +2,6 @@
from datetime import datetime, timezone
import pytest
from meshcore_hub.common.hash_utils import (
compute_advertisement_hash,
compute_message_hash,