mirror of
https://github.com/ipnet-mesh/meshcore-hub.git
synced 2026-08-12 11:52:44 +02:00
Add collector-level event deduplication using content hashes
Replace presentation-layer deduplication with collector-level approach: - Add event_hash column to messages, advertisements, trace_paths, telemetry tables - Handlers compute content hashes and skip duplicate events at insertion time - Use 5-minute time buckets for advertisements and telemetry - Include Alembic migration for schema changes
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
"""Add event_hash column to event tables for deduplication
|
||||
|
||||
Revision ID: 003
|
||||
Revises: 002
|
||||
Create Date: 2024-12-06
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "003"
|
||||
down_revision: Union[str, None] = "002"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Add event_hash column to messages table
|
||||
op.add_column(
|
||||
"messages",
|
||||
sa.Column("event_hash", sa.String(32), nullable=True),
|
||||
)
|
||||
op.create_index("ix_messages_event_hash", "messages", ["event_hash"])
|
||||
|
||||
# Add event_hash column to advertisements table
|
||||
op.add_column(
|
||||
"advertisements",
|
||||
sa.Column("event_hash", sa.String(32), nullable=True),
|
||||
)
|
||||
op.create_index("ix_advertisements_event_hash", "advertisements", ["event_hash"])
|
||||
|
||||
# Add event_hash column to trace_paths table
|
||||
op.add_column(
|
||||
"trace_paths",
|
||||
sa.Column("event_hash", sa.String(32), nullable=True),
|
||||
)
|
||||
op.create_index("ix_trace_paths_event_hash", "trace_paths", ["event_hash"])
|
||||
|
||||
# Add event_hash column to telemetry table
|
||||
op.add_column(
|
||||
"telemetry",
|
||||
sa.Column("event_hash", sa.String(32), nullable=True),
|
||||
)
|
||||
op.create_index("ix_telemetry_event_hash", "telemetry", ["event_hash"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Remove event_hash from telemetry
|
||||
op.drop_index("ix_telemetry_event_hash", table_name="telemetry")
|
||||
op.drop_column("telemetry", "event_hash")
|
||||
|
||||
# Remove event_hash from trace_paths
|
||||
op.drop_index("ix_trace_paths_event_hash", table_name="trace_paths")
|
||||
op.drop_column("trace_paths", "event_hash")
|
||||
|
||||
# Remove event_hash from advertisements
|
||||
op.drop_index("ix_advertisements_event_hash", table_name="advertisements")
|
||||
op.drop_column("advertisements", "event_hash")
|
||||
|
||||
# Remove event_hash from messages
|
||||
op.drop_index("ix_messages_event_hash", table_name="messages")
|
||||
op.drop_column("messages", "event_hash")
|
||||
@@ -9,7 +9,6 @@ from sqlalchemy.orm import aliased, selectinload
|
||||
|
||||
from meshcore_hub.api.auth import RequireRead
|
||||
from meshcore_hub.api.dependencies import DbSession
|
||||
from meshcore_hub.common.hash_utils import compute_advertisement_hash
|
||||
from meshcore_hub.common.models import Advertisement, Node
|
||||
from meshcore_hub.common.schemas.messages import AdvertisementList, AdvertisementRead
|
||||
|
||||
@@ -36,9 +35,6 @@ async def list_advertisements(
|
||||
),
|
||||
since: Optional[datetime] = Query(None, description="Start timestamp"),
|
||||
until: Optional[datetime] = Query(None, description="End timestamp"),
|
||||
dedupe: bool = Query(
|
||||
True, description="Deduplicate advertisements from multiple receivers"
|
||||
),
|
||||
limit: int = Query(50, ge=1, le=100, description="Page size"),
|
||||
offset: int = Query(0, ge=0, description="Page offset"),
|
||||
) -> AdvertisementList:
|
||||
@@ -74,37 +70,12 @@ async def list_advertisements(
|
||||
if until:
|
||||
query = query.where(Advertisement.received_at <= until)
|
||||
|
||||
# When deduplicating, we need to fetch more results and compute distinct count
|
||||
if dedupe:
|
||||
# For deduplicated count, count distinct by public_key within time buckets
|
||||
# We use a 5-minute time bucket for advertisements
|
||||
distinct_subquery = (
|
||||
select(
|
||||
Advertisement.public_key,
|
||||
Advertisement.name,
|
||||
Advertisement.adv_type,
|
||||
Advertisement.flags,
|
||||
# Use date truncation for time bucketing (5 min = 300 seconds)
|
||||
(func.strftime("%s", Advertisement.received_at) / 300).label(
|
||||
"time_bucket"
|
||||
),
|
||||
)
|
||||
.distinct()
|
||||
.select_from(query.subquery())
|
||||
)
|
||||
count_query = select(func.count()).select_from(distinct_subquery.subquery())
|
||||
total = session.execute(count_query).scalar() or 0
|
||||
# Get total count
|
||||
count_query = select(func.count()).select_from(query.subquery())
|
||||
total = session.execute(count_query).scalar() or 0
|
||||
|
||||
# Fetch extra results to account for duplicates
|
||||
fetch_limit = (limit + offset) * 3
|
||||
query = query.order_by(Advertisement.received_at.desc()).limit(fetch_limit)
|
||||
else:
|
||||
# Standard count and pagination
|
||||
count_query = select(func.count()).select_from(query.subquery())
|
||||
total = session.execute(count_query).scalar() or 0
|
||||
query = (
|
||||
query.order_by(Advertisement.received_at.desc()).offset(offset).limit(limit)
|
||||
)
|
||||
# Apply pagination
|
||||
query = query.order_by(Advertisement.received_at.desc()).offset(offset).limit(limit)
|
||||
|
||||
# Execute
|
||||
results = session.execute(query).all()
|
||||
@@ -128,25 +99,8 @@ async def list_advertisements(
|
||||
|
||||
# Build response with node details
|
||||
items = []
|
||||
seen_hashes: set[str] = set()
|
||||
|
||||
for row in results:
|
||||
adv = row[0]
|
||||
|
||||
# Compute hash for deduplication
|
||||
if dedupe:
|
||||
adv_hash = compute_advertisement_hash(
|
||||
public_key=adv.public_key,
|
||||
name=adv.name,
|
||||
adv_type=adv.adv_type,
|
||||
flags=adv.flags,
|
||||
received_at=adv.received_at,
|
||||
bucket_minutes=5,
|
||||
)
|
||||
if adv_hash in seen_hashes:
|
||||
continue
|
||||
seen_hashes.add(adv_hash)
|
||||
|
||||
receiver_node = nodes_by_id.get(row.receiver_id) if row.receiver_id else None
|
||||
source_node = nodes_by_id.get(row.source_id) if row.source_id else None
|
||||
|
||||
@@ -165,14 +119,6 @@ async def list_advertisements(
|
||||
}
|
||||
items.append(AdvertisementRead(**data))
|
||||
|
||||
# Stop once we have enough items (for dedupe mode with pagination)
|
||||
if dedupe and len(items) >= offset + limit:
|
||||
break
|
||||
|
||||
# Apply offset for dedupe mode (we fetched from beginning)
|
||||
if dedupe:
|
||||
items = items[offset : offset + limit]
|
||||
|
||||
return AdvertisementList(
|
||||
items=items,
|
||||
total=total,
|
||||
|
||||
@@ -8,10 +8,6 @@ from sqlalchemy import func, select
|
||||
|
||||
from meshcore_hub.api.auth import RequireRead
|
||||
from meshcore_hub.api.dependencies import DbSession
|
||||
from meshcore_hub.common.hash_utils import (
|
||||
compute_advertisement_hash,
|
||||
compute_message_hash,
|
||||
)
|
||||
from meshcore_hub.common.models import Advertisement, Message, Node, NodeTag
|
||||
from meshcore_hub.common.schemas.messages import (
|
||||
ChannelMessage,
|
||||
@@ -47,110 +43,45 @@ async def get_stats(
|
||||
or 0
|
||||
)
|
||||
|
||||
# Total messages (deduplicated by content hash)
|
||||
distinct_messages = (
|
||||
select(
|
||||
Message.text,
|
||||
Message.pubkey_prefix,
|
||||
Message.channel_idx,
|
||||
Message.sender_timestamp,
|
||||
Message.txt_type,
|
||||
)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
# Total messages
|
||||
total_messages = (
|
||||
session.execute(select(func.count()).select_from(distinct_messages)).scalar()
|
||||
or 0
|
||||
session.execute(select(func.count()).select_from(Message)).scalar() or 0
|
||||
)
|
||||
|
||||
# Messages today (deduplicated)
|
||||
distinct_messages_today = (
|
||||
select(
|
||||
Message.text,
|
||||
Message.pubkey_prefix,
|
||||
Message.channel_idx,
|
||||
Message.sender_timestamp,
|
||||
Message.txt_type,
|
||||
)
|
||||
.where(Message.received_at >= today_start)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
# Messages today
|
||||
messages_today = (
|
||||
session.execute(
|
||||
select(func.count()).select_from(distinct_messages_today)
|
||||
select(func.count())
|
||||
.select_from(Message)
|
||||
.where(Message.received_at >= today_start)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Total advertisements (deduplicated by public_key + 5min time bucket)
|
||||
distinct_advertisements = (
|
||||
select(
|
||||
Advertisement.public_key,
|
||||
Advertisement.name,
|
||||
Advertisement.adv_type,
|
||||
Advertisement.flags,
|
||||
(func.strftime("%s", Advertisement.received_at) / 300).label("time_bucket"),
|
||||
)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
# Total advertisements
|
||||
total_advertisements = (
|
||||
session.execute(
|
||||
select(func.count()).select_from(distinct_advertisements)
|
||||
).scalar()
|
||||
or 0
|
||||
session.execute(select(func.count()).select_from(Advertisement)).scalar() or 0
|
||||
)
|
||||
|
||||
# Advertisements in last 24h (deduplicated)
|
||||
distinct_advertisements_24h = (
|
||||
select(
|
||||
Advertisement.public_key,
|
||||
Advertisement.name,
|
||||
Advertisement.adv_type,
|
||||
Advertisement.flags,
|
||||
(func.strftime("%s", Advertisement.received_at) / 300).label("time_bucket"),
|
||||
)
|
||||
.where(Advertisement.received_at >= yesterday)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
# Advertisements in last 24h
|
||||
advertisements_24h = (
|
||||
session.execute(
|
||||
select(func.count()).select_from(distinct_advertisements_24h)
|
||||
select(func.count())
|
||||
.select_from(Advertisement)
|
||||
.where(Advertisement.received_at >= yesterday)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Recent advertisements (last 10, deduplicated)
|
||||
# Fetch more to ensure we have 10 unique after deduplication
|
||||
recent_ads_raw = (
|
||||
# Recent advertisements (last 10)
|
||||
recent_ads = (
|
||||
session.execute(
|
||||
select(Advertisement).order_by(Advertisement.received_at.desc()).limit(30)
|
||||
select(Advertisement).order_by(Advertisement.received_at.desc()).limit(10)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
# Deduplicate by hash
|
||||
seen_ad_hashes: set[str] = set()
|
||||
recent_ads = []
|
||||
for ad in recent_ads_raw:
|
||||
ad_hash = compute_advertisement_hash(
|
||||
public_key=ad.public_key,
|
||||
name=ad.name,
|
||||
adv_type=ad.adv_type,
|
||||
flags=ad.flags,
|
||||
received_at=ad.received_at,
|
||||
bucket_minutes=5,
|
||||
)
|
||||
if ad_hash not in seen_ad_hashes:
|
||||
seen_ad_hashes.add(ad_hash)
|
||||
recent_ads.append(ad)
|
||||
if len(recent_ads) >= 10:
|
||||
break
|
||||
|
||||
# Get node names, adv_types, and friendly_name tags for the advertised nodes
|
||||
ad_public_keys = [ad.public_key for ad in recent_ads]
|
||||
node_names: dict[str, str] = {}
|
||||
@@ -188,57 +119,29 @@ async def get_stats(
|
||||
for ad in recent_ads
|
||||
]
|
||||
|
||||
# Channel message counts (deduplicated)
|
||||
distinct_channel_messages = (
|
||||
select(
|
||||
Message.channel_idx,
|
||||
Message.text,
|
||||
Message.pubkey_prefix,
|
||||
Message.sender_timestamp,
|
||||
Message.txt_type,
|
||||
)
|
||||
# Channel message counts
|
||||
channel_counts_query = (
|
||||
select(Message.channel_idx, func.count())
|
||||
.where(Message.message_type == "channel")
|
||||
.where(Message.channel_idx.isnot(None))
|
||||
.distinct()
|
||||
.subquery()
|
||||
.group_by(Message.channel_idx)
|
||||
)
|
||||
channel_counts_query = select(
|
||||
distinct_channel_messages.c.channel_idx, func.count()
|
||||
).group_by(distinct_channel_messages.c.channel_idx)
|
||||
channel_results = session.execute(channel_counts_query).all()
|
||||
channel_message_counts = {
|
||||
int(channel): int(count) for channel, count in channel_results
|
||||
}
|
||||
|
||||
# Get latest 5 messages for each channel that has messages (deduplicated)
|
||||
# Get latest 5 messages for each channel that has messages
|
||||
channel_messages: dict[int, list[ChannelMessage]] = {}
|
||||
for channel_idx, _ in channel_results:
|
||||
# Fetch more messages to deduplicate
|
||||
messages_query = (
|
||||
select(Message)
|
||||
.where(Message.message_type == "channel")
|
||||
.where(Message.channel_idx == channel_idx)
|
||||
.order_by(Message.received_at.desc())
|
||||
.limit(15)
|
||||
.limit(5)
|
||||
)
|
||||
channel_msgs_raw = session.execute(messages_query).scalars().all()
|
||||
|
||||
# Deduplicate
|
||||
seen_msg_hashes: set[str] = set()
|
||||
channel_msgs = []
|
||||
for m in channel_msgs_raw:
|
||||
msg_hash = compute_message_hash(
|
||||
text=m.text,
|
||||
pubkey_prefix=m.pubkey_prefix,
|
||||
channel_idx=m.channel_idx,
|
||||
sender_timestamp=m.sender_timestamp,
|
||||
txt_type=m.txt_type,
|
||||
)
|
||||
if msg_hash not in seen_msg_hashes:
|
||||
seen_msg_hashes.add(msg_hash)
|
||||
channel_msgs.append(m)
|
||||
if len(channel_msgs) >= 5:
|
||||
break
|
||||
channel_msgs = session.execute(messages_query).scalars().all()
|
||||
|
||||
# Look up sender names for these messages
|
||||
msg_prefixes = [m.pubkey_prefix for m in channel_msgs if m.pubkey_prefix]
|
||||
|
||||
@@ -9,7 +9,6 @@ from sqlalchemy.orm import aliased, selectinload
|
||||
|
||||
from meshcore_hub.api.auth import RequireRead
|
||||
from meshcore_hub.api.dependencies import DbSession
|
||||
from meshcore_hub.common.hash_utils import compute_message_hash
|
||||
from meshcore_hub.common.models import Message, Node, NodeTag
|
||||
from meshcore_hub.common.schemas.messages import MessageList, MessageRead
|
||||
|
||||
@@ -39,9 +38,6 @@ async def list_messages(
|
||||
since: Optional[datetime] = Query(None, description="Start timestamp"),
|
||||
until: Optional[datetime] = Query(None, description="End timestamp"),
|
||||
search: Optional[str] = Query(None, description="Search in message text"),
|
||||
dedupe: bool = Query(
|
||||
True, description="Deduplicate messages from multiple receivers"
|
||||
),
|
||||
limit: int = Query(50, ge=1, le=100, description="Page size"),
|
||||
offset: int = Query(0, ge=0, description="Page offset"),
|
||||
) -> MessageList:
|
||||
@@ -78,33 +74,12 @@ async def list_messages(
|
||||
if search:
|
||||
query = query.where(Message.text.ilike(f"%{search}%"))
|
||||
|
||||
# When deduplicating, we need to fetch more results to ensure we have enough
|
||||
# after removing duplicates, and compute distinct count differently
|
||||
if dedupe:
|
||||
# For deduplicated count, we need to count distinct content
|
||||
# Use a subquery that groups by content-identifying fields
|
||||
distinct_subquery = (
|
||||
select(
|
||||
Message.text,
|
||||
Message.pubkey_prefix,
|
||||
Message.channel_idx,
|
||||
Message.sender_timestamp,
|
||||
Message.txt_type,
|
||||
)
|
||||
.distinct()
|
||||
.select_from(query.subquery())
|
||||
)
|
||||
count_query = select(func.count()).select_from(distinct_subquery.subquery())
|
||||
total = session.execute(count_query).scalar() or 0
|
||||
# Get total count
|
||||
count_query = select(func.count()).select_from(query.subquery())
|
||||
total = session.execute(count_query).scalar() or 0
|
||||
|
||||
# Fetch extra results to account for duplicates (3x limit + offset)
|
||||
fetch_limit = (limit + offset) * 3
|
||||
query = query.order_by(Message.received_at.desc()).limit(fetch_limit)
|
||||
else:
|
||||
# Standard count and pagination
|
||||
count_query = select(func.count()).select_from(query.subquery())
|
||||
total = session.execute(count_query).scalar() or 0
|
||||
query = query.order_by(Message.received_at.desc()).offset(offset).limit(limit)
|
||||
# Apply pagination
|
||||
query = query.order_by(Message.received_at.desc()).offset(offset).limit(limit)
|
||||
|
||||
# Execute
|
||||
results = session.execute(query).all()
|
||||
@@ -153,24 +128,8 @@ async def list_messages(
|
||||
|
||||
# Build response with sender info and received_by
|
||||
items = []
|
||||
seen_hashes: set[str] = set()
|
||||
|
||||
for row in results:
|
||||
m = row[0]
|
||||
|
||||
# Compute hash for deduplication
|
||||
if dedupe:
|
||||
msg_hash = compute_message_hash(
|
||||
text=m.text,
|
||||
pubkey_prefix=m.pubkey_prefix,
|
||||
channel_idx=m.channel_idx,
|
||||
sender_timestamp=m.sender_timestamp,
|
||||
txt_type=m.txt_type,
|
||||
)
|
||||
if msg_hash in seen_hashes:
|
||||
continue
|
||||
seen_hashes.add(msg_hash)
|
||||
|
||||
receiver_pk = row.receiver_pk
|
||||
receiver_name = row.receiver_name
|
||||
receiver_node = (
|
||||
@@ -203,14 +162,6 @@ async def list_messages(
|
||||
}
|
||||
items.append(MessageRead(**msg_dict))
|
||||
|
||||
# Stop once we have enough items (for dedupe mode with pagination)
|
||||
if dedupe and len(items) >= offset + limit:
|
||||
break
|
||||
|
||||
# Apply offset for dedupe mode (we fetched from beginning)
|
||||
if dedupe:
|
||||
items = items[offset : offset + limit]
|
||||
|
||||
return MessageList(
|
||||
items=items,
|
||||
total=total,
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
from sqlalchemy import select
|
||||
|
||||
from meshcore_hub.common.database import DatabaseManager
|
||||
from meshcore_hub.common.hash_utils import compute_advertisement_hash
|
||||
from meshcore_hub.common.models import Advertisement, Node
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -40,7 +41,31 @@ def handle_advertisement(
|
||||
flags = payload.get("flags")
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# Compute event hash for deduplication (5-minute time bucket)
|
||||
event_hash = compute_advertisement_hash(
|
||||
public_key=adv_public_key,
|
||||
name=name,
|
||||
adv_type=adv_type,
|
||||
flags=flags,
|
||||
received_at=now,
|
||||
bucket_minutes=5,
|
||||
)
|
||||
|
||||
with db.session_scope() as session:
|
||||
# Check if advertisement with same hash already exists
|
||||
existing = session.execute(
|
||||
select(Advertisement.id).where(Advertisement.event_hash == event_hash)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existing:
|
||||
logger.debug(f"Duplicate advertisement skipped (hash={event_hash[:8]}...)")
|
||||
# Still update node last_seen even for duplicate advertisements
|
||||
node_query = select(Node).where(Node.public_key == adv_public_key)
|
||||
node = session.execute(node_query).scalar_one_or_none()
|
||||
if node:
|
||||
node.last_seen = now
|
||||
return
|
||||
|
||||
# Find or create receiver node
|
||||
receiver_node = None
|
||||
if public_key:
|
||||
@@ -91,6 +116,7 @@ def handle_advertisement(
|
||||
adv_type=adv_type,
|
||||
flags=flags,
|
||||
received_at=now,
|
||||
event_hash=event_hash,
|
||||
)
|
||||
session.add(advertisement)
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
from sqlalchemy import select
|
||||
|
||||
from meshcore_hub.common.database import DatabaseManager
|
||||
from meshcore_hub.common.hash_utils import compute_message_hash
|
||||
from meshcore_hub.common.models import Message, Node
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -84,7 +85,25 @@ def _handle_message(
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
# Compute event hash for deduplication
|
||||
event_hash = compute_message_hash(
|
||||
text=text,
|
||||
pubkey_prefix=pubkey_prefix,
|
||||
channel_idx=channel_idx,
|
||||
sender_timestamp=sender_timestamp,
|
||||
txt_type=txt_type,
|
||||
)
|
||||
|
||||
with db.session_scope() as session:
|
||||
# Check if message with same hash already exists
|
||||
existing = session.execute(
|
||||
select(Message.id).where(Message.event_hash == event_hash)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existing:
|
||||
logger.debug(f"Duplicate message skipped (hash={event_hash[:8]}...)")
|
||||
return
|
||||
|
||||
# Find receiver node
|
||||
receiver_node = None
|
||||
if public_key:
|
||||
@@ -115,6 +134,7 @@ def _handle_message(
|
||||
snr=snr,
|
||||
sender_timestamp=sender_timestamp,
|
||||
received_at=now,
|
||||
event_hash=event_hash,
|
||||
)
|
||||
session.add(message)
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
from sqlalchemy import select
|
||||
|
||||
from meshcore_hub.common.database import DatabaseManager
|
||||
from meshcore_hub.common.hash_utils import compute_telemetry_hash
|
||||
from meshcore_hub.common.models import Node, Telemetry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -49,7 +50,26 @@ def handle_telemetry(
|
||||
except ValueError:
|
||||
lpp_bytes = lpp_data.encode()
|
||||
|
||||
# Compute event hash for deduplication (5-minute time bucket)
|
||||
event_hash = compute_telemetry_hash(
|
||||
node_public_key=node_public_key,
|
||||
parsed_data=parsed_data,
|
||||
received_at=now,
|
||||
bucket_minutes=5,
|
||||
)
|
||||
|
||||
with db.session_scope() as session:
|
||||
# Check if telemetry with same hash already exists
|
||||
existing = session.execute(
|
||||
select(Telemetry.id).where(Telemetry.event_hash == event_hash)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existing:
|
||||
logger.debug(
|
||||
f"Duplicate telemetry skipped (node={node_public_key[:12]}...)"
|
||||
)
|
||||
return
|
||||
|
||||
# Find receiver node
|
||||
receiver_node = None
|
||||
if public_key:
|
||||
@@ -92,6 +112,7 @@ def handle_telemetry(
|
||||
lpp_data=lpp_bytes,
|
||||
parsed_data=parsed_data,
|
||||
received_at=now,
|
||||
event_hash=event_hash,
|
||||
)
|
||||
session.add(telemetry)
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
from sqlalchemy import select
|
||||
|
||||
from meshcore_hub.common.database import DatabaseManager
|
||||
from meshcore_hub.common.hash_utils import compute_trace_hash
|
||||
from meshcore_hub.common.models import Node, TracePath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -40,7 +41,19 @@ def handle_trace_data(
|
||||
snr_values = payload.get("snr_values")
|
||||
hop_count = payload.get("hop_count")
|
||||
|
||||
# Compute event hash for deduplication (initiator_tag is unique per trace)
|
||||
event_hash = compute_trace_hash(initiator_tag=initiator_tag)
|
||||
|
||||
with db.session_scope() as session:
|
||||
# Check if trace with same hash already exists
|
||||
existing = session.execute(
|
||||
select(TracePath.id).where(TracePath.event_hash == event_hash)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existing:
|
||||
logger.debug(f"Duplicate trace skipped (tag={initiator_tag})")
|
||||
return
|
||||
|
||||
# Find receiver node
|
||||
receiver_node = None
|
||||
if public_key:
|
||||
@@ -69,6 +82,7 @@ def handle_trace_data(
|
||||
snr_values=snr_values,
|
||||
hop_count=hop_count,
|
||||
received_at=now,
|
||||
event_hash=event_hash,
|
||||
)
|
||||
session.add(trace_path)
|
||||
|
||||
|
||||
@@ -58,8 +58,15 @@ class Advertisement(Base, UUIDMixin, TimestampMixin):
|
||||
default=utc_now,
|
||||
nullable=False,
|
||||
)
|
||||
event_hash: Mapped[Optional[str]] = mapped_column(
|
||||
String(32),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
__table_args__ = (Index("ix_advertisements_received_at", "received_at"),)
|
||||
__table_args__ = (
|
||||
Index("ix_advertisements_received_at", "received_at"),
|
||||
Index("ix_advertisements_event_hash", "event_hash"),
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<Advertisement(id={self.id}, public_key={self.public_key[:12]}..., name={self.name})>"
|
||||
|
||||
@@ -76,12 +76,17 @@ class Message(Base, UUIDMixin, TimestampMixin):
|
||||
default=utc_now,
|
||||
nullable=False,
|
||||
)
|
||||
event_hash: Mapped[Optional[str]] = mapped_column(
|
||||
String(32),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_messages_message_type", "message_type"),
|
||||
Index("ix_messages_pubkey_prefix", "pubkey_prefix"),
|
||||
Index("ix_messages_channel_idx", "channel_idx"),
|
||||
Index("ix_messages_received_at", "received_at"),
|
||||
Index("ix_messages_event_hash", "event_hash"),
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
|
||||
@@ -54,8 +54,15 @@ class Telemetry(Base, UUIDMixin, TimestampMixin):
|
||||
default=utc_now,
|
||||
nullable=False,
|
||||
)
|
||||
event_hash: Mapped[Optional[str]] = mapped_column(
|
||||
String(32),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
__table_args__ = (Index("ix_telemetry_received_at", "received_at"),)
|
||||
__table_args__ = (
|
||||
Index("ix_telemetry_received_at", "received_at"),
|
||||
Index("ix_telemetry_event_hash", "event_hash"),
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, String
|
||||
from sqlalchemy.dialects.sqlite import JSON
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
@@ -67,10 +67,15 @@ class TracePath(Base, UUIDMixin, TimestampMixin):
|
||||
default=utc_now,
|
||||
nullable=False,
|
||||
)
|
||||
event_hash: Mapped[Optional[str]] = mapped_column(
|
||||
String(32),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_trace_paths_initiator_tag", "initiator_tag"),
|
||||
Index("ix_trace_paths_received_at", "received_at"),
|
||||
Index("ix_trace_paths_event_hash", "event_hash"),
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from meshcore_hub.common.hash_utils import (
|
||||
compute_advertisement_hash,
|
||||
compute_message_hash,
|
||||
|
||||
Reference in New Issue
Block a user