adam/pyMC_Repeater
1
0
Fork 0
mirror of https://github.com/pyMC-dev/pyMC_Repeater.git synced 2026-08-08 09:52:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor: companion FrameServer and related (substantive only, no Black)

Browse Source 
Reapply refactor from ce8381a (replace monolithic FrameServer with thin
pymc_core subclass, re-export constants, SQLite persistence hooks) while
preserving pre-refactor whitespace where patch applied cleanly. Remaining
files match refactor commit exactly. Diff vs ce8381a is whitespace-only.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
agessaman
2026-02-21 15:35:47 -08:00
parent 22e337a707
commit c2f8a2e3cd
57 changed files with 4385 additions and 4922 deletions
Download Patch File Download Diff File Expand all files Collapse all files
repeater/data_acquisition/__init__.py
+3 -3
View File
@@ -1,6 +1,6 @@
from .sqlite_handler import SQLiteHandler
from .rrdtool_handler import RRDToolHandler
from .mqtt_handler import MQTTHandler
from .rrdtool_handler import RRDToolHandler
from .sqlite_handler import SQLiteHandler
from .storage_collector import StorageCollector
__all__ = ['SQLiteHandler', 'RRDToolHandler', 'MQTTHandler', 'StorageCollector']
__all__ = ["SQLiteHandler", "RRDToolHandler", "MQTTHandler", "StorageCollector"]
repeater/data_acquisition/hardware_stats.py
+36 -51
View File
@@ -5,13 +5,14 @@ KISS - Keep It Simple Stupid approach.
try:
import psutil
PSUTIL_AVAILABLE = True
except ImportError:
PSUTIL_AVAILABLE = False
psutil = None
import time
import logging
import time
logger = logging.getLogger("HardwareStats")
@@ -26,10 +27,8 @@ class HardwareStatsCollector:
if not PSUTIL_AVAILABLE:
logger.error("psutil not available - cannot collect hardware stats")
return {
"error": "psutil library not available - cannot collect hardware statistics"
}
return {"error": "psutil library not available - cannot collect hardware statistics"}
try:
# Get current timestamp
now = time.time()
@@ -42,10 +41,10 @@ class HardwareStatsCollector:
# Memory stats
memory = psutil.virtual_memory()
# Disk stats
disk = psutil.disk_usage('/')
disk = psutil.disk_usage("/")
# Network stats (total across all interfaces)
net_io = psutil.net_io_counters()
@@ -79,48 +78,39 @@ class HardwareStatsCollector:
"usage_percent": cpu_percent,
"count": cpu_count,
"frequency": cpu_freq.current if cpu_freq else 0,
"load_avg": {
"1min": load_avg[0],
"5min": load_avg[1],
"15min": load_avg[2]
}
"load_avg": {"1min": load_avg[0], "5min": load_avg[1], "15min": load_avg[2]},
},
"memory": {
"total": memory.total,
"available": memory.available,
"used": memory.used,
"usage_percent": memory.percent
"usage_percent": memory.percent,
},
"disk": {
"total": disk.total,
"used": disk.used,
"free": disk.free,
"usage_percent": round((disk.used / disk.total) * 100, 1)
"usage_percent": round((disk.used / disk.total) * 100, 1),
},
"network": {
"bytes_sent": net_io.bytes_sent,
"bytes_recv": net_io.bytes_recv,
"packets_sent": net_io.packets_sent,
"packets_recv": net_io.packets_recv
"packets_recv": net_io.packets_recv,
},
"system": {
"uptime": system_uptime,
"boot_time": boot_time
}
"system": {"uptime": system_uptime, "boot_time": boot_time},
}
# Add temperatures if available
if temperatures:
stats["temperatures"] = temperatures
return stats
except Exception as e:
logger.error(f"Error collecting hardware stats: {e}")
return {
"error": str(e)
}
return {"error": str(e)}
def get_processes_summary(self, limit=10):
"""
Get top processes by CPU and memory usage.
@@ -131,44 +121,39 @@ class HardwareStatsCollector:
return {
"processes": [],
"total_processes": 0,
"error": "psutil library not available - cannot collect process statistics"
"error": "psutil library not available - cannot collect process statistics",
}
try:
processes = []
# Get all processes
for proc in psutil.process_iter(['pid', 'name', 'cpu_percent', 'memory_percent', 'memory_info']):
for proc in psutil.process_iter(
["pid", "name", "cpu_percent", "memory_percent", "memory_info"]
):
try:
pinfo = proc.info
# Calculate memory in MB
memory_mb = 0
if pinfo['memory_info']:
memory_mb = pinfo['memory_info'].rss / 1024 / 1024 # RSS in MB
if pinfo["memory_info"]:
memory_mb = pinfo["memory_info"].rss / 1024 / 1024 # RSS in MB
process_data = {
"pid": pinfo['pid'],
"name": pinfo['name'] or 'Unknown',
"cpu_percent": pinfo['cpu_percent'] or 0.0,
"memory_percent": pinfo['memory_percent'] or 0.0,
"memory_mb": round(memory_mb, 1)
"pid": pinfo["pid"],
"name": pinfo["name"] or "Unknown",
"cpu_percent": pinfo["cpu_percent"] or 0.0,
"memory_percent": pinfo["memory_percent"] or 0.0,
"memory_mb": round(memory_mb, 1),
}
processes.append(process_data)
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
# Sort by CPU usage and get top processes
top_processes = sorted(processes, key=lambda x: x['cpu_percent'], reverse=True)[:limit]
return {
"processes": top_processes,
"total_processes": len(processes)
}
top_processes = sorted(processes, key=lambda x: x["cpu_percent"], reverse=True)[:limit]
return {"processes": top_processes, "total_processes": len(processes)}
except Exception as e:
logger.error(f"Error collecting process stats: {e}")
return {
"processes": [],
"total_processes": 0,
"error": str(e)
}
return {"processes": [], "total_processes": 0, "error": str(e)}
repeater/data_acquisition/letsmesh_handler.py
+49 -41
View File
@@ -1,24 +1,27 @@
import base64
import binascii
import json
import logging
import binascii
import base64
import paho.mqtt.client as mqtt
import threading
from datetime import UTC, datetime, timedelta
from typing import Callable, Dict, List, Optional
from datetime import datetime, timedelta, UTC
import paho.mqtt.client as mqtt
from nacl.signing import SigningKey
from typing import Callable, Optional, List, Dict
from .. import __version__
from .. import __version__
# Try to import paho-mqtt error code mappings
try:
from paho.mqtt.reasoncodes import ReasonCode
HAS_REASON_CODES = True
except ImportError:
HAS_REASON_CODES = False
logger = logging.getLogger("LetsMeshHandler")
# --------------------------------------------------------------------
# Helper: Base64URL without padding
# --------------------------------------------------------------------
@@ -117,7 +120,7 @@ class _BrokerConnection:
payload_b64 = b64url(json.dumps(payload, separators=(",", ":")).encode())
signing_input = f"{header_b64}.{payload_b64}".encode()
# Sign using LocalIdentity (supports both standard and firmware keys)
try:
signature = self.local_identity.sign(signing_input)
@@ -126,10 +129,10 @@ class _BrokerConnection:
logging.error(f" - public_key: {self.public_key}")
logging.error(f" - signing_input length: {len(signing_input)}")
raise
signature_hex = binascii.hexlify(signature).decode()
token = f"{header_b64}.{payload_b64}.{signature_hex}"
logging.debug(f"JWT token generated for {self.broker['name']}: {token[:50]}...")
return token
@@ -152,7 +155,7 @@ class _BrokerConnection:
"""MQTT disconnection callback"""
was_running = self._running
self._running = False
if rc != 0: # Unexpected disconnect
error_msg = get_mqtt_error_message(rc, is_disconnect=True)
logging.warning(f"Disconnected from {self.broker['name']} (rc={rc}): {error_msg}")
@@ -160,7 +163,7 @@ class _BrokerConnection:
self._schedule_reconnect(reason=error_msg)
else:
logging.info(f"Clean disconnect from {self.broker['name']}")
if self._on_disconnect_callback:
self._on_disconnect_callback(self.broker["name"])
@@ -168,29 +171,31 @@ class _BrokerConnection:
"""Schedule reconnection with exponential backoff"""
if self._reconnect_timer:
self._reconnect_timer.cancel()
# Exponential backoff: 5s, 10s, 20s, 40s, 80s, up to max
delay = min(5 * (2 ** self._reconnect_attempts), self._max_reconnect_delay)
delay = min(5 * (2**self._reconnect_attempts), self._max_reconnect_delay)
self._reconnect_attempts += 1
logging.info(f"Scheduling reconnect to {self.broker['name']} in {delay}s (attempt {self._reconnect_attempts}, reason: {reason})")
logging.info(
f"Scheduling reconnect to {self.broker['name']} in {delay}s (attempt {self._reconnect_attempts}, reason: {reason})"
)
self._reconnect_timer = threading.Timer(delay, lambda: self._attempt_reconnect(reason))
self._reconnect_timer.daemon = True
self._reconnect_timer.start()
def _attempt_reconnect(self, reason: str = "connection lost"):
"""Attempt to reconnect to broker with fresh JWT"""
try:
logging.info(f"Attempting reconnection to {self.broker['name']} (reason: {reason})...")
# Stop the loop if it's still running (websocket mode requires clean restart)
try:
self.client.loop_stop()
except:
pass
self._set_jwt_credentials()
# Reconnect and restart loop
self.client.connect(self.broker["host"], self.broker["port"], keepalive=60)
self.client.loop_start()
@@ -198,7 +203,7 @@ class _BrokerConnection:
except Exception as e:
logging.error(f"Reconnection failed for {self.broker['name']}: {e}")
self._schedule_reconnect() # Try again later
def _set_jwt_credentials(self):
"""Set JWT token credentials before connecting (CONNECT handshake only)"""
try:
@@ -242,7 +247,7 @@ class _BrokerConnection:
"""Disconnect from broker"""
self._running = False
self._loop_running = False
# Cancel any pending timers
if self._reconnect_timer:
self._reconnect_timer.cancel()
@@ -250,7 +255,7 @@ class _BrokerConnection:
if self._jwt_refresh_timer:
self._jwt_refresh_timer.cancel()
self._jwt_refresh_timer = None
self.client.loop_stop()
self.client.disconnect()
logging.info(f"Disconnected from {self.broker['name']}")
@@ -265,7 +270,7 @@ class _BrokerConnection:
def is_connected(self) -> bool:
"""Check if connection is active"""
return self._running
def has_pending_reconnect(self) -> bool:
"""Check if a reconnection is scheduled"""
return self._reconnect_timer is not None and self._reconnect_timer.is_alive()
@@ -281,19 +286,19 @@ class _BrokerConnection:
stagger_offset = self.broker_index * 0.05
refresh_threshold = 0.80 + stagger_offset
return elapsed >= expiry_seconds * refresh_threshold
def _schedule_jwt_refresh(self):
"""Schedule proactive JWT refresh before token expires"""
if self._jwt_refresh_timer:
self._jwt_refresh_timer.cancel()
expiry_seconds = self.jwt_expiry_minutes * 60
# Stagger refresh by 5% per broker to prevent simultaneous disconnects
# Broker 0: 80%, Broker 1: 85%, Broker 2: 90%, etc.
stagger_offset = self.broker_index * 0.05
refresh_threshold = 0.80 + stagger_offset
refresh_delay = expiry_seconds * refresh_threshold
logging.info(
f"JWT refresh scheduled for {self.broker['name']} in {refresh_delay:.0f}s "
f"({refresh_threshold*100:.0f}% of {self.jwt_expiry_minutes}min token lifetime)"
@@ -301,12 +306,12 @@ class _BrokerConnection:
self._jwt_refresh_timer = threading.Timer(refresh_delay, self.reconnect_for_token_expiry)
self._jwt_refresh_timer.daemon = True
self._jwt_refresh_timer.start()
def reconnect_for_token_expiry(self):
"""Proactively reconnect with new JWT before current one expires"""
if not self._running:
return
logging.info(f"JWT token expiring soon for {self.broker['name']}, refreshing...")
self._running = False
self._jwt_refresh_timer = None
@@ -330,7 +335,7 @@ class MeshCoreToMqttJwtPusher:
# Store local identity and get public key
self.local_identity = local_identity
public_key = local_identity.get_public_key().hex().upper()
# Extract values from config
from ..config import get_node_info
@@ -356,9 +361,11 @@ class MeshCoreToMqttJwtPusher:
elif broker_index is None or broker_index == -1:
# Connect to all built-in brokers + additional ones
self.brokers = LETSMESH_BROKERS.copy()
logging.info(f"Multi-broker mode: connecting to all {len(LETSMESH_BROKERS)} built-in brokers")
logging.info(
f"Multi-broker mode: connecting to all {len(LETSMESH_BROKERS)} built-in brokers"
)
else:
if broker_index >= len(LETSMESH_BROKERS):
raise ValueError(f"Invalid broker_index {broker_index}")
self.brokers = [LETSMESH_BROKERS[broker_index]]
@@ -372,7 +379,7 @@ class MeshCoreToMqttJwtPusher:
logging.info(f"Added custom broker: {broker_config['name']}")
else:
logging.warning(f"Skipping invalid broker config: {broker_config}")
# Validate that we have at least one broker
if not self.brokers:
raise ValueError(
@@ -432,7 +439,7 @@ class MeshCoreToMqttJwtPusher:
# Check if all connections are down AND none have pending reconnects
all_down = all(not conn.is_connected() for conn in self.connections)
any_reconnecting = any(conn.has_pending_reconnect() for conn in self.connections)
if all_down and not any_reconnecting:
logging.warning("All broker connections lost with no pending reconnects")
elif all_down:
@@ -454,7 +461,7 @@ class MeshCoreToMqttJwtPusher:
timer.start()
except Exception as e:
logging.error(f"Failed to connect to {conn.broker['name']}: {e}")
def _delayed_connect(self, conn):
"""Connect a broker after a delay (called by timer)"""
try:
@@ -471,6 +478,7 @@ class MeshCoreToMqttJwtPusher:
self.publish_status(state="offline", origin=self.node_name, radio_config=self.radio_config)
import time
time.sleep(0.5) # Give time for messages to be sent
# Disconnect all brokers
@@ -493,7 +501,7 @@ class MeshCoreToMqttJwtPusher:
state="online", origin=self.node_name, radio_config=self.radio_config
)
logging.debug(f"Status heartbeat sent (next in {self.status_interval}s)")
time.sleep(self.status_interval)
except Exception as e:
logging.error(f"Status heartbeat error: {e}")
@@ -579,14 +587,15 @@ class MeshCoreToMqttJwtPusher:
# Helper Functions
# ====================================================================
def get_mqtt_error_message(rc: int, is_disconnect: bool = False) -> str:
"""
Get human-readable MQTT error message.
Args:
rc: Return code from paho-mqtt
is_disconnect: True if from on_disconnect, False if from on_connect
Returns:
Human-readable error message
"""
@@ -596,7 +605,7 @@ def get_mqtt_error_message(rc: int, is_disconnect: bool = False) -> str:
return f"{reason.name}: {reason.value}"
except (ValueError, AttributeError):
pass
# Fallback to manual mappings
connect_errors = {
0: "connection accepted",
@@ -607,7 +616,7 @@ def get_mqtt_error_message(rc: int, is_disconnect: bool = False) -> str:
5: "not authorized (JWT signature/format invalid)",
6: "reserved error code",
}
disconnect_errors = {
0: "normal disconnect",
1: "unacceptable protocol version",
@@ -618,7 +627,6 @@ def get_mqtt_error_message(rc: int, is_disconnect: bool = False) -> str:
16: "connection lost / protocol error",
17: "client timeout",
}
error_dict = disconnect_errors if is_disconnect else connect_errors
return error_dict.get(rc, f"unknown error code {rc}")
repeater/data_acquisition/mqtt_handler.py
+8 -7
View File
@@ -1,10 +1,11 @@
import json
import logging
import ssl
from typing import Dict, Any, Optional
from typing import Any, Dict, Optional
try:
import paho.mqtt.client as mqtt
MQTT_AVAILABLE = True
except ImportError:
MQTT_AVAILABLE = False
@@ -102,17 +103,17 @@ class MQTTHandler:
try:
base_topic = self.mqtt_config.get("base_topic", "meshcore/repeater")
topic = f"{base_topic}/{self.node_name}/{record_type}"
if record_type == "packet":
packet_record = PacketRecord.from_packet_record(
record,
origin=self.node_name,
origin_id=self.node_id
record, origin=self.node_name, origin_id=self.node_id
)
if not packet_record:
logger.debug("Skipping MQTT publish: packet missing required data for PacketRecord")
logger.debug(
"Skipping MQTT publish: packet missing required data for PacketRecord"
)
return
payload = packet_record.to_dict()
logger.debug("Publishing packet using PacketRecord format")
else:
repeater/data_acquisition/rrdtool_handler.py
+101 -91
View File
@@ -1,10 +1,11 @@
import logging
import time
from pathlib import Path
from typing import Optional, Dict, Any
from typing import Any, Dict, Optional
try:
import rrdtool
RRDTOOL_AVAILABLE = True
except ImportError:
RRDTOOL_AVAILABLE = False
@@ -23,17 +24,18 @@ class RRDToolHandler:
if not self.available:
logger.warning("RRDTool not available - skipping RRD initialization")
return
if self.rrd_path.exists():
logger.info(f"RRD database exists: {self.rrd_path}")
return
try:
rrdtool.create(
str(self.rrd_path),
"--step", "60",
"--start", str(int(time.time() - 60)),
"--step",
"60",
"--start",
str(int(time.time() - 60)),
"DS:rx_count:COUNTER:120:0:U",
"DS:tx_count:COUNTER:120:0:U",
"DS:drop_count:COUNTER:120:0:U",
@@ -42,7 +44,6 @@ class RRDToolHandler:
"DS:avg_length:GAUGE:120:0:256",
"DS:avg_score:GAUGE:120:0:1",
"DS:neighbor_count:GAUGE:120:0:U",
"DS:type_0:COUNTER:120:0:U",
"DS:type_1:COUNTER:120:0:U",
"DS:type_2:COUNTER:120:0:U",
@@ -60,25 +61,24 @@ class RRDToolHandler:
"DS:type_14:COUNTER:120:0:U",
"DS:type_15:COUNTER:120:0:U",
"DS:type_other:COUNTER:120:0:U",
"RRA:AVERAGE:0.5:1:10080",
"RRA:AVERAGE:0.5:5:8640",
"RRA:AVERAGE:0.5:60:8760",
"RRA:MAX:0.5:1:10080",
"RRA:MIN:0.5:1:10080"
"RRA:MIN:0.5:1:10080",
)
logger.info(f"RRD database created: {self.rrd_path}")
except Exception as e:
logger.error(f"Failed to create RRD database: {e}")
def update_packet_metrics(self, record: dict, cumulative_counts: dict):
if not self.available or not self.rrd_path.exists():
return
try:
timestamp = int(record.get("timestamp", time.time()))
try:
info = rrdtool.info(str(self.rrd_path))
last_update = int(info.get("last_update", timestamp - 60))
@@ -86,104 +86,114 @@ class RRDToolHandler:
return
except Exception as e:
logger.debug(f"Failed to get RRD info for packet update: {e}")
rx_total = cumulative_counts.get("rx_total", 0)
tx_total = cumulative_counts.get("tx_total", 0)
drop_total = cumulative_counts.get("drop_total", 0)
type_counts = cumulative_counts.get("type_counts", {})
type_values = []
for i in range(16):
type_values.append(str(type_counts.get(f"type_{i}", 0)))
type_values.append(str(type_counts.get("type_other", 0)))
# Handle None values for TX packets - use 'U' (unknown) for RRD
rssi = record.get('rssi')
snr = record.get('snr')
score = record.get('score')
rssi_val = 'U' if rssi is None else str(rssi)
snr_val = 'U' if snr is None else str(snr)
score_val = 'U' if score is None else str(score)
length_val = str(record.get('length', 0))
basic_values = f"{timestamp}:{rx_total}:{tx_total}:{drop_total}:" \
f"{rssi_val}:{snr_val}:{length_val}:{score_val}:" \
f"U"
rssi = record.get("rssi")
snr = record.get("snr")
score = record.get("score")
rssi_val = "U" if rssi is None else str(rssi)
snr_val = "U" if snr is None else str(snr)
score_val = "U" if score is None else str(score)
length_val = str(record.get("length", 0))
basic_values = (
f"{timestamp}:{rx_total}:{tx_total}:{drop_total}:"
f"{rssi_val}:{snr_val}:{length_val}:{score_val}:"
f"U"
)
type_values_str = ":".join(type_values)
values = f"{basic_values}:{type_values_str}"
rrdtool.update(str(self.rrd_path), values)
except Exception as e:
logger.error(f"Failed to update RRD packet metrics: {e}")
logger.debug(f"RRD packet update failed - record: {record}")
def get_data(self, start_time: Optional[int] = None, end_time: Optional[int] = None,
resolution: str = "average") -> Optional[dict]:
def get_data(
self,
start_time: Optional[int] = None,
end_time: Optional[int] = None,
resolution: str = "average",
) -> Optional[dict]:
if not self.available or not self.rrd_path.exists():
logger.error(f"RRD not available: available={self.available}, rrd_path exists={self.rrd_path.exists()}")
logger.error(
f"RRD not available: available={self.available}, rrd_path exists={self.rrd_path.exists()}"
)
return None
try:
if end_time is None:
end_time = int(time.time())
if start_time is None:
start_time = end_time - (24 * 3600)
fetch_result = rrdtool.fetch(
str(self.rrd_path),
resolution.upper(),
"--start", str(start_time),
"--end", str(end_time)
"--start",
str(start_time),
"--end",
str(end_time),
)
if not fetch_result:
logger.error("RRD fetch returned None")
return None
(start, end, step), data_sources, data_points = fetch_result
if not data_points:
logger.warning("No data points returned from RRD fetch")
result = {
"start_time": start,
"end_time": end,
"step": step,
"data_sources": data_sources,
"packet_types": {},
"metrics": {}
"metrics": {},
}
timestamps = []
current_time = start
for ds in data_sources:
if ds.startswith('type_'):
if 'packet_types' not in result:
result['packet_types'] = {}
result['packet_types'][ds] = []
if ds.startswith("type_"):
if "packet_types" not in result:
result["packet_types"] = {}
result["packet_types"][ds] = []
else:
result['metrics'][ds] = []
result["metrics"][ds] = []
for point in data_points:
timestamps.append(current_time)
for i, value in enumerate(point):
ds_name = data_sources[i]
if ds_name.startswith('type_'):
result['packet_types'][ds_name].append(value)
if ds_name.startswith("type_"):
result["packet_types"][ds_name].append(value)
else:
result['metrics'][ds_name].append(value)
result["metrics"][ds_name].append(value)
current_time += step
result['timestamps'] = timestamps
result["timestamps"] = timestamps
return result
except Exception as e:
logger.error(f"Failed to get RRD data: {e}")
return None
@@ -192,65 +202,65 @@ class RRDToolHandler:
try:
end_time = int(time.time())
start_time = end_time - (hours * 3600)
rrd_data = self.get_data(start_time, end_time)
if not rrd_data or 'packet_types' not in rrd_data:
if not rrd_data or "packet_types" not in rrd_data:
logger.warning(f"No RRD data available")
return None
type_totals = {}
packet_type_names = {
'type_0': 'Request (REQ)',
'type_1': 'Response (RESPONSE)',
'type_2': 'Plain Text Message (TXT_MSG)',
'type_3': 'Acknowledgment (ACK)',
'type_4': 'Node Advertisement (ADVERT)',
'type_5': 'Group Text Message (GRP_TXT)',
'type_6': 'Group Datagram (GRP_DATA)',
'type_7': 'Anonymous Request (ANON_REQ)',
'type_8': 'Returned Path (PATH)',
'type_9': 'Trace (TRACE)',
'type_10': 'Multi-part Packet',
'type_11': 'Control Packet Data',
'type_12': 'Reserved Type 12',
'type_13': 'Reserved Type 13',
'type_14': 'Reserved Type 14',
'type_15': 'Custom Packet (RAW_CUSTOM)',
'type_other': 'Other Types (>15)'
"type_0": "Request (REQ)",
"type_1": "Response (RESPONSE)",
"type_2": "Plain Text Message (TXT_MSG)",
"type_3": "Acknowledgment (ACK)",
"type_4": "Node Advertisement (ADVERT)",
"type_5": "Group Text Message (GRP_TXT)",
"type_6": "Group Datagram (GRP_DATA)",
"type_7": "Anonymous Request (ANON_REQ)",
"type_8": "Returned Path (PATH)",
"type_9": "Trace (TRACE)",
"type_10": "Multi-part Packet",
"type_11": "Control Packet Data",
"type_12": "Reserved Type 12",
"type_13": "Reserved Type 13",
"type_14": "Reserved Type 14",
"type_15": "Custom Packet (RAW_CUSTOM)",
"type_other": "Other Types (>15)",
}
total_valid_points = 0
for type_key, data_points in rrd_data['packet_types'].items():
for type_key, data_points in rrd_data["packet_types"].items():
valid_points = [p for p in data_points if p is not None]
total_valid_points += len(valid_points)
if total_valid_points < 10:
logger.warning(f"RRD data too sparse ({total_valid_points} valid points)")
return None
for type_key, data_points in rrd_data['packet_types'].items():
for type_key, data_points in rrd_data["packet_types"].items():
valid_points = [p for p in data_points if p is not None]
if len(valid_points) >= 2:
total = max(valid_points) - min(valid_points)
elif len(valid_points) == 1:
total = valid_points[0]
else:
total = 0
type_name = packet_type_names.get(type_key, type_key)
type_totals[type_name] = max(0, total or 0)
result = {
"hours": hours,
"packet_type_totals": type_totals,
"total_packets": sum(type_totals.values()),
"period": f"{hours} hours",
"data_source": "rrd"
"data_source": "rrd",
}
return result
except Exception as e:
logger.error(f"Failed to get packet type stats from RRD: {e}")
return None
return None
repeater/data_acquisition/sqlite_handler.py
+713 -483
View File
File diff suppressed because it is too large Load Diff
repeater/data_acquisition/storage_collector.py
+39 -31
View File
@@ -3,15 +3,14 @@ import logging
import time
from datetime import datetime
from pathlib import Path
from typing import Optional, Dict, Any
from typing import Any, Dict, Optional
from .sqlite_handler import SQLiteHandler
from .rrdtool_handler import RRDToolHandler
from .mqtt_handler import MQTTHandler
from .letsmesh_handler import MeshCoreToMqttJwtPusher
from .mqtt_handler import MQTTHandler
from .rrdtool_handler import RRDToolHandler
from .sqlite_handler import SQLiteHandler
from .storage_utils import PacketRecord
logger = logging.getLogger("StorageCollector")
@@ -62,16 +61,18 @@ class StorageCollector:
self.disallowed_packet_types = set()
else:
self.disallowed_packet_types = set()
# Initialize hardware stats collector
from .hardware_stats import HardwareStatsCollector
self.hardware_stats = HardwareStatsCollector()
logger.info("Hardware stats collector initialized")
# Initialize WebSocket handler for real-time updates
self.websocket_available = False
try:
from .websocket_handler import broadcast_packet, broadcast_stats
self.websocket_broadcast_packet = broadcast_packet
self.websocket_broadcast_stats = broadcast_stats
self.websocket_available = True
@@ -87,23 +88,23 @@ class StorageCollector:
"packets_sent": 0,
"packets_received": 0,
"errors": 0,
"queue_len": 0
"queue_len": 0,
}
uptime_secs = int(time.time() - self.repeater_handler.start_time)
# Get airtime stats
airtime_stats = self.repeater_handler.airtime_mgr.get_stats()
# Get latest noise floor from database
noise_floor = None
try:
recent_noise = self.sqlite_handler.get_noise_floor_history(hours=0.5, limit=1)
if recent_noise and len(recent_noise) > 0:
noise_floor = recent_noise[-1].get('noise_floor_dbm')
noise_floor = recent_noise[-1].get("noise_floor_dbm")
except Exception as e:
logger.debug(f"Could not fetch noise floor: {e}")
stats = {
"uptime_secs": uptime_secs,
"packets_sent": self.repeater_handler.forwarded_count,
@@ -111,22 +112,22 @@ class StorageCollector:
"errors": 0,
"queue_len": 0, # N/A for Python repeater
}
# Add airtime stats
if airtime_stats:
stats["tx_air_secs"] = airtime_stats["total_airtime_ms"] / 1000
stats["current_airtime_ms"] = airtime_stats["current_airtime_ms"]
stats["utilization_percent"] = airtime_stats["utilization_percent"]
# Add noise floor if available
if noise_floor is not None:
stats["noise_floor"] = noise_floor
return stats
def record_packet(self, packet_record: dict, skip_letsmesh_if_invalid: bool = True):
"""Record packet to storage and publish to MQTT/LetsMesh
Args:
packet_record: Dictionary containing packet information
skip_letsmesh_if_invalid: If True, don't publish packets with drop_reason to LetsMesh
@@ -141,28 +142,34 @@ class StorageCollector:
cumulative_counts = self.sqlite_handler.get_cumulative_counts()
self.rrd_handler.update_packet_metrics(packet_record, cumulative_counts)
self.mqtt_handler.publish(packet_record, "packet")
# Broadcast to WebSocket clients for real-time updates
if self.websocket_available:
try:
self.websocket_broadcast_packet(packet_record)
# Broadcast 24-hour packet stats (same as /api/packet_stats?hours=24)
packet_stats_24h = self.sqlite_handler.get_packet_stats(hours=24)
uptime_seconds = time.time() - self.repeater_handler.start_time if self.repeater_handler else 0
self.websocket_broadcast_stats({
"packet_stats": packet_stats_24h,
"system_stats": {
"uptime_seconds": uptime_seconds,
uptime_seconds = (
time.time() - self.repeater_handler.start_time if self.repeater_handler else 0
)
self.websocket_broadcast_stats(
{
"packet_stats": packet_stats_24h,
"system_stats": {
"uptime_seconds": uptime_seconds,
},
}
})
)
except Exception as e:
logger.debug(f"WebSocket broadcast failed: {e}")
# Publish to LetsMesh if enabled (skip invalid packets if requested)
if skip_letsmesh_if_invalid and packet_record.get('drop_reason'):
logger.debug(f"Skipping LetsMesh publish for packet with drop_reason: {packet_record.get('drop_reason')}")
if skip_letsmesh_if_invalid and packet_record.get("drop_reason"):
logger.debug(
f"Skipping LetsMesh publish for packet with drop_reason: {packet_record.get('drop_reason')}"
)
else:
self._publish_to_letsmesh(packet_record)
@@ -247,23 +254,24 @@ class StorageCollector:
def get_neighbors(self) -> dict:
return self.sqlite_handler.get_neighbors()
def get_node_name_by_pubkey(self, pubkey: str) -> Optional[str]:
"""
Lookup node name from adverts table by public key.
Args:
pubkey: Public key in hex string format
Returns:
Node name if found, None otherwise
"""
try:
import sqlite3
with sqlite3.connect(self.sqlite_handler.sqlite_path) as conn:
result = conn.execute(
"SELECT node_name FROM adverts WHERE pubkey = ? AND node_name IS NOT NULL ORDER BY last_seen DESC LIMIT 1",
(pubkey,)
(pubkey,),
).fetchone()
return result[0] if result else None
except Exception as e:
repeater/data_acquisition/storage_utils.py
+1 -1
View File
@@ -1,6 +1,6 @@
"""Storage utility classes and functions for data acquisition."""
from dataclasses import dataclass, asdict
from dataclasses import asdict, dataclass
from datetime import datetime
from typing import Optional
repeater/data_acquisition/websocket_handler.py
+14 -8
View File
@@ -1,19 +1,21 @@
"""
WebSocket handler for real-time packet updates - simple ws4py implementation
"""
import json
import logging
import threading
import time
import cherrypy
from urllib.parse import parse_qs
from ws4py.websocket import WebSocket
import cherrypy
from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool
from ws4py.websocket import WebSocket
logger = logging.getLogger("WebSocket")
# Suppress noisy ws4py error logs for normal disconnections (ConnectionResetError, etc.)
logging.getLogger('ws4py').setLevel(logging.CRITICAL)
logging.getLogger("ws4py").setLevel(logging.CRITICAL)
# Global set of connected clients
_connected_clients = set()
@@ -69,14 +71,18 @@ class PacketWebSocket(WebSocket):
# Auth success - store user and add to connected clients
self.user = payload.get("sub") # type: ignore[attr-defined]
_connected_clients.add(self)
logger.info(f"WebSocket connected ({self.user or 'unknown user'}). Total clients: {len(_connected_clients)}")
logger.info(
f"WebSocket connected ({self.user or 'unknown user'}). Total clients: {len(_connected_clients)}"
)
def closed(self, code, reason=None):
"""Called when a WebSocket connection is closed"""
_connected_clients.discard(self)
user = getattr(self, 'user', 'unknown')
logger.info(f"WebSocket disconnected (user: {user}, code: {code}, reason: {reason}). Total clients: {len(_connected_clients)}")
user = getattr(self, "user", "unknown")
logger.info(
f"WebSocket disconnected (user: {user}, code: {code}, reason: {reason}). Total clients: {len(_connected_clients)}"
)
def received_message(self, message):
"""Handle messages from client"""
try: