refactor: Rename core package to mesh_analyzer, formalize project setup, and introduce report regeneration with new configuration options.

This commit is contained in:
eddieoz
2025-11-28 19:04:46 +02:00
parent 2e1f06e6ff
commit ca612603cf
24 changed files with 83 additions and 38 deletions
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
"""
Debug script to capture and display traceroute packet structure.
Run this and send a traceroute to see the actual packet format.
"""
import sys
import os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from meshtastic import serial_interface
import time
import json
def on_receive(packet, interface):
"""Callback for received packets."""
try:
decoded = packet.get('decoded', {})
portnum = decoded.get('portnum')
if portnum == 'TRACEROUTE_APP':
print("\n" + "="*80)
print("TRACEROUTE PACKET RECEIVED")
print("="*80)
print(f"\nFrom: {packet.get('fromId')}")
print(f"To: {packet.get('toId')}")
print(f"\nFull Packet Structure:")
print(json.dumps(packet, indent=2, default=str))
print("\n" + "="*80)
# Check for route fields
print("\nLooking for route data:")
print(f" decoded.route: {decoded.get('route', 'NOT FOUND')}")
print(f" decoded.routeBack: {decoded.get('routeBack', 'NOT FOUND')}")
# Check all keys in decoded
print(f"\nAll keys in decoded: {list(decoded.keys())}")
except Exception as e:
print(f"Error in callback: {e}")
print("Connecting to Meshtastic...")
interface = serial_interface.SerialInterface()
# Subscribe to receive packets
from pubsub import pub
pub.subscribe(on_receive, "meshtastic.receive")
print("Listening for traceroute packets...")
print("Send a traceroute from another device or use: meshtastic --traceroute <node_id>")
print("Press Ctrl+C to exit")
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
print("\nExiting...")
interface.close()
+206
View File
@@ -0,0 +1,206 @@
#!/usr/bin/env python3
"""
Report Generator Tool
Regenerates markdown reports from JSON data files saved by the LoRa Mesh Analyzer.
Usage:
python report_generate.py <json_file_path> [--output <output_path>]
Example:
python report_generate.py reports/report-20251128-145548.json
python report_generate.py reports/report-20251128-145548.json --output custom-report.md
"""
import json
import sys
import os
import argparse
from datetime import datetime
# Add mesh_analyzer to path (parent directory)
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from mesh_analyzer.reporter import NetworkReporter
from mesh_analyzer.route_analyzer import RouteAnalyzer
def load_json_data(json_filepath):
"""
Load raw data from JSON file.
"""
if not os.path.exists(json_filepath):
print(f"Error: File not found: {json_filepath}")
sys.exit(1)
try:
with open(json_filepath, 'r') as f:
data = json.load(f)
return data
except json.JSONDecodeError as e:
print(f"Error: Invalid JSON file: {e}")
sys.exit(1)
except Exception as e:
print(f"Error loading file: {e}")
sys.exit(1)
def generate_report_from_json(json_filepath, output_path=None):
"""
Regenerate markdown report from JSON data.
"""
print(f"Loading data from: {json_filepath}")
# Load the JSON data
full_data = load_json_data(json_filepath)
# Extract session and data
session = full_data.get('session', {})
data = full_data.get('data', {})
# Extract all the components
nodes = data.get('nodes', {})
test_results = data.get('test_results', [])
analysis_issues = data.get('analysis_issues', [])
router_stats = data.get('router_stats', [])
route_analysis = data.get('route_analysis', {})
local_node = data.get('local_node')
config = session.get('config', {})
print(f"Session timestamp: {session.get('timestamp', 'Unknown')}")
print(f"Nodes: {len(nodes)}")
print(f"Test results: {len(test_results)}")
print(f"Analysis issues: {len(analysis_issues)}")
# Create a custom reporter that generates the file at the specified location
if output_path:
# Use the directory and filename from output_path
report_dir = os.path.dirname(output_path) or "."
filename_base = os.path.basename(output_path).replace('.md', '')
else:
# Generate new report in reports/ with regenerated timestamp
report_dir = "reports"
filename_base = None
reporter = NetworkReporter(report_dir=report_dir, config=config)
# Apply manual positions from config to nodes
# This ensures that even if the JSON data lacks positions, we use the latest config
manual_positions = config.get('manual_positions', {})
if manual_positions:
print(f"Applying {len(manual_positions)} manual positions from config...")
for node_id, pos in manual_positions.items():
if node_id in nodes:
node = nodes[node_id]
if 'position' not in node:
node['position'] = {}
if 'lat' in pos and 'lon' in pos:
node['position']['latitude'] = pos['lat']
node['position']['longitude'] = pos['lon']
# Recreate analyzer and re-run analysis to populate cluster_data and ch_util_data
from mesh_analyzer.analyzer import NetworkHealthAnalyzer
analyzer = NetworkHealthAnalyzer(config=config)
# Re-run analysis to populate analyzer data structures AND get new issues
new_issues = analyzer.analyze(nodes, packet_history=[], my_node=local_node, test_results=test_results)
# Run additional checks
if test_results:
new_issues.extend(analyzer.check_router_efficiency(nodes, test_results=test_results))
new_issues.extend(analyzer.check_route_quality(nodes, test_results=test_results))
# Use new issues for the report
analysis_issues = new_issues
# We need to temporarily override the filename generation if custom output is specified
if output_path:
# Monkey-patch the generate_report to use custom filename
original_generate = reporter.generate_report
def custom_generate(nodes, test_results, analysis_issues, local_node=None, router_stats=None, analyzer=None, override_timestamp=None, override_location=None, save_json=True):
# Temporarily change the method to use custom filename
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
custom_filename = os.path.basename(output_path)
filepath = os.path.join(report_dir, custom_filename)
from mesh_analyzer.route_analyzer import RouteAnalyzer
route_analyzer = RouteAnalyzer(nodes)
route_analysis_local = route_analyzer.analyze_routes(test_results)
try:
with open(filepath, "w") as f:
f.write(f"# Meshtastic Network Report\n")
f.write(f"**Date:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
f.write(f"**Regenerated from:** {os.path.basename(json_filepath)}\n\n")
reporter._write_executive_summary(f, nodes, test_results, analysis_issues, local_node)
reporter._write_network_health(f, analysis_issues, analyzer)
if router_stats:
reporter._write_router_performance_table(f, router_stats)
reporter._write_route_analysis(f, route_analysis_local)
reporter._write_traceroute_results(f, test_results, nodes, local_node)
reporter._write_recommendations(f, analysis_issues, test_results, analyzer)
print(f"✅ Report regenerated successfully: {filepath}")
return filepath
except Exception as e:
print(f"❌ Failed to generate report: {e}")
return None
reporter.generate_report = custom_generate
# Extract session metadata
# Use the 'session' variable already extracted from 'full_data'
original_timestamp = session.get('timestamp')
test_location = session.get('test_location')
# Generate the report
result = reporter.generate_report(
nodes=nodes,
test_results=test_results,
analysis_issues=analysis_issues,
local_node=local_node,
router_stats=router_stats,
analyzer=analyzer, # Pass analyzer parameter
override_timestamp=original_timestamp,
override_location=test_location,
save_json=False # Do not overwrite JSON when regenerating
)
return result
def main():
parser = argparse.ArgumentParser(
description='Regenerate markdown reports from JSON data files',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
python report_generate.py reports/report-20251128-145548.json
python report_generate.py reports/report-20251128-145548.json --output custom-report.md
"""
)
parser.add_argument(
'json_file',
help='Path to the JSON data file'
)
parser.add_argument(
'--output', '-o',
help='Custom output path for the markdown report (optional)',
default=None
)
args = parser.parse_args()
# Generate the report
generate_report_from_json(args.json_file, args.output)
if __name__ == "__main__":
main()
+89
View File
@@ -0,0 +1,89 @@
import logging
# Configure logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
def parse_packet(packet):
# Extract route information from traceroute packet
decoded = packet.get('decoded', {})
logger.debug(f"Decoded packet keys: {list(decoded.keys())}")
# The traceroute data is in decoded['traceroute'] (parsed by library)
# or in RouteDiscovery protobuf in payload (if raw)
route = []
route_back = []
# 1. Check for pre-parsed 'traceroute' dict (Meshtastic python lib does this)
if 'traceroute' in decoded:
tr = decoded['traceroute']
if isinstance(tr, dict):
route = tr.get('route', [])
route_back = tr.get('routeBack', [])
logger.debug(f"Found parsed traceroute: route={route}, route_back={route_back}")
# 2. Fallback: Try to parse RouteDiscovery protobuf from payload
elif 'payload' in decoded:
try:
from meshtastic import mesh_pb2
# If payload is bytes, parse it
if isinstance(decoded['payload'], bytes):
route_discovery = mesh_pb2.RouteDiscovery()
route_discovery.ParseFromString(decoded['payload'])
route = list(route_discovery.route)
route_back = list(route_discovery.route_back)
logger.debug(f"Parsed from bytes - route: {route}, route_back: {route_back}")
# If it's already a protobuf object
elif hasattr(decoded['payload'], 'route'):
route = list(decoded['payload'].route)
route_back = list(decoded['payload'].route_back)
logger.debug(f"Extracted from protobuf - route: {route}, route_back: {route_back}")
except Exception as e:
logger.debug(f"Could not parse RouteDiscovery protobuf: {e}")
# 3. Fallback: Old dict keys
if not route:
route = decoded.get('route', [])
route_back = decoded.get('routeBack', [])
return route, route_back
# Real packet data from debug log
packet = {
'from': 2905093827,
'to': 1119572084,
'channel': 1,
'decoded': {
'portnum': 'TRACEROUTE_APP',
'payload': b'\n\x08s\x81w{Z9\xedW\x12\x15\x16\xf2\xff\xff\xff\xff\xff\xff\xff\xff\x01\xb6\xff\xff\xff\xff\xff\xff\xff\xff\x01\x1a\x04s\x81w{"\x0b\xce\xff\xff\xff\xff\xff\xff\xff\xff\x01\x19',
'requestId': 1781248082,
'bitfield': 1,
'traceroute': {
'route': [2071429491, 1475164506],
'snrTowards': [22, -14, -74],
'routeBack': [2071429491],
'snrBack': [-50, 25],
'raw': "route: 2071429491..."
}
},
'id': 4198764725,
'rxSnr': 6.25,
'hopLimit': 3,
'rxRssi': -45,
'hopStart': 4,
'relayNode': 115,
'transportMechanism': 'TRANSPORT_LORA',
'fromId': '!ad2836c3',
'toId': '!42bb5074'
}
print("Testing parsing...")
r, rb = parse_packet(packet)
print(f"Result: route={r}, route_back={rb}")
if r == [2071429491, 1475164506] and rb == [2071429491]:
print("SUCCESS! Parsing logic works.")
else:
print("FAILURE! Parsing logic incorrect.")
+267
View File
@@ -0,0 +1,267 @@
#!/usr/bin/env python3
"""
Test script to verify the report generation refactoring.
Creates mock data and tests both JSON persistence and report regeneration.
"""
import sys
import os
import json
from datetime import datetime
# Add mesh_analyzer to path
sys.path.insert(0, os.path.dirname(__file__))
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from mesh_analyzer.reporter import NetworkReporter
def create_mock_data():
"""Create mock data similar to what the monitor would generate."""
# Mock nodes
nodes = {
"!12345678": {
"user": {"id": "!12345678", "longName": "Test Router 1", "shortName": "TR1"},
"position": {"latitude": 59.4370, "longitude": 24.7536},
"deviceMetrics": {"channelUtilization": 15.5, "airUtilTx": 2.3}
},
"!87654321": {
"user": {"id": "!87654321", "longName": "Test Router 2", "shortName": "TR2"},
"position": {"latitude": 59.4380, "longitude": 24.7550},
"deviceMetrics": {"channelUtilization": 8.2, "airUtilTx": 1.1}
}
}
# Mock test results
test_results = [
{
"node_id": "!12345678",
"status": "success",
"rtt": 2.5,
"hops_to": 2,
"hops_back": 2,
"snr": 8.5,
"route": ["!local", "!relay1", "!12345678"]
},
{
"node_id": "!87654321",
"status": "timeout",
"rtt": None,
"hops_to": None,
"hops_back": None,
"snr": None,
"route": []
}
]
# Mock analysis issues
analysis_issues = [
"Topology: High Router Density! Best positioned seems to be Test Router 1",
"Config: Network Size exceeds recommendations"
]
# Mock router stats
router_stats = [
{
"name": "Test Router 1",
"role": "ROUTER",
"neighbors": 5,
"routers_nearby": 2,
"ch_util": 15.5,
"relay_count": 12,
"status": "Active",
"radius": 2000
},
{
"name": "Test Router 2",
"role": "ROUTER",
"neighbors": 3,
"routers_nearby": 1,
"ch_util": 8.2,
"relay_count": 5,
"status": "Active",
"radius": 2000
}
]
# Mock local node
local_node = {
"user": {"id": "!local", "longName": "Local Node", "shortName": "LN"},
"position": {"latitude": 59.4360, "longitude": 24.7520}
}
# Mock config
config = {
"log_level": "info",
"traceroute_timeout": 60,
"router_density_threshold": 2000,
"analysis_mode": "distance"
}
return nodes, test_results, analysis_issues, router_stats, local_node, config
def test_report_generation():
"""Test that reports are generated in the reports/ folder with JSON."""
print("=" * 60)
print("Testing Report Generation with JSON Persistence")
print("=" * 60)
# Create mock data
nodes, test_results, analysis_issues, router_stats, local_node, config = create_mock_data()
# Create reporter
reporter = NetworkReporter(report_dir="reports", config=config)
print("\n✅ NetworkReporter created successfully")
print(f" Report directory: reports/")
print(f" Config passed: Yes")
# Generate report
print("\n📝 Generating report...")
report_path = reporter.generate_report(
nodes=nodes,
test_results=test_results,
analysis_issues=analysis_issues,
local_node=local_node,
router_stats=router_stats
)
if report_path:
print(f"✅ Report generated: {report_path}")
# Check if markdown report exists
if os.path.exists(report_path):
print(f"✅ Markdown file exists: {report_path}")
# Get file size
size_kb = os.path.getsize(report_path) / 1024
print(f" File size: {size_kb:.2f} KB")
else:
print(f"❌ Markdown file NOT found: {report_path}")
return False
# Check if JSON file exists
json_path = report_path.replace('.md', '.json')
if os.path.exists(json_path):
print(f"✅ JSON file exists: {json_path}")
# Get file size
size_kb = os.path.getsize(json_path) / 1024
print(f" File size: {size_kb:.2f} KB")
# Verify JSON structure
print("\n🔍 Verifying JSON structure...")
with open(json_path, 'r') as f:
data = json.load(f)
# Check session metadata
if 'session' in data:
print("✅ Session metadata present")
session = data['session']
print(f" Timestamp: {session.get('timestamp', 'N/A')}")
print(f" Generated at: {session.get('generated_at', 'N/A')}")
print(f" Config keys: {len(session.get('config', {}))}")
else:
print("❌ Session metadata missing")
return False
# Check data section
if 'data' in data:
print("✅ Data section present")
data_section = data['data']
print(f" Nodes: {len(data_section.get('nodes', {}))}")
print(f" Test results: {len(data_section.get('test_results', []))}")
print(f" Analysis issues: {len(data_section.get('analysis_issues', []))}")
print(f" Router stats: {len(data_section.get('router_stats', []))}")
print(f" Local node: {'present' if data_section.get('local_node') else 'missing'}")
else:
print("❌ Data section missing")
return False
return json_path
else:
print(f"❌ JSON file NOT found: {json_path}")
return False
else:
print("❌ Report generation failed")
return False
def test_report_regeneration(json_path):
"""Test report regeneration from JSON file."""
print("\n" + "=" * 60)
print("Testing Report Regeneration from JSON")
print("=" * 60)
if not json_path or not os.path.exists(json_path):
print(f"❌ JSON file not found: {json_path}")
return False
# Import the report generator
from report_generate import generate_report_from_json
print(f"\n📁 Source JSON: {json_path}")
# Test regeneration with custom output
custom_output = "reports/test-regenerated-report.md"
print(f"🔄 Regenerating report to: {custom_output}")
result = generate_report_from_json(json_path, custom_output)
if result and os.path.exists(custom_output):
print(f"✅ Report regenerated successfully: {custom_output}")
# Compare sizes (should be similar)
original_md = json_path.replace('.json', '.md')
if os.path.exists(original_md):
orig_size = os.path.getsize(original_md)
regen_size = os.path.getsize(custom_output)
print(f" Original size: {orig_size / 1024:.2f} KB")
print(f" Regenerated size: {regen_size / 1024:.2f} KB")
# They should be roughly the same size (within 10%)
if abs(orig_size - regen_size) / orig_size < 0.1:
print("✅ Size comparison: PASS (within 10%)")
else:
print("⚠️ Size comparison: Different (this is OK if content differs)")
return True
else:
print(f"❌ Report regeneration failed")
return False
def main():
print("\n🧪 REPORT GENERATION REFACTORING - VERIFICATION TESTS\n")
# Test 1: Report generation with JSON persistence
json_path = test_report_generation()
if not json_path:
print("\n❌ FAILED: Report generation test")
sys.exit(1)
# Test 2: Report regeneration from JSON
success = test_report_regeneration(json_path)
if not success:
print("\n❌ FAILED: Report regeneration test")
sys.exit(1)
print("\n" + "=" * 60)
print("✅ ALL TESTS PASSED!")
print("=" * 60)
print("\nSummary:")
print(" ✓ Reports are generated in reports/ folder")
print(" ✓ JSON files are created alongside markdown reports")
print(" ✓ JSON contains all session metadata and raw data")
print(" ✓ report_generate.py successfully regenerates reports from JSON")
print("\nNext steps:")
print(" - Clean up test files if needed")
print(" - Test with real data from the monitor")
if __name__ == "__main__":
main()