diff --git a/mesh_monitor/analyzer.py b/mesh_monitor/analyzer.py index d334c6a..8743ce3 100644 --- a/mesh_monitor/analyzer.py +++ b/mesh_monitor/analyzer.py @@ -202,11 +202,11 @@ class NetworkHealthAnalyzer: for s in stats: if "Redundant" in s['status']: - issues.append(f"Efficiency: Router '{s['name']}' is Redundant. Has {s['routers_2km']} other routers within 2km. Consolidate?") + issues.append(f"Efficiency: Router '{s['name']}' is Redundant. Has {s['routers_nearby']} other routers within {s['radius']/1000:.1f}km. Consolidate?") if "Congested" in s['status']: issues.append(f"Efficiency: Router '{s['name']}' is Congested (ChUtil {s['ch_util']:.1f}% > 20%).") if "Ineffective" in s['status']: - issues.append(f"Efficiency: Router '{s['name']}' is Ineffective. Has {s['neighbors_2km']} neighbors but relayed 0 packets in tests.") + issues.append(f"Efficiency: Router '{s['name']}' is Ineffective. Has {s['neighbors']} neighbors but relayed 0 packets in tests.") return issues @@ -413,9 +413,9 @@ class NetworkHealthAnalyzer: # 3. Analyze Clusters and Generate Recommendations for cluster in clusters: - # Sort by relay_count (desc), then neighbors_2km (desc) + # Sort by relay_count (desc), then neighbors (desc) # We want the "best" router first - cluster.sort(key=lambda x: (x['relay_count'], x['neighbors_2km']), reverse=True) + cluster.sort(key=lambda x: (x['relay_count'], x['neighbors']), reverse=True) best_router = cluster[0] others = cluster[1:] diff --git a/mesh_monitor/monitor.py b/mesh_monitor/monitor.py index e365a37..c535912 100644 --- a/mesh_monitor/monitor.py +++ b/mesh_monitor/monitor.py @@ -26,7 +26,7 @@ class MeshMonitor: self.hostname = hostname self.config = self.load_config(config_file) self.analyzer = NetworkHealthAnalyzer(config=self.config, ignore_no_position=ignore_no_position) - self.reporter = NetworkReporter() + self.reporter = NetworkReporter(report_dir="reports", config=self.config) self.active_tester = None self.running = False self.config = self.load_config(config_file) @@ -353,8 +353,8 @@ class MeshMonitor: # Report Issues if issues: logger.warning(f"Found {len(issues)} potential issues:") - for issue in issues: - logger.warning(f" - {issue}") + # for issue in issues: + # logger.warning(f" - {issue}") else: logger.debug("No critical issues found in current scan.") diff --git a/mesh_monitor/reporter.py b/mesh_monitor/reporter.py index 1dac285..9374944 100644 --- a/mesh_monitor/reporter.py +++ b/mesh_monitor/reporter.py @@ -1,6 +1,7 @@ import logging import time import os +import json from datetime import datetime from .utils import get_val, haversine, get_node_name @@ -9,16 +10,23 @@ from mesh_monitor.route_analyzer import RouteAnalyzer logger = logging.getLogger(__name__) class NetworkReporter: - def __init__(self, report_dir="."): + def __init__(self, report_dir="reports", config=None): self.report_dir = report_dir + self.config = config or {} + + # Ensure report directory exists + os.makedirs(self.report_dir, exist_ok=True) def generate_report(self, nodes, test_results, analysis_issues, local_node=None, router_stats=None): """ Generates a Markdown report based on collected data. + Also persists all raw data to JSON format. """ timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") filename = f"report-{timestamp}.md" + json_filename = f"report-{timestamp}.json" filepath = os.path.join(self.report_dir, filename) + json_filepath = os.path.join(self.report_dir, json_filename) logger.info(f"Generating network report: {filepath}") @@ -27,6 +35,7 @@ class NetworkReporter: route_analysis = route_analyzer.analyze_routes(test_results) try: + # --- Generate Markdown Report --- with open(filepath, "w") as f: # Header f.write(f"# Meshtastic Network Report\n") @@ -52,11 +61,103 @@ class NetworkReporter: self._write_recommendations(f, analysis_issues, test_results) logger.info(f"Report generated successfully: {filepath}") + + # --- Persist Raw Data to JSON --- + try: + self._save_json_data( + json_filepath, + timestamp, + nodes, + test_results, + analysis_issues, + local_node, + router_stats, + route_analysis + ) + logger.info(f"Raw data saved to: {json_filepath}") + except Exception as json_e: + logger.error(f"Failed to save JSON data: {json_e}") + return filepath except Exception as e: logger.error(f"Failed to generate report: {e}") return None + def _serialize_object(self, obj, visited=None): + """ + Recursively convert objects to JSON-serializable format. + Handles protobuf objects, custom classes, and nested structures. + Prevents infinite recursion from circular references. + """ + if visited is None: + visited = set() + + # Check for None and primitives first (before id check) + if obj is None: + return None + elif isinstance(obj, (str, int, float, bool)): + return obj + + # Check for circular references using object id + obj_id = id(obj) + if obj_id in visited: + # Return a placeholder for circular references + return "" + + # Mark this object as visited + visited.add(obj_id) + + try: + if isinstance(obj, (list, tuple)): + return [self._serialize_object(item, visited) for item in obj] + elif isinstance(obj, dict): + return {key: self._serialize_object(value, visited) for key, value in obj.items()} + elif hasattr(obj, '__dict__'): + # Convert objects with __dict__ to dictionary + return self._serialize_object(obj.__dict__, visited) + else: + # Fallback: convert to string + return str(obj) + finally: + # Remove from visited set when done processing this branch + visited.discard(obj_id) + + def _save_json_data(self, filepath, timestamp, nodes, test_results, analysis_issues, + local_node, router_stats, route_analysis): + """ + Saves all raw data to JSON file with session metadata. + """ + # Serialize local_node + local_node_data = None + if local_node: + if hasattr(local_node, '__dict__'): + local_node_data = self._serialize_object(local_node) + elif isinstance(local_node, dict): + local_node_data = self._serialize_object(local_node) + else: + local_node_data = str(local_node) + + # Build the JSON structure + data = { + "session": { + "timestamp": timestamp, + "generated_at": datetime.now().isoformat(), + "config": self._serialize_object(self.config) + }, + "data": { + "nodes": self._serialize_object(nodes), + "test_results": self._serialize_object(test_results), + "analysis_issues": analysis_issues, # Already a list of strings + "router_stats": self._serialize_object(router_stats), + "route_analysis": self._serialize_object(route_analysis), + "local_node": local_node_data + } + } + + # Write to file with pretty formatting + with open(filepath, 'w') as f: + json.dump(data, f, indent=2, default=str) + def _write_executive_summary(self, f, nodes, test_results, analysis_issues, local_node=None): f.write("## 1. Executive Summary\n") @@ -249,11 +350,14 @@ class NetworkReporter: f.write("No routers found.\n\n") return - # Get radius from first stat entry (default to 2000m if missing) - radius_m = router_stats[0].get('radius', 2000) - radius_km = radius_m / 1000.0 + # Get cluster_radius and router_density_threshold from config + cluster_radius_m = self.config.get('cluster_radius', 3000) + router_density_m = self.config.get('thresholds', {}).get('router_density_threshold', 2000) + + cluster_radius_km = cluster_radius_m / 1000.0 + router_density_km = router_density_m / 1000.0 - f.write(f"| Name | Role | Neighbors ({radius_km:.1f}km) | Routers ({radius_km:.1f}km) | ChUtil | Relayed | Status |\n") + f.write(f"| Name | Role | Neighbors ({cluster_radius_km:.1f}km) | Routers ({router_density_km:.1f}km) | ChUtil | Relayed | Status |\n") f.write("|---|---|---|---|---|---|---|\n") for s in router_stats: diff --git a/report_generate.py b/report_generate.py new file mode 100755 index 0000000..25ef034 --- /dev/null +++ b/report_generate.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +""" +Report Generator Tool + +Regenerates markdown reports from JSON data files saved by the LoRa Mesh Analyzer. + +Usage: + python report_generate.py [--output ] + +Example: + python report_generate.py reports/report-20251128-145548.json + python report_generate.py reports/report-20251128-145548.json --output custom-report.md +""" + +import json +import sys +import os +import argparse +from datetime import datetime + +# Add mesh_monitor to path +sys.path.insert(0, os.path.dirname(__file__)) + +from mesh_monitor.reporter import NetworkReporter +from mesh_monitor.route_analyzer import RouteAnalyzer + + +def load_json_data(json_filepath): + """ + Load raw data from JSON file. + """ + if not os.path.exists(json_filepath): + print(f"Error: File not found: {json_filepath}") + sys.exit(1) + + try: + with open(json_filepath, 'r') as f: + data = json.load(f) + return data + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON file: {e}") + sys.exit(1) + except Exception as e: + print(f"Error loading file: {e}") + sys.exit(1) + + +def generate_report_from_json(json_filepath, output_path=None): + """ + Regenerate markdown report from JSON data. + """ + print(f"Loading data from: {json_filepath}") + + # Load the JSON data + full_data = load_json_data(json_filepath) + + # Extract session and data + session = full_data.get('session', {}) + data = full_data.get('data', {}) + + # Extract all the components + nodes = data.get('nodes', {}) + test_results = data.get('test_results', []) + analysis_issues = data.get('analysis_issues', []) + router_stats = data.get('router_stats', []) + route_analysis = data.get('route_analysis', {}) + local_node = data.get('local_node') + config = session.get('config', {}) + + print(f"Session timestamp: {session.get('timestamp', 'Unknown')}") + print(f"Nodes: {len(nodes)}") + print(f"Test results: {len(test_results)}") + print(f"Analysis issues: {len(analysis_issues)}") + + # Create a custom reporter that generates the file at the specified location + if output_path: + # Use the directory and filename from output_path + report_dir = os.path.dirname(output_path) or "." + filename_base = os.path.basename(output_path).replace('.md', '') + else: + # Generate new report in reports/ with regenerated timestamp + report_dir = "reports" + filename_base = None + + reporter = NetworkReporter(report_dir=report_dir, config=config) + + # We need to temporarily override the filename generation if custom output is specified + if output_path: + # Monkey-patch the generate_report to use custom filename + original_generate = reporter.generate_report + + def custom_generate(nodes, test_results, analysis_issues, local_node=None, router_stats=None): + # Temporarily change the method to use custom filename + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + custom_filename = os.path.basename(output_path) + filepath = os.path.join(report_dir, custom_filename) + + from mesh_monitor.route_analyzer import RouteAnalyzer + route_analyzer = RouteAnalyzer(nodes) + route_analysis_local = route_analyzer.analyze_routes(test_results) + + try: + with open(filepath, "w") as f: + f.write(f"# Meshtastic Network Report\\n") + f.write(f"**Date:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\\n") + f.write(f"**Regenerated from:** {os.path.basename(json_filepath)}\\n\\n") + + reporter._write_executive_summary(f, nodes, test_results, analysis_issues, local_node) + reporter._write_network_health(f, analysis_issues) + + if router_stats: + reporter._write_router_performance_table(f, router_stats) + + reporter._write_route_analysis(f, route_analysis_local) + reporter._write_traceroute_results(f, test_results, nodes, local_node) + reporter._write_recommendations(f, analysis_issues, test_results) + + print(f"โœ… Report regenerated successfully: {filepath}") + return filepath + except Exception as e: + print(f"โŒ Failed to generate report: {e}") + return None + + reporter.generate_report = custom_generate + + # Generate the report + result = reporter.generate_report( + nodes=nodes, + test_results=test_results, + analysis_issues=analysis_issues, + local_node=local_node, + router_stats=router_stats + ) + + return result + + +def main(): + parser = argparse.ArgumentParser( + description='Regenerate markdown reports from JSON data files', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python report_generate.py reports/report-20251128-145548.json + python report_generate.py reports/report-20251128-145548.json --output custom-report.md + """ + ) + + parser.add_argument( + 'json_file', + help='Path to the JSON data file' + ) + + parser.add_argument( + '--output', '-o', + help='Custom output path for the markdown report (optional)', + default=None + ) + + args = parser.parse_args() + + # Generate the report + generate_report_from_json(args.json_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/test_report_refactoring.py b/test_report_refactoring.py new file mode 100644 index 0000000..b32fabd --- /dev/null +++ b/test_report_refactoring.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +""" +Test script to verify the report generation refactoring. +Creates mock data and tests both JSON persistence and report regeneration. +""" + +import sys +import os +import json +from datetime import datetime + +# Add mesh_monitor to path +sys.path.insert(0, os.path.dirname(__file__)) + +from mesh_monitor.reporter import NetworkReporter + + +def create_mock_data(): + """Create mock data similar to what the monitor would generate.""" + + # Mock nodes + nodes = { + "!12345678": { + "user": {"id": "!12345678", "longName": "Test Router 1", "shortName": "TR1"}, + "position": {"latitude": 59.4370, "longitude": 24.7536}, + "deviceMetrics": {"channelUtilization": 15.5, "airUtilTx": 2.3} + }, + "!87654321": { + "user": {"id": "!87654321", "longName": "Test Router 2", "shortName": "TR2"}, + "position": {"latitude": 59.4380, "longitude": 24.7550}, + "deviceMetrics": {"channelUtilization": 8.2, "airUtilTx": 1.1} + } + } + + # Mock test results + test_results = [ + { + "node_id": "!12345678", + "status": "success", + "rtt": 2.5, + "hops_to": 2, + "hops_back": 2, + "snr": 8.5, + "route": ["!local", "!relay1", "!12345678"] + }, + { + "node_id": "!87654321", + "status": "timeout", + "rtt": None, + "hops_to": None, + "hops_back": None, + "snr": None, + "route": [] + } + ] + + # Mock analysis issues + analysis_issues = [ + "Topology: High Router Density! Best positioned seems to be Test Router 1", + "Config: Network Size exceeds recommendations" + ] + + # Mock router stats + router_stats = [ + { + "name": "Test Router 1", + "role": "ROUTER", + "neighbors": 5, + "routers_nearby": 2, + "ch_util": 15.5, + "relay_count": 12, + "status": "Active", + "radius": 2000 + }, + { + "name": "Test Router 2", + "role": "ROUTER", + "neighbors": 3, + "routers_nearby": 1, + "ch_util": 8.2, + "relay_count": 5, + "status": "Active", + "radius": 2000 + } + ] + + # Mock local node + local_node = { + "user": {"id": "!local", "longName": "Local Node", "shortName": "LN"}, + "position": {"latitude": 59.4360, "longitude": 24.7520} + } + + # Mock config + config = { + "log_level": "info", + "traceroute_timeout": 60, + "router_density_threshold": 2000, + "analysis_mode": "distance" + } + + return nodes, test_results, analysis_issues, router_stats, local_node, config + + +def test_report_generation(): + """Test that reports are generated in the reports/ folder with JSON.""" + print("=" * 60) + print("Testing Report Generation with JSON Persistence") + print("=" * 60) + + # Create mock data + nodes, test_results, analysis_issues, router_stats, local_node, config = create_mock_data() + + # Create reporter + reporter = NetworkReporter(report_dir="reports", config=config) + + print("\nโœ… NetworkReporter created successfully") + print(f" Report directory: reports/") + print(f" Config passed: Yes") + + # Generate report + print("\n๐Ÿ“ Generating report...") + report_path = reporter.generate_report( + nodes=nodes, + test_results=test_results, + analysis_issues=analysis_issues, + local_node=local_node, + router_stats=router_stats + ) + + if report_path: + print(f"โœ… Report generated: {report_path}") + + # Check if markdown report exists + if os.path.exists(report_path): + print(f"โœ… Markdown file exists: {report_path}") + + # Get file size + size_kb = os.path.getsize(report_path) / 1024 + print(f" File size: {size_kb:.2f} KB") + else: + print(f"โŒ Markdown file NOT found: {report_path}") + return False + + # Check if JSON file exists + json_path = report_path.replace('.md', '.json') + if os.path.exists(json_path): + print(f"โœ… JSON file exists: {json_path}") + + # Get file size + size_kb = os.path.getsize(json_path) / 1024 + print(f" File size: {size_kb:.2f} KB") + + # Verify JSON structure + print("\n๐Ÿ” Verifying JSON structure...") + with open(json_path, 'r') as f: + data = json.load(f) + + # Check session metadata + if 'session' in data: + print("โœ… Session metadata present") + session = data['session'] + print(f" Timestamp: {session.get('timestamp', 'N/A')}") + print(f" Generated at: {session.get('generated_at', 'N/A')}") + print(f" Config keys: {len(session.get('config', {}))}") + else: + print("โŒ Session metadata missing") + return False + + # Check data section + if 'data' in data: + print("โœ… Data section present") + data_section = data['data'] + print(f" Nodes: {len(data_section.get('nodes', {}))}") + print(f" Test results: {len(data_section.get('test_results', []))}") + print(f" Analysis issues: {len(data_section.get('analysis_issues', []))}") + print(f" Router stats: {len(data_section.get('router_stats', []))}") + print(f" Local node: {'present' if data_section.get('local_node') else 'missing'}") + else: + print("โŒ Data section missing") + return False + + return json_path + else: + print(f"โŒ JSON file NOT found: {json_path}") + return False + else: + print("โŒ Report generation failed") + return False + + +def test_report_regeneration(json_path): + """Test report regeneration from JSON file.""" + print("\n" + "=" * 60) + print("Testing Report Regeneration from JSON") + print("=" * 60) + + if not json_path or not os.path.exists(json_path): + print(f"โŒ JSON file not found: {json_path}") + return False + + # Import the report generator + from report_generate import generate_report_from_json + + print(f"\n๐Ÿ“ Source JSON: {json_path}") + + # Test regeneration with custom output + custom_output = "reports/test-regenerated-report.md" + print(f"๐Ÿ”„ Regenerating report to: {custom_output}") + + result = generate_report_from_json(json_path, custom_output) + + if result and os.path.exists(custom_output): + print(f"โœ… Report regenerated successfully: {custom_output}") + + # Compare sizes (should be similar) + original_md = json_path.replace('.json', '.md') + if os.path.exists(original_md): + orig_size = os.path.getsize(original_md) + regen_size = os.path.getsize(custom_output) + print(f" Original size: {orig_size / 1024:.2f} KB") + print(f" Regenerated size: {regen_size / 1024:.2f} KB") + + # They should be roughly the same size (within 10%) + if abs(orig_size - regen_size) / orig_size < 0.1: + print("โœ… Size comparison: PASS (within 10%)") + else: + print("โš ๏ธ Size comparison: Different (this is OK if content differs)") + + return True + else: + print(f"โŒ Report regeneration failed") + return False + + +def main(): + print("\n๐Ÿงช REPORT GENERATION REFACTORING - VERIFICATION TESTS\n") + + # Test 1: Report generation with JSON persistence + json_path = test_report_generation() + + if not json_path: + print("\nโŒ FAILED: Report generation test") + sys.exit(1) + + # Test 2: Report regeneration from JSON + success = test_report_regeneration(json_path) + + if not success: + print("\nโŒ FAILED: Report regeneration test") + sys.exit(1) + + print("\n" + "=" * 60) + print("โœ… ALL TESTS PASSED!") + print("=" * 60) + print("\nSummary:") + print(" โœ“ Reports are generated in reports/ folder") + print(" โœ“ JSON files are created alongside markdown reports") + print(" โœ“ JSON contains all session metadata and raw data") + print(" โœ“ report_generate.py successfully regenerates reports from JSON") + print("\nNext steps:") + print(" - Clean up test files if needed") + print(" - Test with real data from the monitor") + + +if __name__ == "__main__": + main()