mirror of
https://github.com/SpudGunMan/meshing-around.git
synced 2026-03-28 17:32:36 +01:00
b351ed0cf4
Adding a log analyser that will take the logfile and parse it into a html file for monitoring. Run the python script within the logfolder from meshingaround. by default this is sent to /var/www/html/index.html See: https://ibb.co/ymz9TxZ
561 lines
20 KiB
Python
561 lines
20 KiB
Python
import os
|
|
import re
|
|
from datetime import datetime
|
|
from collections import Counter, defaultdict
|
|
import json
|
|
import platform
|
|
import subprocess
|
|
|
|
def parse_log_file(file_path):
|
|
with open(file_path, 'r') as file:
|
|
lines = file.readlines()
|
|
|
|
log_data = {
|
|
'command_counts': Counter(),
|
|
'message_types': Counter(),
|
|
'unique_users': set(),
|
|
'warnings': [],
|
|
'errors': [],
|
|
'hourly_activity': defaultdict(int),
|
|
'bbs_messages': 0,
|
|
'total_messages': 0,
|
|
'gps_coordinates': defaultdict(list),
|
|
'command_timestamps': [],
|
|
'message_timestamps': [],
|
|
}
|
|
|
|
for line in lines:
|
|
timestamp_match = re.match(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),\d+', line)
|
|
if timestamp_match:
|
|
timestamp = datetime.strptime(timestamp_match.group(1), '%Y-%m-%d %H:%M:%S')
|
|
log_data['hourly_activity'][timestamp.strftime('%Y-%m-%d %H:00:00')] += 1
|
|
|
|
if 'Bot detected Commands' in line:
|
|
command = re.search(r"'cmd': '(\w+)'", line)
|
|
if command:
|
|
cmd = command.group(1)
|
|
log_data['command_counts'][cmd] += 1
|
|
log_data['command_timestamps'].append((timestamp.isoformat(), cmd))
|
|
|
|
if 'Sending DM:' in line or 'Sending Multi-Chunk DM:' in line:
|
|
log_data['message_types']['Outgoing DM'] += 1
|
|
log_data['total_messages'] += 1
|
|
log_data['message_timestamps'].append((timestamp.isoformat(), 'Outgoing DM'))
|
|
|
|
if 'Received DM:' in line:
|
|
log_data['message_types']['Incoming DM'] += 1
|
|
log_data['total_messages'] += 1
|
|
log_data['message_timestamps'].append((timestamp.isoformat(), 'Incoming DM'))
|
|
|
|
user_match = re.search(r'From: (\w+)', line)
|
|
if user_match:
|
|
log_data['unique_users'].add(user_match.group(1))
|
|
|
|
if '| WARNING |' in line:
|
|
log_data['warnings'].append(line.strip())
|
|
|
|
if '| ERROR |' in line:
|
|
log_data['errors'].append(line.strip())
|
|
|
|
bbs_match = re.search(r'📡BBSdb has (\d+) messages', line)
|
|
if bbs_match:
|
|
log_data['bbs_messages'] = int(bbs_match.group(1))
|
|
|
|
gps_match = re.search(r'location data for (\d+) is ([-\d.]+),([-\d.]+)', line)
|
|
if gps_match:
|
|
node_id, lat, lon = gps_match.groups()
|
|
log_data['gps_coordinates'][node_id].append((float(lat), float(lon)))
|
|
|
|
log_data['unique_users'] = list(log_data['unique_users'])
|
|
return log_data
|
|
|
|
def get_system_info():
|
|
def get_command_output(command):
|
|
try:
|
|
return subprocess.check_output(command, shell=True).decode('utf-8').strip()
|
|
except subprocess.CalledProcessError:
|
|
return "N/A"
|
|
|
|
if platform.system() == "Linux":
|
|
uptime = get_command_output("uptime -p")
|
|
memory_total = get_command_output("free -m | awk '/Mem:/ {print $2}'")
|
|
memory_available = get_command_output("free -m | awk '/Mem:/ {print $7}'")
|
|
disk_total = get_command_output("df -h / | awk 'NR==2 {print $2}'")
|
|
disk_free = get_command_output("df -h / | awk 'NR==2 {print $4}'")
|
|
elif platform.system() == "Darwin": # macOS
|
|
uptime = get_command_output("uptime | awk '{print $3,$4,$5}'")
|
|
memory_total = get_command_output("sysctl -n hw.memsize | awk '{print $0/1024/1024}'")
|
|
memory_available = "N/A" # Not easily available on macOS without additional tools
|
|
disk_total = get_command_output("df -h / | awk 'NR==2 {print $2}'")
|
|
disk_free = get_command_output("df -h / | awk 'NR==2 {print $4}'")
|
|
else:
|
|
return {
|
|
'uptime': "N/A",
|
|
'memory_total': "N/A",
|
|
'memory_available': "N/A",
|
|
'disk_total': "N/A",
|
|
'disk_free': "N/A",
|
|
}
|
|
|
|
return {
|
|
'uptime': uptime,
|
|
'memory_total': f"{memory_total} MB",
|
|
'memory_available': f"{memory_available} MB" if memory_available != "N/A" else "N/A",
|
|
'disk_total': disk_total,
|
|
'disk_free': disk_free,
|
|
}
|
|
|
|
def generate_main_html(log_data, system_info):
|
|
html_template = """
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Meshbot (BBS) Network Statistics</title>
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
|
<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns/dist/chartjs-adapter-date-fns.bundle.min.js"></script>
|
|
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.7.1/dist/leaflet.css" />
|
|
<script src="https://unpkg.com/leaflet@1.7.1/dist/leaflet.js"></script>
|
|
<style>
|
|
body {
|
|
font-family: Arial, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background-color: #f0f0f0;
|
|
display: flex;
|
|
}
|
|
.header {
|
|
background-color: #333;
|
|
color: white;
|
|
padding: 10px;
|
|
font-size: 24px;
|
|
position: fixed;
|
|
top: 0;
|
|
left: 0;
|
|
right: 0;
|
|
z-index: 1000;
|
|
}
|
|
.sidebar {
|
|
width: 200px;
|
|
background-color: #ddd;
|
|
padding: 10px;
|
|
height: 100vh;
|
|
position: fixed;
|
|
top: 50px;
|
|
left: 0;
|
|
overflow-y: auto;
|
|
}
|
|
.content {
|
|
margin-left: 220px;
|
|
margin-top: 60px;
|
|
padding: 20px;
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
|
|
gap: 20px;
|
|
flex-grow: 1;
|
|
}
|
|
.chart-container {
|
|
background-color: white;
|
|
border: 1px solid #ccc;
|
|
border-radius: 4px;
|
|
padding: 10px;
|
|
height: 400px;
|
|
display: flex;
|
|
flex-direction: column;
|
|
}
|
|
.chart-title {
|
|
font-size: 16px;
|
|
font-weight: bold;
|
|
margin-bottom: 10px;
|
|
}
|
|
#map, .chart-content {
|
|
flex-grow: 1;
|
|
width: 100%;
|
|
}
|
|
.list-container {
|
|
background-color: white;
|
|
border: 1px solid #ccc;
|
|
border-radius: 4px;
|
|
padding: 10px;
|
|
height: 400px;
|
|
overflow-y: auto;
|
|
}
|
|
ul {
|
|
list-style-type: none;
|
|
padding: 0;
|
|
margin: 0;
|
|
}
|
|
li {
|
|
padding: 5px 0;
|
|
border-bottom: 1px solid #eee;
|
|
}
|
|
li:last-child {
|
|
border-bottom: none;
|
|
}
|
|
#iframe-content {
|
|
display: none;
|
|
position: fixed;
|
|
top: 50px;
|
|
left: 220px;
|
|
right: 0;
|
|
bottom: 0;
|
|
background: white;
|
|
z-index: 900;
|
|
}
|
|
iframe {
|
|
width: 100%;
|
|
height: 100%;
|
|
border: none;
|
|
}
|
|
.timestamp-list {
|
|
height: 200px;
|
|
overflow-y: auto;
|
|
font-size: 12px;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="header">Meshbot (BBS) Network Statistics</div>
|
|
<div class="sidebar">
|
|
<ul>
|
|
<li><a href="#" onclick="showDashboard(); return false;">Dashboard</a></li>
|
|
<li><a href="#" onclick="showIframe('network_map_${date}.html'); return false;">Network Map</a></li>
|
|
<li><a href="#" onclick="showIframe('hosts_${date}.html'); return false;">Host</a></li>
|
|
</ul>
|
|
</div>
|
|
<div class="content" id="dashboard-content">
|
|
<div class="chart-container">
|
|
<div class="chart-title">Node Locations</div>
|
|
<div id="map"></div>
|
|
</div>
|
|
<div class="chart-container">
|
|
<div class="chart-title">Network Activity</div>
|
|
<div class="chart-content">
|
|
<canvas id="activityChart"></canvas>
|
|
</div>
|
|
</div>
|
|
<div class="chart-container">
|
|
<div class="chart-title">Command Usage</div>
|
|
<div class="chart-content">
|
|
<canvas id="commandChart"></canvas>
|
|
</div>
|
|
</div>
|
|
<div class="chart-container">
|
|
<div class="chart-title">Message Types</div>
|
|
<div class="chart-content">
|
|
<canvas id="messageChart"></canvas>
|
|
</div>
|
|
</div>
|
|
<div class="chart-container">
|
|
<div class="chart-title">Message Counts</div>
|
|
<div class="chart-content">
|
|
<canvas id="messageCountChart"></canvas>
|
|
</div>
|
|
</div>
|
|
<div class="chart-container">
|
|
<div class="chart-title">Recent Commands</div>
|
|
<div class="timestamp-list">
|
|
<ul>
|
|
${command_timestamps}
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
<div class="chart-container">
|
|
<div class="chart-title">Recent Messages</div>
|
|
<div class="timestamp-list">
|
|
<ul>
|
|
${message_timestamps}
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
<div class="list-container">
|
|
<div class="chart-title">Unique Users</div>
|
|
<ul>
|
|
${unique_users}
|
|
</ul>
|
|
</div>
|
|
<div class="list-container">
|
|
<div class="chart-title">Warnings</div>
|
|
<ul>
|
|
${warnings}
|
|
</ul>
|
|
</div>
|
|
<div class="list-container">
|
|
<div class="chart-title">Errors</div>
|
|
<ul>
|
|
${errors}
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
<div id="iframe-content">
|
|
<iframe id="content-iframe" src=""></iframe>
|
|
</div>
|
|
<script>
|
|
const commandData = ${command_data};
|
|
const messageData = ${message_data};
|
|
const activityData = ${activity_data};
|
|
const messageCountData = {
|
|
labels: ['BBSdb Messages', 'Total Messages'],
|
|
datasets: [{
|
|
label: 'Message Counts',
|
|
data: [${bbs_messages}, ${total_messages}],
|
|
backgroundColor: ['rgba(255, 206, 86, 0.6)', 'rgba(75, 192, 192, 0.6)']
|
|
}]
|
|
};
|
|
|
|
const chartOptions = {
|
|
responsive: true,
|
|
maintainAspectRatio: false
|
|
};
|
|
|
|
new Chart(document.getElementById('commandChart'), {
|
|
type: 'bar',
|
|
data: {
|
|
labels: Object.keys(commandData),
|
|
datasets: [{
|
|
label: 'Command Usage',
|
|
data: Object.values(commandData),
|
|
backgroundColor: 'rgba(75, 192, 192, 0.6)'
|
|
}]
|
|
},
|
|
options: chartOptions
|
|
});
|
|
|
|
new Chart(document.getElementById('messageChart'), {
|
|
type: 'pie',
|
|
data: {
|
|
labels: Object.keys(messageData),
|
|
datasets: [{
|
|
data: Object.values(messageData),
|
|
backgroundColor: ['rgba(255, 99, 132, 0.6)', 'rgba(54, 162, 235, 0.6)']
|
|
}]
|
|
},
|
|
options: chartOptions
|
|
});
|
|
|
|
new Chart(document.getElementById('activityChart'), {
|
|
type: 'line',
|
|
data: {
|
|
labels: Object.keys(activityData),
|
|
datasets: [{
|
|
label: 'Hourly Activity',
|
|
data: Object.entries(activityData).map(([time, count]) => ({x: new Date(time), y: count})),
|
|
borderColor: 'rgba(153, 102, 255, 1)',
|
|
fill: false
|
|
}]
|
|
},
|
|
options: {
|
|
...chartOptions,
|
|
scales: {
|
|
x: {
|
|
type: 'time',
|
|
time: {
|
|
unit: 'hour',
|
|
displayFormats: {
|
|
hour: 'MMM d, HH:mm'
|
|
}
|
|
},
|
|
title: {
|
|
display: true,
|
|
text: 'Time'
|
|
}
|
|
},
|
|
y: {
|
|
beginAtZero: true,
|
|
title: {
|
|
display: true,
|
|
text: 'Activity Count'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
new Chart(document.getElementById('messageCountChart'), {
|
|
type: 'bar',
|
|
data: messageCountData,
|
|
options: {
|
|
...chartOptions,
|
|
scales: {
|
|
y: {
|
|
beginAtZero: true
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
var map = L.map('map').setView([0, 0], 2);
|
|
L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
|
|
attribution: '© OpenStreetMap contributors'
|
|
}).addTo(map);
|
|
|
|
var gpsCoordinates = ${gps_coordinates};
|
|
for (var nodeId in gpsCoordinates) {
|
|
var coords = gpsCoordinates[nodeId][0];
|
|
L.marker(coords).addTo(map)
|
|
.bindPopup("Node ID: " + nodeId);
|
|
}
|
|
|
|
var bounds = [];
|
|
for (var nodeId in gpsCoordinates) {
|
|
bounds.push(gpsCoordinates[nodeId][0]);
|
|
}
|
|
map.fitBounds(bounds);
|
|
|
|
function showIframe(src) {
|
|
document.getElementById('dashboard-content').style.display = 'none';
|
|
document.getElementById('iframe-content').style.display = 'block';
|
|
document.getElementById('content-iframe').src = src;
|
|
}
|
|
|
|
function showDashboard() {
|
|
document.getElementById('dashboard-content').style.display = 'grid';
|
|
document.getElementById('iframe-content').style.display = 'none';
|
|
document.getElementById('content-iframe').src = '';
|
|
}
|
|
</script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
from string import Template
|
|
template = Template(html_template)
|
|
return template.safe_substitute(
|
|
date=datetime.now().strftime('%Y_%m_%d'),
|
|
command_data=json.dumps(log_data['command_counts']),
|
|
message_data=json.dumps(log_data['message_types']),
|
|
activity_data=json.dumps(log_data['hourly_activity']),
|
|
bbs_messages=log_data['bbs_messages'],
|
|
total_messages=log_data['total_messages'],
|
|
gps_coordinates=json.dumps(log_data['gps_coordinates']),
|
|
unique_users='\n'.join(f'<li>{user}</li>' for user in log_data['unique_users']),
|
|
warnings='\n'.join(f'<li>{warning}</li>' for warning in log_data['warnings']),
|
|
errors='\n'.join(f'<li>{error}</li>' for error in log_data['errors']),
|
|
command_timestamps='\n'.join(f'<li>{timestamp}: {cmd}</li>' for timestamp, cmd in reversed(log_data['command_timestamps'][-50:])),
|
|
message_timestamps='\n'.join(f'<li>{timestamp}: {msg_type}</li>' for timestamp, msg_type in reversed(log_data['message_timestamps'][-50:]))
|
|
)
|
|
|
|
def generate_network_map_html(log_data):
|
|
html_template = """
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Network Map</title>
|
|
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.7.1/dist/leaflet.css" />
|
|
<script src="https://unpkg.com/leaflet@1.7.1/dist/leaflet.js"></script>
|
|
<style>
|
|
body { margin: 0; padding: 0; }
|
|
#map { position: absolute; top: 0; bottom: 0; width: 100%; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div id="map"></div>
|
|
<script>
|
|
var map = L.map('map').setView([0, 0], 2);
|
|
L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
|
|
attribution: '© OpenStreetMap contributors'
|
|
}).addTo(map);
|
|
|
|
var gpsCoordinates = ${gps_coordinates};
|
|
for (var nodeId in gpsCoordinates) {
|
|
var coords = gpsCoordinates[nodeId][0];
|
|
L.marker(coords).addTo(map)
|
|
.bindPopup("Node ID: " + nodeId);
|
|
}
|
|
|
|
var bounds = [];
|
|
for (var nodeId in gpsCoordinates) {
|
|
bounds.push(gpsCoordinates[nodeId][0]);
|
|
}
|
|
map.fitBounds(bounds);
|
|
</script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
from string import Template
|
|
template = Template(html_template)
|
|
return template.safe_substitute(gps_coordinates=json.dumps(log_data['gps_coordinates']))
|
|
|
|
def generate_hosts_html(system_info):
|
|
html_template = """
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Host Information</title>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; line-height: 1.6; padding: 20px; }
|
|
h1 { color: #333; }
|
|
table { border-collapse: collapse; width: 100%; }
|
|
th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
|
|
th { background-color: #f2f2f2; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>Host Information</h1>
|
|
<table>
|
|
<tr><th>Metric</th><th>Value</th></tr>
|
|
<tr><td>Uptime</td><td>${uptime}</td></tr>
|
|
<tr><td>Total Memory</td><td>${memory_total}</td></tr>
|
|
<tr><td>Available Memory</td><td>${memory_available}</td></tr>
|
|
<tr><td>Total Disk Space</td><td>${disk_total}</td></tr>
|
|
<tr><td>Free Disk Space</td><td>${disk_free}</td></tr>
|
|
</table>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
from string import Template
|
|
template = Template(html_template)
|
|
return template.safe_substitute(system_info)
|
|
|
|
def main():
|
|
log_dir = '/opt/meshing-around/logs'
|
|
today = datetime.now().strftime('%Y_%m_%d')
|
|
log_file = f'meshbot{today}.log'
|
|
log_path = os.path.join(log_dir, log_file)
|
|
|
|
log_data = parse_log_file(log_path)
|
|
system_info = get_system_info()
|
|
|
|
main_html = generate_main_html(log_data, system_info)
|
|
network_map_html = generate_network_map_html(log_data)
|
|
hosts_html = generate_hosts_html(system_info)
|
|
|
|
output_dir = '/var/www/html'
|
|
index_path = os.path.join(output_dir, 'index.html')
|
|
|
|
try:
|
|
# Create backup of existing index.html if it exists
|
|
if os.path.exists(index_path):
|
|
backup_path = os.path.join(output_dir, f'index_backup_{today}.html')
|
|
os.rename(index_path, backup_path)
|
|
print(f"Existing index.html backed up to {backup_path}")
|
|
|
|
# Write main HTML to index.html
|
|
with open(index_path, 'w') as f:
|
|
f.write(main_html)
|
|
print(f"Main dashboard written to {index_path}")
|
|
|
|
# Write other HTML files
|
|
with open(os.path.join(output_dir, f'network_map_{today}.html'), 'w') as f:
|
|
f.write(network_map_html)
|
|
|
|
with open(os.path.join(output_dir, f'hosts_{today}.html'), 'w') as f:
|
|
f.write(hosts_html)
|
|
|
|
print(f"HTML reports generated for {today} in {output_dir}")
|
|
|
|
except PermissionError:
|
|
print("Error: Permission denied. Please run the script with appropriate permissions (e.g., using sudo).")
|
|
except Exception as e:
|
|
print(f"An error occurred while writing the output: {str(e)}")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|