enhance sentry with expire out old records

choosing to resolve https://github.com/SpudGunMan/meshing-around/issues/47 with filtering out after 24 hours

README.md

@@ -142,7 +142,9 @@ signalCooldown = 5
 signalCycleLimit = 5
 ```
 # requirements
-can also be installed with `pip install -r requirements.txt`
+Python 3.4 and likely higher is needed, developed on latest release.
+
+The following can also be installed with `pip install -r requirements.txt` or using the install.sh script for venv and automation
 
 ```
 pip install meshtastic

config.template

@@ -43,6 +43,8 @@ zuluTime = False
 urlTimeout = 10
 # logging to file of the non Bot messages
 LogMessagesToFile = False
+# Logging of system messages to file
+SyslogToFile = False
 
 
 [sentry]

etc/bbsdb_admin.py

@@ -7,13 +7,21 @@ try:
     with open('../bbsdb.pkl', 'rb') as f:
         bbs_messages = pickle.load(f)
 except:
-    print ("\nSystem: bbsdb.pkl not found")
+    try:
+        with open('bbsdb.pkl', 'rb') as f:
+            bbs_messages = pickle.load(f)
+    except:
+        print ("\nSystem: bbsdb.pkl not found")
 
 try:
     with open('../bbsdm.pkl', 'rb') as f:
         bbs_dm = pickle.load(f)
 except:
-    print ("\nSystem: bbsdm.pkl not found")
+    try:
+        with open('bbsdm.pkl', 'rb') as f:
+            bbs_dm = pickle.load(f)
+    except:
+        print ("\nSystem: bbsdm.pkl not found")
 
 print ("\nSystem: bbs_messages")
 print (bbs_messages)

install.sh

@@ -44,6 +44,7 @@ if [ $venv == "y" ]; then
 
     # install dependencies
     pip install -U -r requirements.txt
+    fi
 else
     printf "\nSkipping virtual environment...\n"
     # install dependencies
@@ -93,7 +94,9 @@ if [ $bot == "n" ]; then
     if [ -f launch.sh ]; then
         printf "\nTo run the bot, use the command: ./launch.sh\n"
         ./launch.sh
+    fi
 fi
 
-echo "Goodbye!"
+
+printf "\nGoodbye!"
 exit 0

mesh_bot.py

@@ -50,7 +50,7 @@ def auto_response(message, snr, rssi, hop, message_from_id, channel_number, devi
     if len(cmds) > 0:
         # sort the commands by index value
         cmds = sorted(cmds, key=lambda k: k['index'])
-        logger.debug(f"System: Bot Detected: {cmds}")
+        logger.debug(f"System: Bot detected Commands:{cmds}")
         # run the first command after sorting
         bot_response = command_handler[cmds[0]['cmd']]()
 
@@ -122,8 +122,6 @@ def handle_bbspost(message, message_from_id, deviceID):
             toNode = get_num_from_short_name(toNode, deviceID)
             if toNode == 0:
                 return "Node not found " + message.split("@")[1].split("#")[0]
-            else:
-                logger.debug(f"System: bbspost, name lookup found: {toNode}")
         if "#" in message:
             body = message.split("#")[1]
             return bbs_post_dm(toNode, body, message_from_id)
@@ -326,7 +324,7 @@ def onReceive(packet, interface):
                             send_message(auto_response(message_string, snr, rssi, hop, message_from_id, channel_number, rxNode), channel_number, 0, rxNode)
                 else:
                     # message is not for bot to respond to
-                    # ignore the message but add it to the message history and repeat it if enabled
+                    # ignore the message but add it to the message history list
                     if zuluTime:
                         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                     else:
@@ -337,10 +335,15 @@ def onReceive(packet, interface):
                     else:
                         msg_history.pop(0)
                         msg_history.append((get_name_from_number(message_from_id, 'long', rxNode), message_string, channel_number, timestamp, rxNode))
-                    
-                    # check if repeater is enabled and the other interface is enabled
+
+                    # print the message to the log and sdout
+                    logger.info(f"Device:{rxNode} Channel:{channel_number} " + CustomFormatter.green + "Ignoring Message:" + CustomFormatter.white +\
+                                f" {message_string} " + CustomFormatter.purple + "From:" + CustomFormatter.white + f" {get_name_from_number(message_from_id)}")
+                    if log_messages_to_file:
+                        msgLogger.info(f"Device:{rxNode} Channel:{channel_number} | {get_name_from_number(message_from_id, 'long', rxNode)} | " + message_string.replace('\n', '-nl-'))
+
+                     # repeat the message on the other device
                     if repeater_enabled and interface2_enabled:         
-                        # repeat the message on the other device
                         # wait a 700ms to avoid message collision from lora-ack.
                         time.sleep(0.7)
                         rMsg = (f"{message_string} From:{get_name_from_number(message_from_id, 'short', rxNode)}")
@@ -352,12 +355,6 @@ def onReceive(packet, interface):
                             elif rxNode == 2:
                                 logger.debug(f"Repeating message on Device1 Channel:{channel_number}")
                                 send_message(rMsg, channel_number, 0, 1)
-                        msgLogger.info(f"Device:{rxNode} Channel:{channel_number} | {get_name_from_number(message_from_id, 'long', rxNode)} | " + message_string.replace('\n', '-nl-'))
-                    else: 
-                        # nothing to do for us
-                        logger.info(f"Device:{rxNode} Channel:{channel_number} " + CustomFormatter.green + "Ignoring Message:" + CustomFormatter.white +\
-                                     f" {message_string} " + CustomFormatter.purple + "From:" + CustomFormatter.white + f" {get_name_from_number(message_from_id)}")
-                        msgLogger.info(f"Device:{rxNode} Channel:{channel_number} | {get_name_from_number(message_from_id, 'long', rxNode)} | " + message_string.replace('\n', '-nl-'))
     except KeyError as e:
         logger.critical(f"System: Error processing packet: {e} Device:{rxNode}")
         print(packet) # print the packet for debugging
@@ -374,6 +371,8 @@ async def start_rx():
                     f"{get_name_from_number(myNodeNum2, 'short', 2)}. NodeID: {myNodeNum2}, {decimal_to_hex(myNodeNum2)}")
     if log_messages_to_file:
         logger.debug(f"System: Logging Messages to disk")
+    if syslog_to_file:
+        logger.debug(f"System: Logging System Logs to disk")
     if bbs_enabled:
         logger.debug(f"System: BBS Enabled, {bbsdb} has {len(bbs_messages)} messages. Direct Mail Messages waiting: {(len(bbs_dm) - 1)}")
     if solar_conditions_enabled:

modules/locationdata.py

@@ -1,4 +1,4 @@
-# helper functions to use location data
+# helper functions to use location data like NOAA weather
 # K7MHI Kelly Keeton 2024
 
 import json # pip install json
@@ -15,8 +15,8 @@ def where_am_i(lat=0, lon=0):
     whereIam = ""
     grid = mh.to_maiden(float(lat), float(lon))
     
-    if float(lat) == 0 and float(lon) == 0:
-        logger.error("Location: No GPS data, cant find where you are")
+    if int(float(lat)) == 0 and int(float(lon)) == 0:
+        logger.error("Location: No GPS data, try sending location")
         return NO_DATA_NOGPS
     
     # initialize Nominatim API
@@ -42,7 +42,7 @@ def where_am_i(lat=0, lon=0):
 def get_tide(lat=0, lon=0):
     station_id = ""
     if float(lat) == 0 and float(lon) == 0:
-        logger.error("Location:No GPS data, cant find where you are for tide")
+        logger.error("Location:No GPS data, try sending location for tide")
         return NO_DATA_NOGPS
     station_lookup_url = "https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi/tidepredstations.json?lat=" + str(lat) + "&lon=" + str(lon) + "&radius=50"
     try:
@@ -248,7 +248,7 @@ def getActiveWeatherAlertsDetail(lat=0, lon=0):
     # get the latest details of weather alerts from NOAA
     alerts = ""
     if float(lat) == 0 and float(lon) == 0:
-        logger.error("Location:No GPS data, cant find where you are for weather alerts")
+        logger.error("Location:No GPS data, try sending location for weather alerts")
         return NO_DATA_NOGPS
 
     alert_url = "https://api.weather.gov/alerts/active.atom?point=" + str(lat) + "," + str(lon)

modules/log.py

@@ -1,3 +1,7 @@
+# Custom logger for MeshBot and PongBot
+# you can change the sdtout_handler level to logging.INFO to only show INFO level logs
+# stdout_handler.setLevel(logging.INFO)vs stdout_handler.setLevel(logging.DEBUG)
+# 2024 Kelly Keeton K7MHI
 import logging
 from datetime import datetime
 from modules.settings import *
@@ -42,6 +46,7 @@ msgLogger.propagate = False
 # Define format for logs
 logFormat = '%(asctime)s | %(levelname)8s | %(message)s'
 msgLogFormat = '%(asctime)s | %(message)s'
+today = datetime.now()
 
 # Create stdout handler for logging to the console
 stdout_handler = logging.StreamHandler()
@@ -50,13 +55,18 @@ stdout_handler.setLevel(logging.DEBUG)
 # Set format for stdout handler
 stdout_handler.setFormatter(CustomFormatter(logFormat))
 
-# Create file handler for logging to a file
-today = datetime.now()
-file_handler = logging.FileHandler('messages{}.log'.format(today.strftime('%Y_%m_%d')))
-file_handler.setLevel(logging.INFO)
-file_handler.setFormatter(logging.Formatter(msgLogFormat))
-
 # Add handlers to the logger
 logger.addHandler(stdout_handler)
+if syslog_to_file:
+    # Create file handler for logging to a file
+    file_handler = logging.FileHandler('system{}.log'.format(today.strftime('%Y_%m_%d')))
+    file_handler.setLevel(logging.DEBUG) # DEBUG used for system logs
+    file_handler.setFormatter(logging.Formatter(logFormat))
+    logger.addHandler(file_handler)
+
 if log_messages_to_file:
+    # Create file handler for logging to a file
+    file_handler = logging.FileHandler('messages{}.log'.format(today.strftime('%Y_%m_%d')))
+    file_handler.setLevel(logging.INFO) # INFO used for messages
+    file_handler.setFormatter(logging.Formatter(msgLogFormat))
     msgLogger.addHandler(file_handler)

modules/settings.py

@@ -1,3 +1,5 @@
+# Settings for MeshBot and PongBot
+# 2024 Kelly Keeton K7MHI
 import configparser
 
 # messages
@@ -82,6 +84,7 @@ try:
     publicChannel = config['general'].getint('defaultChannel', 0) # the meshtastic public channel
     zuluTime = config['general'].getboolean('zuluTime', False) # aka 24 hour time
     log_messages_to_file = config['general'].getboolean('LogMessagesToFile', True) # default True
+    syslog_to_file = config['general'].getboolean('SyslogToFile', False) # default True
     urlTimeoutSeconds = config['general'].getint('urlTimeout', 10) # default 10 seconds
     store_forward_enabled = config['general'].getboolean('StoreForward', True) # default False
     storeFlimit = config['general'].getint('StoreLimit', 3) # default 3 messages for S&F

modules/solarconditions.py

@@ -1,5 +1,5 @@
 # helper functions to get HF band conditions, DRAP X-ray flux, and sunrise/sunset times
-# some code from https://github.com/Murturtle/MeshLink
+# HF code from https://github.com/Murturtle/MeshLink
 # K7MHI Kelly Keeton 2024
 
 import requests # pip install requests

modules/system.py

@@ -1,4 +1,4 @@
-# helper functions for system related tasks
+# helper functions and init for system related tasks
 # K7MHI Kelly Keeton 2024
 
 import meshtastic.serial_interface #pip install meshtastic
@@ -6,6 +6,7 @@ import meshtastic.tcp_interface
 import meshtastic.ble_interface
 import time
 import asyncio
+import contextlib # for suppressing output on watchdog
 from modules.log import *
 
 # Global Variables
@@ -125,6 +126,8 @@ if interface2_enabled:
 else:
     myNodeNum2 = 777
 
+# functions below
+
 def decimal_to_hex(decimal_number):
     return f"!{decimal_number:08x}"
 
@@ -166,25 +169,33 @@ def get_num_from_short_name(short_name, nodeInt=1):
     logger.debug(f"System: Getting Node Number from Short Name: {short_name} on Device: {nodeInt}")
     if nodeInt == 1:
         for node in interface1.nodes.values():
-            if str(short_name.lower()) == node['user']['shortName'].lower():
+            #logger.debug(f"System: Checking Node: {node['user']['shortName']} against {short_name} for number {node['num']}")
+            if short_name == node['user']['shortName']:
+                return node['num']
+            elif str(short_name.lower()) == node['user']['shortName'].lower():
                 return node['num']
             else:
                 # try other interface
                 if interface2_enabled:
                     for node in interface2.nodes.values():
-                        if str(short_name.lower()) == node['user']['shortName'].lower():
+                        if short_name == node['user']['shortName']:
+                            return node['num']
+                        elif str(short_name.lower()) == node['user']['shortName'].lower():
                             return node['num']
     if nodeInt == 2:
         for node in interface2.nodes.values():
-            if str(short_name.lower()) == node['user']['shortName'].lower():
+            if short_name == node['user']['shortName']:
+                return node['num']
+            elif str(short_name.lower()) == node['user']['shortName'].lower():
                 return node['num']
             else:
                 # try other interface
                 if interface2_enabled:
                     for node in interface1.nodes.values():
-                        if str(short_name.lower()) == node['user']['shortName'].lower():
+                        if short_name == node['user']['shortName']:
+                            return node['num']
+                        elif str(short_name.lower()) == node['user']['shortName'].lower():
                             return node['num']
-    
     return 0
     
 def get_node_list(nodeInt=1):
@@ -263,6 +274,7 @@ def get_node_location(number, nodeInt=1, channel=0):
     latitude = latitudeValue
     longitude = longitudeValue
     position = [latitudeValue,longitudeValue]
+    lastheard = 0
     if nodeInt == 1:
         if interface1.nodes:
             for node in interface1.nodes.values():
@@ -278,17 +290,15 @@ def get_node_location(number, nodeInt=1, channel=0):
                         return position
                     else:
                         logger.warning(f"System: No location data for {number} using default location")
-
                         # request location data
-                        try:
-                            logger.debug(f"System: Requesting location data for {number}")
-                            if nodeInt == 1:
-                                interface1.sendPosition(destinationId=number, wantResponse=False, channelIndex=channel)
-                            if nodeInt == 2:
-                                interface2.sendPosition(destinationId=number, wantResponse=False, channelIndex=channel)
-                        except Exception as e:
-                            logger.error(f"System: Error requesting location data for {number}. Error: {e}")
-
+                        # try:
+                        #     logger.debug(f"System: Requesting location data for {number}")
+                        #     if nodeInt == 1:
+                        #         interface1.sendPosition(destinationId=number, wantResponse=False, channelIndex=channel)
+                        #     if nodeInt == 2:
+                        #         interface2.sendPosition(destinationId=number, wantResponse=False, channelIndex=channel)
+                        # except Exception as e:
+                        #     logger.error(f"System: Error requesting location data for {number}. Error: {e}")
                         return position
         else:
             logger.warning(f"System: No nodes found")
@@ -315,6 +325,7 @@ def get_node_location(number, nodeInt=1, channel=0):
 
 def get_closest_nodes(nodeInt=1,returnCount=3):
     node_list = []
+
     if nodeInt == 1:
         if interface1.nodes:
             for node in interface1.nodes.values():
@@ -324,13 +335,18 @@ def get_closest_nodes(nodeInt=1,returnCount=3):
                         latitude = node['position']['latitude']
                         longitude = node['position']['longitude']
 
-                        # set radius around BOT position
-                        distance = round(geopy.distance.geodesic((latitudeValue, longitudeValue), (latitude, longitude)).m, 2)
+                        #lastheard time in unix time
+                        lastheard = node['lastHeard']
+                        #if last heard is over 24 hours ago, ignore the node
+                        if lastheard < (time.time() - 86400):
+                            continue
 
+                        # Calculate distance to node from config.ini location
+                        distance = round(geopy.distance.geodesic((latitudeValue, longitudeValue), (latitude, longitude)).m, 2)
+                        
                         if (distance < sentry_radius):
                             if nodeID != myNodeNum1 and myNodeNum2 and str(nodeID) not in sentryIgnoreList:
                                 node_list.append({'id': nodeID, 'latitude': latitude, 'longitude': longitude, 'distance': distance})
-                                # calculate distance to node and report
                                 
                     except Exception as e:
                         pass
@@ -350,6 +366,7 @@ def get_closest_nodes(nodeInt=1,returnCount=3):
         else:
             logger.error(f"System: No nodes found in closest_nodes on interface {nodeInt}")
             return ERROR_FETCHING_DATA
+
     if nodeInt == 2:
         if interface2.nodes:
             for node in interface2.nodes.values():
@@ -359,20 +376,23 @@ def get_closest_nodes(nodeInt=1,returnCount=3):
                         latitude = node['position']['latitude']
                         longitude = node['position']['longitude']
 
-                        # set radius around BOT position
-                        distance = geopy.distance.geodesic((latitudeValue, longitudeValue), (latitude, longitude)).m
+                        #lastheard time in unix time
+                        lastheard = node['lastHeard']
+                        #if last heard is over 24 hours ago, ignore the node
+                        if lastheard < (time.time() - 86400):
+                            continue
 
+                        # Calculate distance to node from config.ini location
+                        distance = round(geopy.distance.geodesic((latitudeValue, longitudeValue), (latitude, longitude)).m, 2)
+                        
                         if (distance < sentry_radius):
                             if nodeID != myNodeNum1 and myNodeNum2 and str(nodeID) not in sentryIgnoreList:
                                 node_list.append({'id': nodeID, 'latitude': latitude, 'longitude': longitude, 'distance': distance})
-                                # calculate distance to node and report
                                 
                     except Exception as e:
                         pass
-                    
-            #sort by distance closest to lattitudeValue, longitudeValue
-            node_list.sort(key=lambda x: (x['latitude']-latitudeValue)**2 + (x['longitude']-longitudeValue)**2)
-
+            # sort by distance closest
+            node_list.sort(key=lambda x: x['distance'])
             # return the first 3 closest nodes by default
             return node_list[:returnCount]
         else:
@@ -409,7 +429,7 @@ def send_message(message, ch, nodeid=0, nodeInt=1):
 
         for m in message_list:
             if nodeid == 0:
-                #Send to channel
+                # Send to channel
                 logger.info(f"Device:{nodeInt} Channel:{ch} " + CustomFormatter.red + "Sending Multi-Chunk Message: " + CustomFormatter.white + m.replace('\n', ' '))
                 if nodeInt == 1:
                     interface1.sendText(text=m, channelIndex=ch)
@@ -580,22 +600,6 @@ async def retry_interface(nodeID=1):
     except Exception as e:
         logger.error(f"System: opening interface2: {e}")
 
-# this is a workaround because .localNode.getMetadata spits out a lot of debug info which cant be suppressed
-
-from contextlib import contextmanager
-import os
-import sys
-
-@contextmanager
-def suppress_stdout():
-    with open(os.devnull, "w") as devnull:
-        old_stdout = sys.stdout
-        sys.stdout = devnull
-        try:  
-            yield
-        finally:
-            sys.stdout = old_stdout
-
 async def watchdog():
     global retry_int1, retry_int2
     if sentry_enabled:
@@ -611,8 +615,10 @@ async def watchdog():
         #print(f"MeshBot System: watchdog running\r", end="")
         if interface1 is not None and not retry_int1:
             try:
-                with suppress_stdout():
+                # this is a workaround because .localNode.getMetadata spits out a lot of debug info which cant be suppressed
+                with contextlib.redirect_stdout(None):
                     interface1.localNode.getMetadata()
+                    print(f"System: if you see this upgrade python to >3.4")
                 #if "device_state_version:" not in meta:
             except Exception as e:
                 logger.error(f"System: communicating with interface1, trying to reconnect: {e}")
@@ -652,8 +658,9 @@ async def watchdog():
         if interface2_enabled:
             if interface2 is not None and not retry_int2:
                 try:
-                    with suppress_stdout():
+                    with contextlib.redirect_stdout(None):
                         interface2.localNode.getMetadata()
+                        print(f"System: if you see this upgrade python to >3.4")
                 except Exception as e:
                     logger.error(f"System: communicating with interface2, trying to reconnect: {e}")
                     retry_int2 = True
@@ -668,7 +675,7 @@ async def watchdog():
                                 enemySpotted2 += ", " + get_name_from_number(closest_nodes2[0]['id'], 'short', 2)
                                 enemySpotted2 += ", " + str(closest_nodes2[0]['id'])
                                 enemySpotted2 += ", " + decimal_to_hex(closest_nodes2[0]['id'])
-                                enemySpotted += f" at {closest_nodes1[0]['distance']}m"
+                                enemySpotted2 += f" at {closest_nodes2[0]['distance']}m"
                     except Exception as e:
                         pass
                     

pong_bot.py

@@ -33,7 +33,7 @@ def auto_response(message, snr, rssi, hop, message_from_id, channel_number, devi
     if len(cmds) > 0:
         # sort the commands by index value
         cmds = sorted(cmds, key=lambda k: k['index'])
-        logger.debug(f"System: Bot Detected: {cmds}")
+        logger.debug(f"System: Bot detected Commands:{cmds}")
         # run the first command after sorting
         bot_response = command_handler[cmds[0]['cmd']]()
 
@@ -198,7 +198,7 @@ def onReceive(packet, interface):
                             send_message(auto_response(message_string, snr, rssi, hop, message_from_id, channel_number, rxNode), channel_number, 0, rxNode)
                 else:
                     # message is not for bot to respond to
-                    # ignore the message but add it to the message history and repeat it if enabled
+                    # ignore the message but add it to the message history list
                     if zuluTime:
                         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                     else:
@@ -209,14 +209,19 @@ def onReceive(packet, interface):
                     else:
                         msg_history.pop(0)
                         msg_history.append((get_name_from_number(message_from_id, 'long', rxNode), message_string, channel_number, timestamp, rxNode))
-                    
-                    # check if repeater is enabled and the other interface is enabled
+
+                    # print the message to the log and sdout
+                    logger.info(f"Device:{rxNode} Channel:{channel_number} " + CustomFormatter.green + "Ignoring Message:" + CustomFormatter.white +\
+                                f" {message_string} " + CustomFormatter.purple + "From:" + CustomFormatter.white + f" {get_name_from_number(message_from_id)}")
+                    if log_messages_to_file:
+                        msgLogger.info(f"Device:{rxNode} Channel:{channel_number} | {get_name_from_number(message_from_id, 'long', rxNode)} | " + message_string.replace('\n', '-nl-'))
+
+                     # repeat the message on the other device
                     if repeater_enabled and interface2_enabled:         
-                        # repeat the message on the other device
+                        # wait a 700ms to avoid message collision from lora-ack.
+                        time.sleep(0.7)
                         rMsg = (f"{message_string} From:{get_name_from_number(message_from_id, 'short', rxNode)}")
                         # if channel found in the repeater list repeat the message
-                        # wait a 700ms to avoid message collision from lora-ack
-                        time.sleep(0.7)
                         if str(channel_number) in repeater_channels:
                             if rxNode == 1:
                                 logger.debug(f"Repeating message on Device2 Channel:{channel_number}")
@@ -224,11 +229,6 @@ def onReceive(packet, interface):
                             elif rxNode == 2:
                                 logger.debug(f"Repeating message on Device1 Channel:{channel_number}")
                                 send_message(rMsg, channel_number, 0, 1)
-                    else: 
-                         # nothing to do for us
-                        logger.info(f"Ignoring Device:{rxNode} Channel:{channel_number} " + CustomFormatter.green + "Message:" + CustomFormatter.white +\
-                                     f" {message_string} " + CustomFormatter.purple + "From:" + CustomFormatter.white + f" {get_name_from_number(message_from_id)}")
-                        msgLogger.info(f"Device:{rxNode} Channel:{channel_number} | {get_name_from_number(message_from_id, 'long', rxNode)} | {message_string}")
     except KeyError as e:
         logger.critical(f"System: Error processing packet: {e} Device:{rxNode}")
         print(packet) # print the packet for debugging