ignore

.gitignore

@@ -7,4 +7,7 @@ client.db
 client.log
 settings.log
 config.json
-default_config.log
+default_config.log
+client.db-shm
+client.db-wal
+client.db.bk

ui/control_ui.py

@@ -37,6 +37,10 @@ field_mapping, help_text = parse_ini_file(translation_file)
 def display_menu(current_menu, menu_path, selected_index, show_save_option, help_text):
     min_help_window_height = 6
     num_items = len(current_menu) + (1 if show_save_option else 0)
+        # Track visible range
+    global start_index
+    if 'start_index' not in globals():
+        start_index = [0]  # Initialize if not set
 
     # Determine the available height for the menu
     max_menu_height = curses.LINES 
@@ -90,7 +94,7 @@ def display_menu(current_menu, menu_path, selected_index, show_save_option, help
 
     menu_win.refresh()
     menu_pad.refresh(
-        0, 0,
+        start_index[-1], 0,
         menu_win.getbegyx()[0] + 3, menu_win.getbegyx()[1] + 4,
         menu_win.getbegyx()[0] + 3 + menu_win.getmaxyx()[0] - 5 - (2 if show_save_option else 0),
         menu_win.getbegyx()[1] + menu_win.getmaxyx()[1] - 8
@@ -241,36 +245,45 @@ def get_wrapped_help_text(help_text, transformed_path, selected_option, width, m
 
 
 def move_highlight(old_idx, new_idx, options, show_save_option, menu_win, menu_pad, help_win, help_text, menu_path, max_help_lines):
-
     if old_idx == new_idx:  # No-op
         return
 
     max_index = len(options) + (1 if show_save_option else 0) - 1
+    visible_height = menu_win.getmaxyx()[0] - 5 - (2 if show_save_option else 0)
 
-    if show_save_option and old_idx == max_index:  # Special case un-highlight "Save" option
+    # Adjust start_index only when moving out of visible range
+    if new_idx < start_index[-1]:  # Moving above the visible area
+        start_index[-1] = new_idx
+    elif new_idx >= start_index[-1] + visible_height:  # Moving below the visible area
+        start_index[-1] = new_idx - visible_height 
+
+    # Ensure start_index is within bounds
+    start_index[-1] = max(0, min(start_index[-1], max_index - visible_height + 1))
+
+    # Clear old selection
+    if show_save_option and old_idx == max_index:
         menu_win.chgat(menu_win.getmaxyx()[0] - 2, (width - len(save_option)) // 2, len(save_option), get_color("settings_save"))
     else:
         menu_pad.chgat(old_idx, 0, menu_pad.getmaxyx()[1], get_color("settings_sensitive") if options[old_idx] in sensitive_settings else get_color("settings_default"))
 
-    if show_save_option and new_idx == max_index:  # Special case highlight "Save" option
+    # Highlight new selection
+    if show_save_option and new_idx == max_index:
         menu_win.chgat(menu_win.getmaxyx()[0] - 2, (width - len(save_option)) // 2, len(save_option), get_color("settings_save", reverse=True))
     else:
         menu_pad.chgat(new_idx, 0, menu_pad.getmaxyx()[1], get_color("settings_sensitive", reverse=True) if options[new_idx] in sensitive_settings else get_color("settings_default", reverse=True))
 
     menu_win.refresh()
-
-    start_index = max(0, new_idx - (menu_win.getmaxyx()[0] - 5 - (2 if show_save_option else 0)) - (1 if show_save_option and new_idx == max_index else 0))
-    menu_pad.refresh(start_index, 0,
+    
+    # Refresh pad only if scrolling is needed
+    menu_pad.refresh(start_index[-1], 0,
                      menu_win.getbegyx()[0] + 3, menu_win.getbegyx()[1] + 4,
-                     menu_win.getbegyx()[0] + 3 + menu_win.getmaxyx()[0] - 5 - (2 if show_save_option else 0), 
+                     menu_win.getbegyx()[0] + 3 + visible_height, 
                      menu_win.getbegyx()[1] + menu_win.getmaxyx()[1] - 8)
 
-    # Transform menu path
+    # Update help window
     transformed_path = transform_menu_path(menu_path)
     selected_option = options[new_idx] if new_idx < len(options) else None
     help_y = menu_win.getbegyx()[0] + menu_win.getmaxyx()[0]
-
-    # Call helper function to update the help window
     help_win = update_help_window(help_win, help_text, transformed_path, selected_option, max_help_lines, width, help_y, menu_win.getbegyx()[1])
 
 
@@ -337,6 +350,7 @@ def settings_menu(stdscr, interface):
 
         elif key == curses.KEY_RIGHT or key == ord('\n'):
             need_redraw = True
+            start_index.append(0)
             menu_win.erase()
             help_win.erase()
 
@@ -372,6 +386,7 @@ def settings_menu(stdscr, interface):
                 filename = get_text_input("Enter a filename for the config file")
                 if not filename:
                     logging.info("Export aborted: No filename provided.")
+                    start_index.pop()
                     continue  # Go back to the menu
                 if not filename.lower().endswith(".yaml"):
                     filename += ".yaml"
@@ -384,14 +399,14 @@ def settings_menu(stdscr, interface):
                         overwrite = get_list_input(f"{filename} already exists. Overwrite?", None, ["Yes", "No"])
                         if overwrite == "No":
                             logging.info("Export cancelled: User chose not to overwrite.")
+                            start_index.pop()
                             continue  # Return to menu
-
                     os.makedirs(os.path.dirname(yaml_file_path), exist_ok=True)
                     with open(yaml_file_path, "w", encoding="utf-8") as file:
                         file.write(config_text)
                     logging.info(f"Config file saved to {yaml_file_path}")
                     dialog(stdscr, "Config File Saved:", yaml_file_path)
-                    
+                    start_index.pop()
                     continue
                 except PermissionError:
                     logging.error(f"Permission denied: Unable to write to {yaml_file_path}")
@@ -399,8 +414,9 @@ def settings_menu(stdscr, interface):
                     logging.error(f"OS error while saving config: {e}")
                 except Exception as e:
                     logging.error(f"Unexpected error: {e}")
+                start_index.pop()
                 continue
-
+                
             elif selected_option == "Load Config File":
                 folder_path = os.path.join(app_directory, config_folder)
 
@@ -422,6 +438,7 @@ def settings_menu(stdscr, interface):
                     overwrite = get_list_input(f"Are you sure you want to load {filename}?", None, ["Yes", "No"])
                     if overwrite == "Yes":
                         config_import(interface, file_path)
+                start_index.pop()
                 continue
 
             elif selected_option == "Config URL":
@@ -433,6 +450,7 @@ def settings_menu(stdscr, interface):
                     if overwrite == "Yes":
                         interface.localNode.setURL(new_value)
                         logging.info(f"New Config URL sent to node")
+                start_index.pop()
                 continue
 
             elif selected_option == "Reboot":
@@ -440,6 +458,7 @@ def settings_menu(stdscr, interface):
                 if confirmation == "Yes":
                     interface.localNode.reboot()
                     logging.info(f"Node Reboot Requested by menu")
+                start_index.pop()
                 continue
 
             elif selected_option == "Reset Node DB":
@@ -447,6 +466,7 @@ def settings_menu(stdscr, interface):
                 if confirmation == "Yes":
                     interface.localNode.resetNodeDb()
                     logging.info(f"Node DB Reset Requested by menu")
+                start_index.pop()
                 continue
 
             elif selected_option == "Shutdown":
@@ -454,6 +474,7 @@ def settings_menu(stdscr, interface):
                 if confirmation == "Yes":
                     interface.localNode.shutdown()
                     logging.info(f"Node Shutdown Requested by menu")
+                start_index.pop()
                 continue
 
             elif selected_option == "Factory Reset":
@@ -461,6 +482,7 @@ def settings_menu(stdscr, interface):
                 if confirmation == "Yes":
                     interface.localNode.factoryReset()
                     logging.info(f"Factory Reset Requested by menu")
+                start_index.pop()
                 continue
 
             elif selected_option == "App Settings":
@@ -495,6 +517,8 @@ def settings_menu(stdscr, interface):
                     for option, (field, value) in current_menu.items():
                         modified_settings[option] = value
 
+                    start_index.pop()
+
                 elif selected_option in ['latitude', 'longitude', 'altitude']:
                     new_value = get_text_input(f"{human_readable_name} is currently: {current_value}")
                     new_value = current_value if new_value is None else new_value
@@ -504,37 +528,47 @@ def settings_menu(stdscr, interface):
                         if option in current_menu:
                             modified_settings[option] = current_menu[option][1]
 
+                    start_index.pop()
+
                 elif selected_option == "admin_key":
                     new_values = get_admin_key_input(current_value)
                     new_value = current_value if new_values is None else [base64.b64decode(key) for key in new_values]
+                    start_index.pop()
 
                 elif field.type == 8:  # Handle boolean type
                     new_value = get_list_input(human_readable_name, str(current_value),  ["True", "False"])
                     new_value = new_value == "True" or new_value is True
+                    start_index.pop()
 
                 elif field.label == field.LABEL_REPEATED:  # Handle repeated field - Not currently used
                     new_value = get_repeated_input(current_value)
                     new_value = current_value if new_value is None else new_value.split(", ")
+                    start_index.pop()
 
                 elif field.enum_type:  # Enum field
                     enum_options = {v.name: v.number for v in field.enum_type.values}
                     new_value_name = get_list_input(human_readable_name, current_value, list(enum_options.keys()))
                     new_value = enum_options.get(new_value_name, current_value)
+                    start_index.pop()
 
                 elif field.type == 7: # Field type 7 corresponds to FIXED32
                     new_value = get_fixed32_input(current_value)
+                    start_index.pop()
 
                 elif field.type == 13: # Field type 13 corresponds to UINT32
                     new_value = get_text_input(f"{human_readable_name} is currently: {current_value}")
                     new_value = current_value if new_value is None else int(new_value)
+                    start_index.pop()
 
                 elif field.type == 2: # Field type 13 corresponds to INT64
                     new_value = get_text_input(f"{human_readable_name} is currently: {current_value}")
                     new_value = current_value if new_value is None else float(new_value)
+                    start_index.pop()
 
                 else:  # Handle other field types
                     new_value = get_text_input(f"{human_readable_name} is currently: {current_value}")
                     new_value = current_value if new_value is None else new_value
+                    start_index.pop()
                 
                 for key in menu_path[3:]:  # Skip "Main Menu"
                     modified_settings = modified_settings.setdefault(key, {})
@@ -554,6 +588,7 @@ def settings_menu(stdscr, interface):
                 menu_index.append(selected_index)
                 selected_index = 0
 
+
         elif key == curses.KEY_LEFT:
             need_redraw = True
 
@@ -576,7 +611,8 @@ def settings_menu(stdscr, interface):
                 for step in menu_path[1:]:
                     current_menu = current_menu.get(step, {})
                 selected_index = menu_index.pop()
-
+                start_index.pop()
+                
         elif key == 27:  # Escape key
             menu_win.erase()
             menu_win.refresh()
@@ -599,5 +635,4 @@ def set_region(interface):
     new_region_number = region_name_to_number.get(new_region_name, 0)  # Default to 0 if not found
 
     node.localConfig.lora.region = new_region_number
-    node.writeConfig("lora")
-    
+    node.writeConfig("lora")

ui/curses_ui.py

@@ -319,7 +319,10 @@ def draw_channel_list():
         if isinstance(channel, int):
             if is_chat_archived(channel):
                 continue
-            channel = get_name_from_database(channel, type='long')
+            channel_name = get_name_from_database(channel, type='long')
+            if channel_name is None:
+                continue
+            channel = channel_name
 
         # Determine whether to add the notification
         notification = " " + config.notification_symbol if idx in globals.notifications else ""

utilities/db_handler.py

@@ -1,17 +1,42 @@
 import sqlite3
 import time
 import logging
+import re
 from datetime import datetime
 
 from utilities.utils import decimal_to_hex
 import ui.default_config as config
 import globals
 
+
+def get_db_connection():
+    """Get a SQLite connection with optimized PRAGMA settings."""
+    db_connection = sqlite3.connect(config.db_file_path, check_same_thread=False)
+    db_cursor = db_connection.cursor()
+
+    # Check if journal_mode is already set to WAL
+    db_cursor.execute("PRAGMA journal_mode;")
+    current_journal_mode = db_cursor.fetchone()[0]
+
+    if current_journal_mode != "wal":
+        db_cursor.execute("PRAGMA journal_mode=WAL;")
+
+    # Apply remaining PRAGMA settings (these are fine to execute every time)
+    db_cursor.executescript("""
+        PRAGMA synchronous=NORMAL;
+        PRAGMA cache_size=-64000;
+        PRAGMA temp_store=MEMORY;
+        PRAGMA foreign_keys=ON;
+    """)
+
+    return db_connection
+
+
 def get_table_name(channel):
-    # Construct the table name
-    table_name = f"{str(globals.myNodeNum)}_{channel}_messages"
-    quoted_table_name = f'"{table_name}"'  # Quote the table name becuase we begin with numerics and contain spaces
-    return quoted_table_name
+    """Returns a properly formatted and safe table name."""
+    safe_channel = re.sub(r'[^a-zA-Z0-9_]', '', str(channel))
+    table_name = f"{globals.myNodeNum}_{safe_channel}_messages"
+    return f'"{table_name}"'
 
 
 def save_message_to_db(channel, user_id, message_text):
@@ -27,7 +52,7 @@ def save_message_to_db(channel, user_id, message_text):
         '''
         ensure_table_exists(quoted_table_name, schema)
 
-        with sqlite3.connect(config.db_file_path) as db_connection:
+        with get_db_connection() as db_connection:
             db_cursor = db_connection.cursor()
             timestamp = int(time.time())
 
@@ -49,7 +74,7 @@ def save_message_to_db(channel, user_id, message_text):
 
 def update_ack_nak(channel, timestamp, message, ack):
     try:
-        with sqlite3.connect(config.db_file_path) as db_connection:
+        with get_db_connection() as db_connection:
             db_cursor = db_connection.cursor()
             update_query = f"""
                 UPDATE {get_table_name(channel)}
@@ -72,7 +97,7 @@ def update_ack_nak(channel, timestamp, message, ack):
 def load_messages_from_db():
     """Load messages from the database for all channels and update globals.all_messages and globals.channel_list."""
     try:
-        with sqlite3.connect(config.db_file_path) as db_connection:
+        with get_db_connection() as db_connection:
             db_cursor = db_connection.cursor()
 
             query = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ?"
@@ -196,7 +221,7 @@ def update_node_info_in_db(user_id, long_name=None, short_name=None, hw_model=No
     try:
         ensure_node_table_exists()  # Ensure the table exists before any operation
 
-        with sqlite3.connect(config.db_file_path) as db_connection:
+        with get_db_connection() as db_connection:
             db_cursor = db_connection.cursor()
             table_name = f'"{globals.myNodeNum}_nodedb"'  # Quote in case of numeric names
 
@@ -223,16 +248,16 @@ def update_node_info_in_db(user_id, long_name=None, short_name=None, hw_model=No
 
             # Upsert logic
             upsert_query = f'''
-                INSERT INTO {table_name} (user_id, long_name, short_name, hw_model, is_licensed, role, public_key, chat_archived)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-                ON CONFLICT(user_id) DO UPDATE SET
-                    long_name = excluded.long_name,
-                    short_name = excluded.short_name,
-                    hw_model = excluded.hw_model,
-                    is_licensed = excluded.is_licensed,
-                    role = excluded.role,
-                    public_key = excluded.public_key,
-                    chat_archived = excluded.chat_archived
+            INSERT INTO {table_name} (user_id, long_name, short_name, hw_model, is_licensed, role, public_key, chat_archived)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(user_id) DO UPDATE SET
+                long_name = excluded.long_name,
+                short_name = excluded.short_name,
+                hw_model = excluded.hw_model,
+                is_licensed = excluded.is_licensed,
+                role = excluded.role,
+                public_key = excluded.public_key,
+                chat_archived = COALESCE(excluded.chat_archived, chat_archived);
             '''
             db_cursor.execute(upsert_query, (user_id, long_name, short_name, hw_model, is_licensed, role, public_key, chat_archived))
             db_connection.commit()
@@ -262,7 +287,7 @@ def ensure_node_table_exists():
 def ensure_table_exists(table_name, schema):
     """Ensure the given table exists in the database."""
     try:
-        with sqlite3.connect(config.db_file_path) as db_connection:
+        with get_db_connection() as db_connection:
             db_cursor = db_connection.cursor()
             create_table_query = f"CREATE TABLE IF NOT EXISTS {table_name} ({schema})"
             db_cursor.execute(create_table_query)
@@ -273,31 +298,32 @@ def ensure_table_exists(table_name, schema):
         logging.error(f"Unexpected error in ensure_table_exists({table_name}): {e}")
 
 
+name_cache = {}
+
 def get_name_from_database(user_id, type="long"):
-    """
-    Retrieve a user's name (long or short) from the node database.
-    
-    :param user_id: The user ID to look up.
-    :param type: "long" for long name, "short" for short name.
-    :return: The retrieved name or the hex of the user id
-    """
+    """Retrieve a user's name from the node database with caching."""
+    # Check if we already cached both long and short names
+    if user_id in name_cache and type in name_cache[user_id]:
+        return name_cache[user_id][type]
+
     try:
-        with sqlite3.connect(config.db_file_path) as db_connection:
+        with get_db_connection() as db_connection:
             db_cursor = db_connection.cursor()
-
-            # Construct table name
             table_name = f"{str(globals.myNodeNum)}_nodedb"
-            nodeinfo_table = f'"{table_name}"'  # Quote table name for safety
-            
-            # Determine the correct column to fetch
-            column_name = "long_name" if type == "long" else "short_name"
+            nodeinfo_table = f'"{table_name}"'
 
-            # Query the database
-            query = f"SELECT {column_name} FROM {nodeinfo_table} WHERE user_id = ?"
-            db_cursor.execute(query, (user_id,))
+            # Fetch both long and short names in one query
+            db_cursor.execute(f"SELECT long_name, short_name FROM {nodeinfo_table} WHERE user_id = ?", (user_id,))
             result = db_cursor.fetchone()
 
-            return result[0] if result else decimal_to_hex(user_id)
+            if result:
+                long_name, short_name = result or ("Unknown", "Unknown")  # Handle empty result
+                name_cache[user_id] = {"long": long_name, "short": short_name}
+                return name_cache[user_id][type]
+
+            # If no result, store a fallback value in the cache to avoid future DB queries
+            name_cache[user_id] = {"long": decimal_to_hex(user_id), "short": decimal_to_hex(user_id)}
+            return name_cache[user_id][type]
 
     except sqlite3.Error as e:
         logging.error(f"SQLite error in get_name_from_database: {e}")
@@ -306,10 +332,12 @@ def get_name_from_database(user_id, type="long"):
     except Exception as e:
         logging.error(f"Unexpected error in get_name_from_database: {e}")
         return "Unknown"
+    
 
 def is_chat_archived(user_id):
+    """Check if a chat is archived, returning 0 (False) if not found."""
     try:
-        with sqlite3.connect(config.db_file_path) as db_connection:
+        with get_db_connection() as db_connection:
             db_cursor = db_connection.cursor()
             table_name = f"{str(globals.myNodeNum)}_nodedb"
             nodeinfo_table = f'"{table_name}"'
@@ -321,9 +349,8 @@ def is_chat_archived(user_id):
 
     except sqlite3.Error as e:
         logging.error(f"SQLite error in is_chat_archived: {e}")
-        return "Unknown"
+        return 0
 
     except Exception as e:
         logging.error(f"Unexpected error in is_chat_archived: {e}")
-        return "Unknown"
-
+        return 0

utilities/input_handlers.py

@@ -391,18 +391,38 @@ def get_list_input(prompt, current_option, list_options):
 
 
 def move_highlight(old_idx, new_idx, options, list_win, list_pad):
+
+    global scroll_offset
+    if 'scroll_offset' not in globals():
+        scroll_offset = 0  # Initialize if not set
+
     if old_idx == new_idx:
-        return # no-op
+        return  # No-op
 
+    max_index = len(options) - 1
+    visible_height = list_win.getmaxyx()[0] - 5
+
+    # Adjust scroll_offset only when moving out of visible range
+    if new_idx < scroll_offset:  # Moving above the visible area
+        scroll_offset = new_idx
+    elif new_idx >= scroll_offset + visible_height:  # Moving below the visible area
+        scroll_offset = new_idx - visible_height
+
+    # Ensure scroll_offset is within bounds
+    scroll_offset = max(0, min(scroll_offset, max_index - visible_height + 1))
+
+    # Clear old highlight
     list_pad.chgat(old_idx, 0, list_pad.getmaxyx()[1], get_color("settings_default"))
-    list_pad.chgat(new_idx, 0, list_pad.getmaxyx()[1], get_color("settings_default", reverse = True))
+
+    # Highlight new selection
+    list_pad.chgat(new_idx, 0, list_pad.getmaxyx()[1], get_color("settings_default", reverse=True))
 
     list_win.refresh()
-
-    start_index = max(0, new_idx - (list_win.getmaxyx()[0] - 5))
-
-    list_win.refresh()
-    list_pad.refresh(start_index, 0,
+    
+    # Refresh pad only if scrolling is needed
+    list_pad.refresh(scroll_offset, 0,
                      list_win.getbegyx()[0] + 3, list_win.getbegyx()[1] + 4,
-                     list_win.getbegyx()[0] + list_win.getmaxyx()[0] - 2, list_win.getbegyx()[1] + 4 + list_win.getmaxyx()[1] - 4)
-    
+                     list_win.getbegyx()[0] + 3 + visible_height, 
+                     list_win.getbegyx()[1] + list_win.getmaxyx()[1] - 4)
+
+    return scroll_offset  # Return updated scroll_offset to be stored externally