diff --git a/contact/ui/contact_ui.py b/contact/ui/contact_ui.py index f010dcb..cdea4f9 100644 --- a/contact/ui/contact_ui.py +++ b/contact/ui/contact_ui.py @@ -10,7 +10,7 @@ from contact.settings import settings_menu from contact.message_handlers.tx_handler import send_message, send_traceroute from contact.utilities.utils import parse_protobuf from contact.ui.colors import get_color -from contact.utilities.db_handler import get_name_from_database, update_node_info_in_db, is_chat_archived +from contact.utilities.db_handler import get_name_from_database, update_node_info_in_db, is_chat_archived, load_older_messages from contact.utilities.input_handlers import get_list_input from contact.utilities.i18n import t from contact.utilities.emoji_utils import normalize_message_text @@ -1162,18 +1162,13 @@ def draw_messages_window(scroll_to_bottom: bool = False) -> None: channel = ui_state.channel_list[ui_state.selected_channel] + msg_line_count = 0 if channel in ui_state.all_messages: messages = ui_state.all_messages[channel] - - msg_line_count = 0 - - row = 0 + rendered_lines = [] for prefix, message in messages: full_message = normalize_message_text(f"{prefix}{message}") wrapped_lines = wrap_text(full_message, messages_win.getmaxyx()[1] - 2) - msg_line_count += len(wrapped_lines) - messages_pad.resize(msg_line_count, messages_win.getmaxyx()[1]) - for line in wrapped_lines: if prefix.startswith("--"): color = get_color("timestamps") @@ -1182,8 +1177,12 @@ def draw_messages_window(scroll_to_bottom: bool = False) -> None: else: color = get_color("rx_messages") - messages_pad.addstr(row, 1, line, color) - row += 1 + rendered_lines.append((line, color)) + + msg_line_count = len(rendered_lines) + messages_pad.resize(max(1, msg_line_count), messages_win.getmaxyx()[1]) + for row, (line, color) in enumerate(rendered_lines): + messages_pad.addstr(row, 1, line, color) paint_frame(messages_win, selected=(ui_state.current_window == 1)) @@ -1283,6 +1282,17 @@ def scroll_channels(direction: int) -> None: def scroll_messages(direction: int) -> None: """Scroll through the messages in the current channel by a given direction.""" + if direction < 0 and ui_state.selected_message == 0 and ui_state.channel_list: + channel = ui_state.channel_list[ui_state.selected_channel] + previous_height = messages_pad.getmaxyx()[0] + if load_older_messages(channel): + draw_messages_window() + added_height = max(0, messages_pad.getmaxyx()[0] - previous_height) + ui_state.selected_message = added_height + ui_state.start_index[1] = added_height + refresh_pad(1) + return + ui_state.selected_message += direction msg_line_count = messages_pad.getmaxyx()[0] diff --git a/contact/ui/ui_state.py b/contact/ui/ui_state.py index 7d1e3db..394f691 100644 --- a/contact/ui/ui_state.py +++ b/contact/ui/ui_state.py @@ -40,6 +40,8 @@ class ChatUIState: redraw_packetlog: bool = False redraw_full_ui: bool = False scroll_messages_to_bottom: bool = False + oldest_message_rowid: Dict[Union[str, int], int] = field(default_factory=dict) + has_older_messages: Dict[Union[str, int], bool] = field(default_factory=dict) @dataclass diff --git a/contact/utilities/db_handler.py b/contact/utilities/db_handler.py index d10fb26..87f5517 100644 --- a/contact/utilities/db_handler.py +++ b/contact/utilities/db_handler.py @@ -11,6 +11,9 @@ import contact.ui.default_config as config from contact.utilities.singleton import ui_state, interface_state +MESSAGE_PAGE_SIZE = 250 + + def get_table_name(channel: str) -> str: # Construct the table name table_name = f"{str(interface_state.myNodeNum)}_{channel}_messages" @@ -75,7 +78,54 @@ def update_ack_nak(channel: str, timestamp: int, message: str, ack: str) -> None logging.error(f"Unexpected error in update_ack_nak: {e}") -def load_messages_from_db() -> None: +def _format_db_messages(db_messages, node_names): + """Format database rows, adding one timestamp separator per hour.""" + hourly_messages = {} + for _rowid, user_id, message, timestamp, ack_type in db_messages: + if user_id is None or message is None or timestamp is None: + logging.warning(f"Skipping row with NULL required field(s): {(user_id, message, timestamp, ack_type)}") + continue + + hour = datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:00") + ack_str = config.ack_unknown_str + if ack_type == "Implicit": + ack_str = config.ack_implicit_str + elif ack_type == "Ack": + ack_str = config.ack_str + elif ack_type == "Nak": + ack_str = config.nak_str + + ts_str = datetime.fromtimestamp(timestamp).strftime("[%H:%M:%S]") + sanitized_message = message.replace("\x00", "") + if user_id == str(interface_state.myNodeNum): + formatted_message = (f"{ts_str} {config.sent_message_prefix}{ack_str}: ", sanitized_message) + else: + try: + fallback_name = decimal_to_hex(int(user_id)) + except (TypeError, ValueError): + fallback_name = str(user_id) + formatted_message = ( + f"{ts_str} {config.message_prefix} {node_names.get(str(user_id), fallback_name)}: ", + sanitized_message, + ) + hourly_messages.setdefault(hour, []).append(formatted_message) + + formatted = [] + for hour, messages in sorted(hourly_messages.items()): + formatted.append((f"-- {hour} --", "")) + formatted.extend(messages) + return formatted + + +def _load_node_names(db_cursor): + table_name = f'"{interface_state.myNodeNum}_nodedb"' + try: + return {str(row[0]): row[1] for row in db_cursor.execute(f"SELECT user_id, short_name FROM {table_name}")} + except sqlite3.Error: + return {} + + +def load_messages_from_db(page_size: int = MESSAGE_PAGE_SIZE) -> None: """Load messages from the database for all channels and update ui_state.all_messages and ui_state.channel_list.""" try: with sqlite3.connect(config.db_file_path, timeout=10.0) as db_connection: @@ -85,6 +135,7 @@ def load_messages_from_db() -> None: query = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ?" db_cursor.execute(query, (f"{str(interface_state.myNodeNum)}_%_messages",)) tables = [row[0] for row in db_cursor.fetchall()] + node_names = _load_node_names(db_cursor) # Iterate through each table and fetch its messages for table_name in tables: @@ -97,15 +148,22 @@ def load_messages_from_db() -> None: db_cursor.execute(update_table_query) db_connection.commit() - query = f"SELECT user_id, message_text, timestamp, ack_type FROM {quoted_table_name}" + query = f""" + SELECT rowid, user_id, message_text, timestamp, ack_type + FROM {quoted_table_name} + ORDER BY rowid DESC LIMIT ? + """ try: # Fetch all messages from the table - db_cursor.execute(query) - db_messages = [(row[0], row[1], row[2], row[3]) for row in db_cursor.fetchall()] # Save as tuples + db_cursor.execute(query, (page_size + 1,)) + rows = db_cursor.fetchall() + has_older = len(rows) > page_size + db_messages = list(reversed(rows[:page_size])) # Extract the channel name from the table name - channel = table_name.split("_")[1] + prefix = f"{interface_state.myNodeNum}_" + channel = table_name[len(prefix) : -len("_messages")] # Convert the channel to an integer if it's numeric, otherwise keep it as a string (nodenum vs channel name) channel = int(channel) if channel.isdigit() else channel @@ -118,49 +176,10 @@ def load_messages_from_db() -> None: if channel not in ui_state.all_messages: ui_state.all_messages[channel] = [] - # Add messages to ui_state.all_messages grouped by hourly timestamp - hourly_messages = {} - for row in db_messages: - user_id, message, timestamp, ack_type = row - - # Only ack_type is allowed to be None - if user_id is None or message is None or timestamp is None: - logging.warning(f"Skipping row with NULL required field(s): {row}") - continue - - hour = datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:00") - if hour not in hourly_messages: - hourly_messages[hour] = [] - - ack_str = config.ack_unknown_str - if ack_type == "Implicit": - ack_str = config.ack_implicit_str - elif ack_type == "Ack": - ack_str = config.ack_str - elif ack_type == "Nak": - ack_str = config.nak_str - - ts_str = datetime.fromtimestamp(timestamp).strftime("[%H:%M:%S]") - - if user_id == str(interface_state.myNodeNum): - sanitized_message = message.replace("\x00", "") - formatted_message = ( - f"{ts_str} {config.sent_message_prefix}{ack_str}: ", - sanitized_message, - ) - else: - sanitized_message = message.replace("\x00", "") - formatted_message = ( - f"{ts_str} {config.message_prefix} {get_name_from_database(int(user_id), 'short')}: ", - sanitized_message, - ) - - hourly_messages[hour].append(formatted_message) - - # Flatten the hourly messages into ui_state.all_messages[channel] - for hour, messages in sorted(hourly_messages.items()): - ui_state.all_messages[channel].append((f"-- {hour} --", "")) - ui_state.all_messages[channel].extend(messages) + ui_state.all_messages[channel].extend(_format_db_messages(db_messages, node_names)) + if db_messages: + ui_state.oldest_message_rowid[channel] = db_messages[0][0] + ui_state.has_older_messages[channel] = has_older except sqlite3.Error as e: logging.error(f"SQLite error while loading messages from table '{table_name}': {e}") @@ -169,6 +188,43 @@ def load_messages_from_db() -> None: logging.error(f"SQLite error in load_messages_from_db: {e}") +def load_older_messages(channel, page_size: int = MESSAGE_PAGE_SIZE) -> int: + """Prepend one older page for a channel and return the number of messages loaded.""" + before_rowid = ui_state.oldest_message_rowid.get(channel) + if before_rowid is None or not ui_state.has_older_messages.get(channel, False): + return 0 + + try: + with sqlite3.connect(config.db_file_path, timeout=10.0) as db_connection: + db_connection.execute("PRAGMA busy_timeout=10000") + db_cursor = db_connection.cursor() + query = f""" + SELECT rowid, user_id, message_text, timestamp, ack_type + FROM {get_table_name(channel)} + WHERE rowid < ? ORDER BY rowid DESC LIMIT ? + """ + db_cursor.execute(query, (before_rowid, page_size + 1)) + rows = db_cursor.fetchall() + has_older = len(rows) > page_size + db_messages = list(reversed(rows[:page_size])) + if not db_messages: + ui_state.has_older_messages[channel] = False + return 0 + + older = _format_db_messages(db_messages, _load_node_names(db_cursor)) + current = ui_state.all_messages.setdefault(channel, []) + last_older_header = next((prefix for prefix, _ in reversed(older) if prefix.startswith("--")), None) + if current and current[0][0] == last_older_header: + current.pop(0) + ui_state.all_messages[channel] = older + current + ui_state.oldest_message_rowid[channel] = db_messages[0][0] + ui_state.has_older_messages[channel] = has_older + return len(db_messages) + except sqlite3.Error as e: + logging.error(f"SQLite error loading older messages for channel '{channel}': {e}") + return 0 + + def init_nodedb() -> None: """Initialize the node database and update it with nodes from the interface.""" diff --git a/tests/test_contact_ui.py b/tests/test_contact_ui.py index e0ea42c..0cb0387 100644 --- a/tests/test_contact_ui.py +++ b/tests/test_contact_ui.py @@ -70,6 +70,25 @@ class ContactUiTests(unittest.TestCase): self.assertFalse(ui_state.redraw_channels) self.assertFalse(ui_state.redraw_messages) + def test_draw_messages_resizes_pad_once(self) -> None: + ui_state.channel_list = ["Primary"] + ui_state.all_messages = {"Primary": [("[10:00] RX: ", "one"), ("[10:01] RX: ", "two")]} + contact_ui.messages_pad = mock.Mock() + contact_ui.messages_win = mock.Mock() + contact_ui.messages_win.getmaxyx.return_value = (10, 40) + contact_ui.packetlog_win = mock.Mock() + contact_ui.packetlog_win.getmaxyx.return_value = (1, 40) + contact_ui.messages_win.getbegyx.return_value = (0, 0) + + with mock.patch.object(contact_ui, "paint_frame"): + with mock.patch.object(contact_ui, "get_color", return_value=0): + with mock.patch.object(contact_ui, "refresh_pad"): + with mock.patch.object(contact_ui, "draw_packetlog_win"): + with mock.patch.object(contact_ui, "draw_window_arrows"): + contact_ui.draw_messages_window() + + contact_ui.messages_pad.resize.assert_called_once_with(2, 40) + def test_refresh_node_selection_reserves_scroll_arrow_column(self) -> None: ui_state.node_list = [101, 202] ui_state.selected_node = 1 diff --git a/tests/test_db_handler.py b/tests/test_db_handler.py index d7ba662..4e79a9a 100644 --- a/tests/test_db_handler.py +++ b/tests/test_db_handler.py @@ -112,6 +112,54 @@ class DbHandlerTests(unittest.TestCase): self.assertTrue(any("RM:" in prefix for prefix, _ in messages)) self.assertEqual(ui_state.all_messages[789][-1][1], "hidden") + def test_message_history_is_loaded_in_bounded_pages(self) -> None: + db_handler.update_node_info_in_db(456, long_name="Remote Node", short_name="RM") + db_handler.ensure_table_exists( + '"123_Primary_messages"', + "user_id TEXT, message_text TEXT, timestamp INTEGER, ack_type TEXT", + ) + with sqlite3.connect(config.db_file_path) as conn: + conn.executemany( + 'INSERT INTO "123_Primary_messages" VALUES (?, ?, ?, ?)', + [("456", f"message-{i}", 1700000000 + i, None) for i in range(7)], + ) + + db_handler.load_messages_from_db(page_size=3) + + loaded_text = [message for _, message in ui_state.all_messages["Primary"] if message] + self.assertEqual(loaded_text, ["message-4", "message-5", "message-6"]) + self.assertTrue(ui_state.has_older_messages["Primary"]) + + self.assertEqual(db_handler.load_older_messages("Primary", page_size=3), 3) + loaded_text = [message for _, message in ui_state.all_messages["Primary"] if message] + self.assertEqual( + loaded_text, + ["message-1", "message-2", "message-3", "message-4", "message-5", "message-6"], + ) + self.assertTrue(ui_state.has_older_messages["Primary"]) + + self.assertEqual(db_handler.load_older_messages("Primary", page_size=3), 1) + loaded_text = [message for _, message in ui_state.all_messages["Primary"] if message] + self.assertEqual(loaded_text, [f"message-{i}" for i in range(7)]) + self.assertFalse(ui_state.has_older_messages["Primary"]) + + def test_message_table_channel_name_may_contain_underscores(self) -> None: + db_handler.ensure_node_table_exists() + db_handler.ensure_table_exists( + '"123_My_Channel_messages"', + "user_id TEXT, message_text TEXT, timestamp INTEGER, ack_type TEXT", + ) + with sqlite3.connect(config.db_file_path) as conn: + conn.execute( + 'INSERT INTO "123_My_Channel_messages" VALUES (?, ?, ?, ?)', + ("123", "hello", 1700000000, None), + ) + + db_handler.load_messages_from_db() + + self.assertIn("My_Channel", ui_state.channel_list) + self.assertEqual(ui_state.all_messages["My_Channel"][-1][1], "hello") + def test_init_nodedb_inserts_nodes_from_interface(self) -> None: interface_state.interface = build_demo_interface() interface_state.myNodeNum = DEMO_LOCAL_NODE_NUM