adam/contact
1
0
Fork 0
mirror of https://github.com/pdxlocations/contact.git synced 2026-07-28 12:32:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Slowness with busy channels

Browse Source 
Fixes #279
This commit is contained in:
pdxlocations
2026-07-18 21:24:32 -07:00
parent 69cb568d2c
commit 52f42591d2
5 changed files with 193 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contact/ui/contact_ui.py
+20 -10
View File
@@ -10,7 +10,7 @@ from contact.settings import settings_menu
from contact.message_handlers.tx_handler import send_message, send_traceroute
from contact.utilities.utils import parse_protobuf
from contact.ui.colors import get_color
from contact.utilities.db_handler import get_name_from_database, update_node_info_in_db, is_chat_archived
from contact.utilities.db_handler import get_name_from_database, update_node_info_in_db, is_chat_archived, load_older_messages
from contact.utilities.input_handlers import get_list_input
from contact.utilities.i18n import t
from contact.utilities.emoji_utils import normalize_message_text
@@ -1162,18 +1162,13 @@ def draw_messages_window(scroll_to_bottom: bool = False) -> None:
channel = ui_state.channel_list[ui_state.selected_channel]
msg_line_count = 0
if channel in ui_state.all_messages:
messages = ui_state.all_messages[channel]
msg_line_count = 0
row = 0
rendered_lines = []
for prefix, message in messages:
full_message = normalize_message_text(f"{prefix}{message}")
wrapped_lines = wrap_text(full_message, messages_win.getmaxyx()[1] - 2)
msg_line_count += len(wrapped_lines)
messages_pad.resize(msg_line_count, messages_win.getmaxyx()[1])
for line in wrapped_lines:
if prefix.startswith("--"):
color = get_color("timestamps")
@@ -1182,8 +1177,12 @@ def draw_messages_window(scroll_to_bottom: bool = False) -> None:
else:
color = get_color("rx_messages")
messages_pad.addstr(row, 1, line, color)
row += 1
rendered_lines.append((line, color))
msg_line_count = len(rendered_lines)
messages_pad.resize(max(1, msg_line_count), messages_win.getmaxyx()[1])
for row, (line, color) in enumerate(rendered_lines):
messages_pad.addstr(row, 1, line, color)
paint_frame(messages_win, selected=(ui_state.current_window == 1))
@@ -1283,6 +1282,17 @@ def scroll_channels(direction: int) -> None:
def scroll_messages(direction: int) -> None:
"""Scroll through the messages in the current channel by a given direction."""
if direction < 0 and ui_state.selected_message == 0 and ui_state.channel_list:
channel = ui_state.channel_list[ui_state.selected_channel]
previous_height = messages_pad.getmaxyx()[0]
if load_older_messages(channel):
draw_messages_window()
added_height = max(0, messages_pad.getmaxyx()[0] - previous_height)
ui_state.selected_message = added_height
ui_state.start_index[1] = added_height
refresh_pad(1)
return
ui_state.selected_message += direction
msg_line_count = messages_pad.getmaxyx()[0]
contact/ui/ui_state.py
+2
View File
@@ -40,6 +40,8 @@ class ChatUIState:
redraw_packetlog: bool = False
redraw_full_ui: bool = False
scroll_messages_to_bottom: bool = False
oldest_message_rowid: Dict[Union[str, int], int] = field(default_factory=dict)
has_older_messages: Dict[Union[str, int], bool] = field(default_factory=dict)
@dataclass
contact/utilities/db_handler.py
+104 -48
View File
@@ -11,6 +11,9 @@ import contact.ui.default_config as config
from contact.utilities.singleton import ui_state, interface_state
MESSAGE_PAGE_SIZE = 250
def get_table_name(channel: str) -> str:
# Construct the table name
table_name = f"{str(interface_state.myNodeNum)}_{channel}_messages"
@@ -75,7 +78,54 @@ def update_ack_nak(channel: str, timestamp: int, message: str, ack: str) -> None
logging.error(f"Unexpected error in update_ack_nak: {e}")
def load_messages_from_db() -> None:
def _format_db_messages(db_messages, node_names):
"""Format database rows, adding one timestamp separator per hour."""
hourly_messages = {}
for _rowid, user_id, message, timestamp, ack_type in db_messages:
if user_id is None or message is None or timestamp is None:
logging.warning(f"Skipping row with NULL required field(s): {(user_id, message, timestamp, ack_type)}")
continue
hour = datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:00")
ack_str = config.ack_unknown_str
if ack_type == "Implicit":
ack_str = config.ack_implicit_str
elif ack_type == "Ack":
ack_str = config.ack_str
elif ack_type == "Nak":
ack_str = config.nak_str
ts_str = datetime.fromtimestamp(timestamp).strftime("[%H:%M:%S]")
sanitized_message = message.replace("\x00", "")
if user_id == str(interface_state.myNodeNum):
formatted_message = (f"{ts_str} {config.sent_message_prefix}{ack_str}: ", sanitized_message)
else:
try:
fallback_name = decimal_to_hex(int(user_id))
except (TypeError, ValueError):
fallback_name = str(user_id)
formatted_message = (
f"{ts_str} {config.message_prefix} {node_names.get(str(user_id), fallback_name)}: ",
sanitized_message,
)
hourly_messages.setdefault(hour, []).append(formatted_message)
formatted = []
for hour, messages in sorted(hourly_messages.items()):
formatted.append((f"-- {hour} --", ""))
formatted.extend(messages)
return formatted
def _load_node_names(db_cursor):
table_name = f'"{interface_state.myNodeNum}_nodedb"'
try:
return {str(row[0]): row[1] for row in db_cursor.execute(f"SELECT user_id, short_name FROM {table_name}")}
except sqlite3.Error:
return {}
def load_messages_from_db(page_size: int = MESSAGE_PAGE_SIZE) -> None:
"""Load messages from the database for all channels and update ui_state.all_messages and ui_state.channel_list."""
try:
with sqlite3.connect(config.db_file_path, timeout=10.0) as db_connection:
@@ -85,6 +135,7 @@ def load_messages_from_db() -> None:
query = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ?"
db_cursor.execute(query, (f"{str(interface_state.myNodeNum)}_%_messages",))
tables = [row[0] for row in db_cursor.fetchall()]
node_names = _load_node_names(db_cursor)
# Iterate through each table and fetch its messages
for table_name in tables:
@@ -97,15 +148,22 @@ def load_messages_from_db() -> None:
db_cursor.execute(update_table_query)
db_connection.commit()
query = f"SELECT user_id, message_text, timestamp, ack_type FROM {quoted_table_name}"
query = f"""
SELECT rowid, user_id, message_text, timestamp, ack_type
FROM {quoted_table_name}
ORDER BY rowid DESC LIMIT ?
"""
try:
# Fetch all messages from the table
db_cursor.execute(query)
db_messages = [(row[0], row[1], row[2], row[3]) for row in db_cursor.fetchall()] # Save as tuples
db_cursor.execute(query, (page_size + 1,))
rows = db_cursor.fetchall()
has_older = len(rows) > page_size
db_messages = list(reversed(rows[:page_size]))
# Extract the channel name from the table name
channel = table_name.split("_")[1]
prefix = f"{interface_state.myNodeNum}_"
channel = table_name[len(prefix) : -len("_messages")]
# Convert the channel to an integer if it's numeric, otherwise keep it as a string (nodenum vs channel name)
channel = int(channel) if channel.isdigit() else channel
@@ -118,49 +176,10 @@ def load_messages_from_db() -> None:
if channel not in ui_state.all_messages:
ui_state.all_messages[channel] = []
# Add messages to ui_state.all_messages grouped by hourly timestamp
hourly_messages = {}
for row in db_messages:
user_id, message, timestamp, ack_type = row
# Only ack_type is allowed to be None
if user_id is None or message is None or timestamp is None:
logging.warning(f"Skipping row with NULL required field(s): {row}")
continue
hour = datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:00")
if hour not in hourly_messages:
hourly_messages[hour] = []
ack_str = config.ack_unknown_str
if ack_type == "Implicit":
ack_str = config.ack_implicit_str
elif ack_type == "Ack":
ack_str = config.ack_str
elif ack_type == "Nak":
ack_str = config.nak_str
ts_str = datetime.fromtimestamp(timestamp).strftime("[%H:%M:%S]")
if user_id == str(interface_state.myNodeNum):
sanitized_message = message.replace("\x00", "")
formatted_message = (
f"{ts_str} {config.sent_message_prefix}{ack_str}: ",
sanitized_message,
)
else:
sanitized_message = message.replace("\x00", "")
formatted_message = (
f"{ts_str} {config.message_prefix} {get_name_from_database(int(user_id), 'short')}: ",
sanitized_message,
)
hourly_messages[hour].append(formatted_message)
# Flatten the hourly messages into ui_state.all_messages[channel]
for hour, messages in sorted(hourly_messages.items()):
ui_state.all_messages[channel].append((f"-- {hour} --", ""))
ui_state.all_messages[channel].extend(messages)
ui_state.all_messages[channel].extend(_format_db_messages(db_messages, node_names))
if db_messages:
ui_state.oldest_message_rowid[channel] = db_messages[0][0]
ui_state.has_older_messages[channel] = has_older
except sqlite3.Error as e:
logging.error(f"SQLite error while loading messages from table '{table_name}': {e}")
@@ -169,6 +188,43 @@ def load_messages_from_db() -> None:
logging.error(f"SQLite error in load_messages_from_db: {e}")
def load_older_messages(channel, page_size: int = MESSAGE_PAGE_SIZE) -> int:
"""Prepend one older page for a channel and return the number of messages loaded."""
before_rowid = ui_state.oldest_message_rowid.get(channel)
if before_rowid is None or not ui_state.has_older_messages.get(channel, False):
return 0
try:
with sqlite3.connect(config.db_file_path, timeout=10.0) as db_connection:
db_connection.execute("PRAGMA busy_timeout=10000")
db_cursor = db_connection.cursor()
query = f"""
SELECT rowid, user_id, message_text, timestamp, ack_type
FROM {get_table_name(channel)}
WHERE rowid < ? ORDER BY rowid DESC LIMIT ?
"""
db_cursor.execute(query, (before_rowid, page_size + 1))
rows = db_cursor.fetchall()
has_older = len(rows) > page_size
db_messages = list(reversed(rows[:page_size]))
if not db_messages:
ui_state.has_older_messages[channel] = False
return 0
older = _format_db_messages(db_messages, _load_node_names(db_cursor))
current = ui_state.all_messages.setdefault(channel, [])
last_older_header = next((prefix for prefix, _ in reversed(older) if prefix.startswith("--")), None)
if current and current[0][0] == last_older_header:
current.pop(0)
ui_state.all_messages[channel] = older + current
ui_state.oldest_message_rowid[channel] = db_messages[0][0]
ui_state.has_older_messages[channel] = has_older
return len(db_messages)
except sqlite3.Error as e:
logging.error(f"SQLite error loading older messages for channel '{channel}': {e}")
return 0
def init_nodedb() -> None:
"""Initialize the node database and update it with nodes from the interface."""
tests/test_contact_ui.py
+19
View File
@@ -70,6 +70,25 @@ class ContactUiTests(unittest.TestCase):
self.assertFalse(ui_state.redraw_channels)
self.assertFalse(ui_state.redraw_messages)
def test_draw_messages_resizes_pad_once(self) -> None:
ui_state.channel_list = ["Primary"]
ui_state.all_messages = {"Primary": [("[10:00] RX: ", "one"), ("[10:01] RX: ", "two")]}
contact_ui.messages_pad = mock.Mock()
contact_ui.messages_win = mock.Mock()
contact_ui.messages_win.getmaxyx.return_value = (10, 40)
contact_ui.packetlog_win = mock.Mock()
contact_ui.packetlog_win.getmaxyx.return_value = (1, 40)
contact_ui.messages_win.getbegyx.return_value = (0, 0)
with mock.patch.object(contact_ui, "paint_frame"):
with mock.patch.object(contact_ui, "get_color", return_value=0):
with mock.patch.object(contact_ui, "refresh_pad"):
with mock.patch.object(contact_ui, "draw_packetlog_win"):
with mock.patch.object(contact_ui, "draw_window_arrows"):
contact_ui.draw_messages_window()
contact_ui.messages_pad.resize.assert_called_once_with(2, 40)
def test_refresh_node_selection_reserves_scroll_arrow_column(self) -> None:
ui_state.node_list = [101, 202]
ui_state.selected_node = 1
tests/test_db_handler.py
+48
View File
@@ -112,6 +112,54 @@ class DbHandlerTests(unittest.TestCase):
self.assertTrue(any("RM:" in prefix for prefix, _ in messages))
self.assertEqual(ui_state.all_messages[789][-1][1], "hidden")
def test_message_history_is_loaded_in_bounded_pages(self) -> None:
db_handler.update_node_info_in_db(456, long_name="Remote Node", short_name="RM")
db_handler.ensure_table_exists(
'"123_Primary_messages"',
"user_id TEXT, message_text TEXT, timestamp INTEGER, ack_type TEXT",
)
with sqlite3.connect(config.db_file_path) as conn:
conn.executemany(
'INSERT INTO "123_Primary_messages" VALUES (?, ?, ?, ?)',
[("456", f"message-{i}", 1700000000 + i, None) for i in range(7)],
)
db_handler.load_messages_from_db(page_size=3)
loaded_text = [message for _, message in ui_state.all_messages["Primary"] if message]
self.assertEqual(loaded_text, ["message-4", "message-5", "message-6"])
self.assertTrue(ui_state.has_older_messages["Primary"])
self.assertEqual(db_handler.load_older_messages("Primary", page_size=3), 3)
loaded_text = [message for _, message in ui_state.all_messages["Primary"] if message]
self.assertEqual(
loaded_text,
["message-1", "message-2", "message-3", "message-4", "message-5", "message-6"],
)
self.assertTrue(ui_state.has_older_messages["Primary"])
self.assertEqual(db_handler.load_older_messages("Primary", page_size=3), 1)
loaded_text = [message for _, message in ui_state.all_messages["Primary"] if message]
self.assertEqual(loaded_text, [f"message-{i}" for i in range(7)])
self.assertFalse(ui_state.has_older_messages["Primary"])
def test_message_table_channel_name_may_contain_underscores(self) -> None:
db_handler.ensure_node_table_exists()
db_handler.ensure_table_exists(
'"123_My_Channel_messages"',
"user_id TEXT, message_text TEXT, timestamp INTEGER, ack_type TEXT",
)
with sqlite3.connect(config.db_file_path) as conn:
conn.execute(
'INSERT INTO "123_My_Channel_messages" VALUES (?, ?, ?, ?)',
("123", "hello", 1700000000, None),
)
db_handler.load_messages_from_db()
self.assertIn("My_Channel", ui_state.channel_list)
self.assertEqual(ui_state.all_messages["My_Channel"][-1][1], "hello")
def test_init_nodedb_inserts_nodes_from_interface(self) -> None:
interface_state.interface = build_demo_interface()
interface_state.myNodeNum = DEMO_LOCAL_NODE_NUM