diff --git a/AGENTS.md b/AGENTS.md index c5dd2cf..01c7ab4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -316,7 +316,7 @@ Read state (`last_read_at`) is tracked **server-side** for consistency across de **State Tracking Keys (Frontend)**: Generated by `getStateKey()` for message times (sidebar sorting): - Channels: `channel-{channel_key}` -- Contacts: `contact-{12-char-pubkey-prefix}` +- Contacts: `contact-{full-public-key}` **Note:** These are NOT the same as `Message.conversation_key` (the database field). @@ -359,7 +359,7 @@ mc.subscribe(EventType.ACK, handler) | `MESHCORE_BLE_PIN` | *(required with BLE)* | BLE PIN code | | `MESHCORE_DATABASE_PATH` | `data/meshcore.db` | SQLite database location | -**Note:** `max_radio_contacts` and `experimental_channel_double_send` are runtime settings stored in the database (`app_settings` table), not environment variables. They are configured via `PATCH /api/settings`. +**Note:** Runtime app settings are stored in the database (`app_settings` table), not environment variables. These include `max_radio_contacts`, `experimental_channel_double_send`, `auto_decrypt_dm_on_advert`, `sidebar_sort_order`, `advert_interval`, `last_advert_time`, `favorites`, `last_message_times`, and `bots`. They are configured via `GET/PATCH /api/settings` (and related settings endpoints). `experimental_channel_double_send` is an opt-in experimental setting: when enabled, channel sends perform a second byte-perfect resend after a 3-second delay. diff --git a/app/AGENTS.md b/app/AGENTS.md index d92ee03..2290e98 100644 --- a/app/AGENTS.md +++ b/app/AGENTS.md @@ -1,717 +1,207 @@ # Backend AGENTS.md -This document provides context for AI assistants and developers working on the FastAPI backend. +This document is the backend working guide for agents and developers. +Keep it aligned with `app/` source files and router behavior. -## Technology Stack +## Stack -- **FastAPI** - Async web framework with automatic OpenAPI docs -- **aiosqlite** - Async SQLite driver -- **meshcore** - MeshCore radio library (local dependency at `../meshcore_py`) -- **Pydantic** - Data validation and settings management -- **PyCryptodome** - AES-128 encryption for packet decryption -- **UV** - Python package manager +- FastAPI +- aiosqlite +- Pydantic +- MeshCore Python library (`references/meshcore_py`) +- PyCryptodome -## Directory Structure +## Backend Map -``` +```text app/ -├── main.py # FastAPI app, lifespan, router registration, static file serving -├── config.py # Pydantic settings (env vars: MESHCORE_*) -├── database.py # SQLite schema, connection management, runs migrations -├── migrations.py # Database migrations using SQLite user_version pragma -├── models.py # Pydantic models for API request/response -├── repository.py # Database CRUD (ContactRepository, ChannelRepository, etc.) -├── radio.py # RadioManager - serial connection to MeshCore device -├── radio_sync.py # Periodic sync, contact auto-loading to radio -├── decoder.py # Packet decryption (channel + direct messages) -├── packet_processor.py # Raw packet processing, advertisement handling -├── keystore.py # Ephemeral key store (private key in memory only) -├── event_handlers.py # Radio event subscriptions, ACK tracking, repeat detection -├── websocket.py # WebSocketManager for real-time client updates -└── routers/ # All routes prefixed with /api - ├── health.py # GET /api/health - ├── radio.py # Radio config, advertise, private key, reboot - ├── contacts.py # Contact CRUD, radio sync, mark-read - ├── channels.py # Channel CRUD, radio sync, mark-read - ├── messages.py # Message list and send (direct/channel) - ├── packets.py # Raw packet endpoints, historical decryption - ├── read_state.py # Read state: unread counts, mark-all-read - ├── settings.py # App settings (max_radio_contacts, experimental_channel_double_send, etc.) - └── ws.py # WebSocket endpoint at /api/ws +├── main.py # App startup/lifespan, router registration, static frontend mounting +├── config.py # Env-driven runtime settings +├── database.py # SQLite connection + base schema + migration runner +├── migrations.py # Schema migrations (SQLite user_version) +├── models.py # Pydantic request/response models +├── repository.py # Data access layer +├── radio.py # RadioManager + auto-reconnect monitor +├── radio_sync.py # Polling, sync, periodic advertisement loop +├── decoder.py # Packet parsing/decryption +├── packet_processor.py # Raw packet pipeline, dedup, path handling +├── event_handlers.py # MeshCore event subscriptions and ACK tracking +├── websocket.py # WS manager + broadcast helpers +├── bot.py # Bot execution and outbound bot sends +├── dependencies.py # Shared FastAPI dependency providers +├── frontend_static.py # Mount/serve built frontend (production) +└── routers/ + ├── health.py + ├── radio.py + ├── contacts.py + ├── channels.py + ├── messages.py + ├── packets.py + ├── read_state.py + ├── settings.py + └── ws.py ``` -## Intentional Security Design Decisions +## Core Runtime Flows -The following are **deliberate design choices**, not bugs. They are documented in the README with appropriate warnings. Do not "fix" these or flag them as vulnerabilities. +### Incoming data -1. **No CORS restrictions**: `CORSMiddleware` in `main.py` allows all origins, methods, and headers. This lets users access their radio from any device/origin on their network. -2. **No authentication or authorization**: All API endpoints and the WebSocket are openly accessible. The app is designed for trusted networks only (home LAN, VPN). -3. **Arbitrary bot code execution**: `bot.py` uses `exec()` with full `__builtins__` to run user-provided Python code. This is intentional — bots are a power-user automation feature. Safeguards are limited to timeouts and concurrency limits, not sandboxing. +1. Radio emits events. +2. `on_rx_log_data` stores raw packet and tries decrypt/pipeline handling. +3. Decrypted messages are inserted into `messages` and broadcast over WS. +4. `CONTACT_MSG_RECV` is a fallback DM path when packet pipeline cannot decrypt. -## Key Architectural Patterns +### Outgoing messages -### Repository Pattern +1. Send endpoints in `routers/messages.py` call MeshCore commands. +2. Message is persisted as outgoing. +3. Endpoint broadcasts WS `message` event so all live clients update. +4. ACK/repeat updates arrive later as `message_acked` events. -All database operations go through repository classes in `repository.py`: +### Connection lifecycle -```python -from app.repository import ContactRepository, ChannelRepository, MessageRepository, RawPacketRepository, AppSettingsRepository +- `RadioManager.start_connection_monitor()` checks health every 5s. +- On reconnect, monitor runs `post_connect_setup()` before broadcasting healthy state. +- Setup includes handler registration, key export, time sync, contact/channel sync, polling/advert tasks. -# Examples -contact = await ContactRepository.get_by_key_prefix("abc123") -await MessageRepository.create(msg_type="PRIV", text="Hello", received_at=timestamp) -await RawPacketRepository.mark_decrypted(packet_id, message_id) +## Important Behaviors -# App settings (single-row pattern) -settings = await AppSettingsRepository.get() -await AppSettingsRepository.update(auto_decrypt_dm_on_advert=True) -await AppSettingsRepository.update(experimental_channel_double_send=True) -await AppSettingsRepository.add_favorite("contact", public_key) -``` +### Read/unread state -### Radio Connection +- Server is source of truth (`contacts.last_read_at`, `channels.last_read_at`). +- `GET /api/read-state/unreads` returns counts, mention flags, and `last_message_times`. -`RadioManager` in `radio.py` handles radio connection over Serial, TCP, or BLE: +### Echo/repeat dedup -```python -from app.radio import radio_manager +- Message uniqueness: `(type, conversation_key, text, sender_timestamp)`. +- Duplicate insert is treated as an echo/repeat; ACK count/path list is updated. -# Access meshcore instance -if radio_manager.meshcore: - await radio_manager.meshcore.commands.send_msg(dst, msg) +### Periodic advertisement -# Check connection info (e.g. "Serial: /dev/ttyUSB0", "TCP: 192.168.1.1:4000", "BLE: AA:BB:CC:DD:EE:FF") -print(radio_manager.connection_info) -``` +- Controlled by `app_settings.advert_interval` (seconds). +- `0` means disabled. +- Last send time tracked in `app_settings.last_advert_time`. -Transport is configured via env vars (see root AGENTS.md). When no transport env vars are set, serial auto-detection is used. - -### Event-Driven Architecture - -Radio events flow through `event_handlers.py`: - -| Event | Handler | Actions | -|-------|---------|---------| -| `CONTACT_MSG_RECV` | `on_contact_message` | **Fallback only** - stores DM if packet processor didn't handle it | -| `RX_LOG_DATA` | `on_rx_log_data` | Store packet, decrypt channels/DMs, broadcast via WS | -| `PATH_UPDATE` | `on_path_update` | Update contact path info | -| `NEW_CONTACT` | `on_new_contact` | Sync contact from radio's internal database | -| `ACK` | `on_ack` | Match pending ACKs, mark message acked, broadcast | - -**Note on DM handling**: Direct messages are primarily handled by the packet processor via -`RX_LOG_DATA`, which decrypts using the exported private key. The `CONTACT_MSG_RECV` handler -exists as a fallback for radios without `ENABLE_PRIVATE_KEY_EXPORT=1` in firmware. - -### WebSocket Broadcasting - -Real-time updates use `ws_manager` singleton: - -```python -from app.websocket import ws_manager - -# Broadcast to all connected clients -await ws_manager.broadcast("message", {"id": 1, "text": "Hello"}) -``` - -Event types: `health`, `contacts`, `channels`, `message`, `contact`, `raw_packet`, `message_acked`, `error` - -**Note:** The WebSocket initial connect only sends `health`. Contacts and channels are fetched -via REST (`GET /api/contacts`, `GET /api/channels`) for faster parallel loading. The WS still -broadcasts real-time `contacts`/`channels` updates when data changes. - -Helper functions for common broadcasts: - -```python -from app.websocket import broadcast_error, broadcast_health - -# Notify clients of errors (shows toast in frontend) -broadcast_error("Operation failed", "Additional details") - -# Notify clients of connection status change -broadcast_health(radio_connected=True, connection_info="Serial: /dev/ttyUSB0") -``` - -### Connection Monitoring - -`RadioManager` includes a background task that monitors connection status: - -- Checks connection every 5 seconds -- Broadcasts `health` event on status change -- Attempts automatic reconnection when connection lost -- **Runs full `post_connect_setup()` after successful reconnect** (event handlers, key export, time sync, contact/channel sync, advertisements, message polling) -- Resilient to transient errors (logs and continues rather than crashing) -- Supports manual reconnection via `POST /api/radio/reconnect` - -```python -from app.radio import radio_manager - -# Manual reconnection -success = await radio_manager.reconnect() - -# Background monitor (started automatically in app lifespan) -await radio_manager.start_connection_monitor() -await radio_manager.stop_connection_monitor() -``` - -### Message Polling - -Periodic message polling serves as a fallback for platforms where push events are unreliable. -Use `pause_polling()` to temporarily suspend polling during operations that need exclusive -radio access (e.g., repeater CLI commands): - -```python -from app.radio_sync import pause_polling, is_polling_paused - -# Pause polling during sensitive operations (supports nesting) -async with pause_polling(): - # Polling is paused here - await do_repeater_operation() - - async with pause_polling(): - # Still paused (nested) - await do_another_operation() - - # Still paused (outer context active) - -# Polling resumes when all contexts exit - -# Check current state -if is_polling_paused(): - print("Polling is currently paused") -``` - -### Periodic Advertisement - -The server automatically sends an advertisement every hour to announce presence on the mesh. -This helps maintain visibility to other nodes and refreshes routing information. - -- Started automatically on radio connection -- Interval: 1 hour (3600 seconds) -- Uses flood mode for maximum reach - -```python -from app.radio_sync import start_periodic_advert, stop_periodic_advert, send_advertisement - -# Start/stop periodic advertising -start_periodic_advert() # Started automatically in lifespan -await stop_periodic_advert() - -# Manual advertisement -await send_advertisement() # Returns True on success -``` - -## Database Schema - -```sql -contacts ( - public_key TEXT PRIMARY KEY, -- 64-char hex - name TEXT, - type INTEGER DEFAULT 0, -- 0=unknown, 1=client, 2=repeater, 3=room - flags INTEGER DEFAULT 0, - last_path TEXT, -- Routing path hex - last_path_len INTEGER DEFAULT -1, - last_advert INTEGER, -- Unix timestamp of last advertisement - lat REAL, lon REAL, - last_seen INTEGER, - on_radio INTEGER DEFAULT 0, -- Boolean: contact loaded on radio - last_contacted INTEGER, -- Unix timestamp of last message sent/received - last_read_at INTEGER -- Unix timestamp when conversation was last read -) - -channels ( - key TEXT PRIMARY KEY, -- 32-char hex channel key - name TEXT NOT NULL, - is_hashtag INTEGER DEFAULT 0, -- Key derived from SHA256(name)[:16] - on_radio INTEGER DEFAULT 0, - last_read_at INTEGER -- Unix timestamp when channel was last read -) - -messages ( - id INTEGER PRIMARY KEY, - type TEXT NOT NULL, -- 'PRIV' or 'CHAN' - conversation_key TEXT NOT NULL, -- User pubkey for PRIV, channel key for CHAN - text TEXT NOT NULL, - sender_timestamp INTEGER, - received_at INTEGER NOT NULL, - paths TEXT, -- JSON array of {path, received_at} for multiple delivery paths - txt_type INTEGER DEFAULT 0, - signature TEXT, - outgoing INTEGER DEFAULT 0, - acked INTEGER DEFAULT 0, - UNIQUE(type, conversation_key, text, sender_timestamp) -- Deduplication -) - -raw_packets ( - id INTEGER PRIMARY KEY, - timestamp INTEGER NOT NULL, - data BLOB NOT NULL, -- Raw packet bytes - message_id INTEGER, -- FK to messages if decrypted - payload_hash TEXT, -- SHA256 of payload for deduplication (UNIQUE index) - FOREIGN KEY (message_id) REFERENCES messages(id) -) - -app_settings ( - id INTEGER PRIMARY KEY CHECK (id = 1), -- Single-row pattern - max_radio_contacts INTEGER DEFAULT 200, - experimental_channel_double_send INTEGER DEFAULT 0, -- Experimental delayed byte-perfect channel resend - favorites TEXT DEFAULT '[]', -- JSON array of {type, id} - auto_decrypt_dm_on_advert INTEGER DEFAULT 0, - sidebar_sort_order TEXT DEFAULT 'recent', -- 'recent' or 'alpha' - last_message_times TEXT DEFAULT '{}', -- JSON object of state_key -> timestamp - preferences_migrated INTEGER DEFAULT 0 -- One-time migration flag -) -``` - -## Database Migrations (`migrations.py`) - -Schema migrations use SQLite's `user_version` pragma for version tracking: - -```python -from app.migrations import get_version, set_version, run_migrations - -# Check current schema version -version = await get_version(conn) # Returns int (0 for new/unmigrated DB) - -# Run pending migrations (called automatically on startup) -applied = await run_migrations(conn) # Returns number of migrations applied -``` - -### How It Works - -1. `database.py` calls `run_migrations()` after schema initialization -2. Each migration checks `user_version` and runs if needed -3. Migrations are idempotent (safe to run multiple times) -4. `ALTER TABLE ADD COLUMN` handles existing columns gracefully - -### Adding a New Migration - -```python -# In migrations.py -async def run_migrations(conn: aiosqlite.Connection) -> int: - version = await get_version(conn) - applied = 0 - - if version < 1: - await _migrate_001_add_last_read_at(conn) - await set_version(conn, 1) - applied += 1 - - # Add new migrations here: - # if version < 2: - # await _migrate_002_something(conn) - # await set_version(conn, 2) - # applied += 1 - - return applied -``` - -## Packet Decryption (`decoder.py`) - -The decoder handles MeshCore packet decryption for historical packet analysis: - -### Packet Types - -```python -class PayloadType(IntEnum): - GROUP_TEXT = 0x05 # Channel messages (decryptable) - TEXT_MESSAGE = 0x02 # Direct messages - ACK = 0x03 - ADVERT = 0x04 - # ... see decoder.py for full list -``` - -### Channel Key Derivation - -Hashtag channels derive keys from name: -```python -channel_key = hashlib.sha256(b"#channelname").digest()[:16] -``` - -### Decryption Flow - -1. Parse packet header to get payload type -2. For `GROUP_TEXT`: extract channel_hash (1 byte), cipher_mac (2 bytes), ciphertext -3. Verify HMAC-SHA256 using 32-byte secret (key + 16 zero bytes) -4. Decrypt with AES-128 ECB -5. Parse decrypted content: timestamp (4 bytes), flags (1 byte), "sender: message" text - -```python -from app.decoder import try_decrypt_packet_with_channel_key - -result = try_decrypt_packet_with_channel_key(raw_bytes, channel_key) -if result: - print(f"{result.sender}: {result.message}") -``` - -### Direct Message Decryption - -Direct messages use ECDH key exchange (Ed25519 → X25519) for shared secret derivation. - -**Key storage**: The private key is exported from the radio on startup and stored in memory -via `keystore.py`. This enables server-side DM decryption even when contacts aren't loaded -on the radio. - -**Primary path (packet processor)**: When an `RX_LOG_DATA` event contains a `TEXT_MESSAGE` -packet, `packet_processor.py` handles the complete flow: -1. Decrypts using known contact public keys and stored private key -2. Filters CLI responses (txt_type=1 in flags) -3. Stores message in database -4. Broadcasts via WebSocket -5. Updates contact's last_contacted timestamp -6. Triggers bot if enabled - -**Fallback path (event handler)**: If the packet processor can't decrypt (no private key -export, unknown contact), `on_contact_message` handles DMs from the MeshCore library's -`CONTACT_MSG_RECV` event. DB deduplication prevents double-storage when both paths fire. - -**Prefix-stored DMs (edge case)**: A rare scenario can occur when the radio can decrypt -a DM (contact is on the radio) but the server cannot (private key not exported or -contact not yet known server-side). The `CONTACT_MSG_RECV` payload may only include a -pubkey prefix. If the full key isn't known yet, the message is stored with the prefix -as `conversation_key`. When a full contact key becomes known (via advertisement or -radio sync), the server attempts to **claim** those prefix messages and upgrade them -to the full key. Claims only occur when the prefix matches exactly one contact to -avoid mis-attribution in large contact sets. Until claimed, these DMs will not show -in the UI because conversations are keyed by full public keys. - -**Outgoing DMs**: Outgoing direct messages are only sent via the app's REST API -(`POST /api/messages/direct`), which stores the plaintext directly in the database. -No decryption is needed for outgoing DMs. The real-time packet processor may also see -the outgoing packet via `RX_LOG_DATA`, but the DB UNIQUE constraint deduplicates it -against the already-stored plaintext. Historical decryption intentionally skips outgoing -packets for the same reason — the app already has the plaintext. - -**Historical decryption**: When creating a contact with `try_historical=True`, the server -attempts to decrypt all stored `TEXT_MESSAGE` packets for that contact. This only recovers -**incoming** messages; outgoing DMs are already stored as plaintext by the send endpoint. - -**Direction detection**: The decoder uses the 1-byte dest_hash and src_hash to determine -if a message is incoming or outgoing. Edge case: when both bytes match (1/256 chance), -defaults to treating as incoming. - -```python -from app.decoder import try_decrypt_dm - -result = try_decrypt_dm(raw_bytes, private_key, contact_public_key) -if result: - print(f"{result.message} (timestamp={result.timestamp})") -``` - -## Advertisement Parsing (`decoder.py`) - -Advertisement packets contain contact information including optional GPS coordinates. - -### Packet Structure - -``` -Bytes 0-31: Public key (32 bytes) -Bytes 32-35: Timestamp (4 bytes, little-endian Unix timestamp) -Bytes 36-99: Signature (64 bytes) -Byte 100: App flags -Bytes 101+: Optional fields (location, name) based on flags -``` - -### App Flags (byte 100) - -- Bits 0-3: Device role (1=Chat, 2=Repeater, 3=Room, 4=Sensor) -- Bit 4: Has location (lat/lon follow) -- Bit 5: Has feature 1 -- Bit 6: Has feature 2 -- Bit 7: Has name (null-terminated string at end) - -### GPS Extraction - -When bit 4 is set, latitude and longitude follow as signed int32 little-endian values, -divided by 1,000,000 to get decimal degrees: - -```python -from app.decoder import parse_advertisement - -advert = parse_advertisement(payload_bytes) -if advert: - print(f"Device role: {advert.device_role}") # 1=Chat, 2=Repeater - if advert.lat and advert.lon: - print(f"Location: {advert.lat}, {advert.lon}") -``` - -### Data Flow - -1. `event_handlers.py` receives ADVERTISEMENT event -2. `packet_processor.py` calls `parse_advertisement()` to extract data -3. Contact is upserted with location data (`lat`, `lon`) and `device_role` as `type` -4. Frontend MapView displays contacts with GPS coordinates - -## ACK and Repeat Detection - -The `acked` field is an integer count, not a boolean: -- `0` = not acked -- `1` = one ACK/echo received -- `2+` = multiple flood echoes received - -### Direct Message ACKs - -When sending a direct message, an expected ACK code is tracked: -```python -from app.event_handlers import track_pending_ack - -track_pending_ack(expected_ack="abc123", message_id=42, timeout_ms=30000) -``` - -When ACK event arrives, the message's ack count is incremented. - -### Channel Message Repeats - -Flood messages echo back through repeaters. Repeats are detected via the database -UNIQUE constraint on `(type, conversation_key, text, sender_timestamp)`. When an INSERT -hits a duplicate, `_handle_duplicate_message()` in `packet_processor.py` increments the -ack count and adds the new delivery path. There is no timestamp-windowed matching; -deduplication is exact-match only. - -Each repeat increments the ack count. The frontend displays: -- `?` = no acks -- `✓` = 1 echo -- `✓2`, `✓3`, etc. = multiple echoes (real-time updates via WebSocket) - -### Auto-Contact Sync to Radio - -To enable the radio to auto-ACK incoming DMs, recent non-repeater contacts are -automatically loaded to the radio. Configured via `max_radio_contacts` setting (default 200). - -- Triggered on each advertisement from a non-repeater contact -- Loads most recently contacted non-repeaters (by `last_contacted` timestamp) -- Throttled to at most once per 30 seconds -- `last_contacted` updated on message send/receive - -```python -from app.radio_sync import sync_recent_contacts_to_radio - -result = await sync_recent_contacts_to_radio(force=True) -# Returns: {"loaded": 5, "already_on_radio": 195, "failed": 0} -``` - -## API Endpoints - -All endpoints are prefixed with `/api`. +## API Surface (all under `/api`) ### Health -- `GET /api/health` - Connection status, connection info +- `GET /health` ### Radio -- `GET /api/radio/config` - Read config (public key, name, radio params) -- `PATCH /api/radio/config` - Update name, lat/lon, tx_power, radio params -- `PUT /api/radio/private-key` - Import private key to radio (write-only) -- `POST /api/radio/advertise?flood=true` - Send advertisement -- `POST /api/radio/reboot` - Reboot radio or reconnect if disconnected -- `POST /api/radio/reconnect` - Manual reconnection attempt +- `GET /radio/config` +- `PATCH /radio/config` +- `PUT /radio/private-key` +- `POST /radio/advertise` +- `POST /radio/reboot` +- `POST /radio/reconnect` ### Contacts -- `GET /api/contacts` - List from database -- `GET /api/contacts/{key}` - Get by public key or prefix -- `POST /api/contacts` - Create contact (optionally trigger historical DM decryption) -- `POST /api/contacts/sync` - Pull from radio to database -- `POST /api/contacts/{key}/add-to-radio` - Push to radio -- `POST /api/contacts/{key}/remove-from-radio` - Remove from radio -- `POST /api/contacts/{key}/mark-read` - Mark conversation as read (updates last_read_at) -- `POST /api/contacts/{key}/telemetry` - Request telemetry from repeater (see below) +- `GET /contacts` +- `GET /contacts/{public_key}` +- `POST /contacts` +- `DELETE /contacts/{public_key}` +- `POST /contacts/sync` +- `POST /contacts/{public_key}/add-to-radio` +- `POST /contacts/{public_key}/remove-from-radio` +- `POST /contacts/{public_key}/mark-read` +- `POST /contacts/{public_key}/telemetry` +- `POST /contacts/{public_key}/command` +- `POST /contacts/{public_key}/trace` ### Channels -- `GET /api/channels` - List from database -- `GET /api/channels/{key}` - Get by channel key -- `POST /api/channels` - Create (hashtag if name starts with # or no key provided) -- `POST /api/channels/sync` - Pull from radio -- `POST /api/channels/{key}/mark-read` - Mark channel as read (updates last_read_at) -- `DELETE /api/channels/{key}` - Delete channel - -### Read State -- `GET /api/read-state/unreads?name=X` - Server-computed unread counts, mention flags, and last message times -- `POST /api/read-state/mark-all-read` - Mark all contacts and channels as read +- `GET /channels` +- `GET /channels/{key}` +- `POST /channels` +- `DELETE /channels/{key}` +- `POST /channels/sync` +- `POST /channels/{key}/mark-read` ### Messages -- `GET /api/messages?type=&conversation_key=&limit=&offset=` - List with filters -- `POST /api/messages/direct` - Send direct message -- `POST /api/messages/channel` - Send channel message (stores outgoing immediately; response includes current ack count) +- `GET /messages` +- `POST /messages/direct` +- `POST /messages/channel` ### Packets -- `GET /api/packets/undecrypted/count` - Count of undecrypted packets -- `POST /api/packets/decrypt/historical` - Try decrypting old packets with new key +- `GET /packets/undecrypted/count` +- `POST /packets/decrypt/historical` +- `POST /packets/maintenance` + +### Read state +- `GET /read-state/unreads` +- `POST /read-state/mark-all-read` ### Settings -- `GET /api/settings` - Get all app settings -- `PATCH /api/settings` - Update settings (max_radio_contacts, experimental_channel_double_send, auto_decrypt_dm_on_advert, sidebar_sort_order) -- `POST /api/settings/favorites` - Add a favorite -- `DELETE /api/settings/favorites` - Remove a favorite -- `POST /api/settings/favorites/toggle` - Toggle favorite status -- `POST /api/settings/migrate` - One-time migration from frontend localStorage +- `GET /settings` +- `PATCH /settings` +- `POST /settings/favorites/toggle` +- `POST /settings/migrate` ### WebSocket -- `WS /api/ws` - Real-time updates (health, contacts, channels, messages, raw packets) +- `WS /ws` -### Static Files (Production) -In production, the backend serves the frontend if `frontend/dist` exists. Users must build the -frontend first (`cd frontend && npm install && npm run build`): -- `/` - Serves `frontend/dist/index.html` -- `/assets/*` - Serves compiled JS/CSS from `frontend/dist/assets/` -- `/*` - Falls back to `index.html` for SPA routing +## WebSocket Events + +- `health` +- `contacts` +- `channels` +- `contact` +- `message` +- `message_acked` +- `raw_packet` +- `error` +- `success` + +Initial WS connect sends `health` only. Contacts/channels are loaded by REST. + +## Data Model Notes + +Main tables: +- `contacts` +- `channels` +- `messages` +- `raw_packets` +- `app_settings` + +`app_settings` fields in active model: +- `max_radio_contacts` +- `experimental_channel_double_send` +- `favorites` +- `auto_decrypt_dm_on_advert` +- `sidebar_sort_order` +- `last_message_times` +- `preferences_migrated` +- `advert_interval` +- `last_advert_time` +- `bots` + +## Security Posture (intentional) + +- No authn/authz. +- No CORS restriction (`*`). +- Bot code executes user-provided Python via `exec()`. + +These are product decisions for trusted-network deployments; do not flag as accidental vulnerabilities. ## Testing -Run tests with: +Run backend tests: + ```bash PYTHONPATH=. uv run pytest tests/ -v ``` -Key test files: -- `tests/test_decoder.py` - Channel + direct message decryption, key exchange, real-world test vectors -- `tests/test_keystore.py` - Ephemeral key store operations -- `tests/test_event_handlers.py` - ACK tracking, repeat detection, CLI response filtering -- `tests/test_api.py` - API endpoint tests, read state tracking -- `tests/test_migrations.py` - Migration system, schema versioning +High-signal suites: +- `tests/test_packet_pipeline.py` +- `tests/test_event_handlers.py` +- `tests/test_send_messages.py` +- `tests/test_radio.py` +- `tests/test_api.py` +- `tests/test_migrations.py` -## Common Tasks +## Editing Checklist -### Adding a New Endpoint - -1. Create or update router in `app/routers/` -2. Define Pydantic models in `app/models.py` if needed -3. Add repository methods in `app/repository.py` for database operations -4. Register router in `app/main.py` if new file -5. Add tests in `tests/` - -### Adding a New Event Handler - -1. Define handler in `app/event_handlers.py` -2. Register in `register_event_handlers()` function -3. Broadcast updates via `ws_manager` as needed - -### Working with Radio Commands - -```python -# Available via radio_manager.meshcore.commands -await mc.commands.send_msg(dst, msg) -await mc.commands.send_chan_msg(chan, msg) -await mc.commands.get_contacts() -await mc.commands.add_contact(contact_dict) -await mc.commands.set_channel(idx, name, key) -await mc.commands.send_advert(flood=True) -``` - -## Repeater Telemetry - -The `POST /api/contacts/{key}/telemetry` endpoint fetches status, neighbors, and ACL from repeaters (contact type=2). - -### Request Flow - -1. Verify contact exists and is a repeater (type=2) -2. Add contact to radio with stored path data (from advertisements) -3. Send login with password -4. Request status with retries (3 attempts, 10s timeout) -5. Fetch neighbors with `fetch_all_neighbours()` (handles pagination) -6. Fetch ACL with `req_acl_sync()` -7. Resolve pubkey prefixes to contact names from database - -### ACL Permission Levels - -```python -ACL_PERMISSION_NAMES = { - 0: "Guest", - 1: "Read-only", - 2: "Read-write", - 3: "Admin", -} -``` - -### Response Models - -```python -class NeighborInfo(BaseModel): - pubkey_prefix: str # 4-12 char prefix - name: str | None # Resolved contact name - snr: float # Signal-to-noise ratio in dB - last_heard_seconds: int # Seconds since last heard - -class AclEntry(BaseModel): - pubkey_prefix: str # 12 char prefix - name: str | None # Resolved contact name - permission: int # 0-3 - permission_name: str # Human-readable name - -class TelemetryResponse(BaseModel): - # Status fields - pubkey_prefix: str - battery_volts: float # Converted from mV - uptime_seconds: int - # ... signal quality, packet counts, etc. - - # Related data - neighbors: list[NeighborInfo] - acl: list[AclEntry] -``` - -## Repeater CLI Commands - -After login via telemetry endpoint, you can send CLI commands to repeaters: - -### Endpoint - -`POST /api/contacts/{key}/command` - Send a CLI command (assumes already logged in) - -### Request/Response - -```python -class CommandRequest(BaseModel): - command: str # CLI command to send - -class CommandResponse(BaseModel): - command: str # Echo of sent command - response: str # Response from repeater - sender_timestamp: int | None # Timestamp from response -``` - -### Common Commands - -``` -get name / set name # Repeater name -get tx / set tx # TX power -get radio / set radio # Radio params -tempradio # Temporary radio change -setperm <0-3> # ACL: 0=guest, 1=ro, 2=rw, 3=admin -clock / clock sync # Get/sync time -ver # Firmware version -reboot # Restart repeater -``` - -### CLI Response Filtering - -CLI responses have `txt_type=1` (vs `txt_type=0` for normal messages). Both DM handling -paths filter these to prevent storage—the command endpoint returns the response directly. - -**Packet processor path** (primary): -```python -# In create_dm_message_from_decrypted() -txt_type = decrypted.flags & 0x0F -if txt_type == 1: - return None # Skip CLI responses -``` - -**Event handler path** (fallback): -```python -# In on_contact_message() -txt_type = payload.get("txt_type", 0) -if txt_type == 1: - return # Skip CLI responses -``` - -### Helper Function - -`prepare_repeater_connection()` handles the login dance: -1. Add contact to radio with stored path from DB (`out_path`, `out_path_len`) -2. Send login with password -3. Wait for key exchange to complete - -### Contact Path Tracking - -When advertisements are received, path data is extracted and stored: -- `last_path`: Hex string of routing path bytes -- `last_path_len`: Number of hops (-1=flood/unknown, 0=direct, >0=hops through repeaters) - -**Shortest path selection**: When receiving echoed advertisements within 60 seconds, the shortest path is kept. This ensures we use the most efficient route when multiple paths exist. +When changing backend behavior: +1. Update/add router and repository tests. +2. Confirm WS event contracts when payload shape changes. +3. Run `PYTHONPATH=. uv run pytest tests/ -v`. +4. If API contract changed, update frontend types and AGENTS docs. diff --git a/app/bot.py b/app/bot.py index 09a399e..50907bd 100644 --- a/app/bot.py +++ b/app/bot.py @@ -173,7 +173,6 @@ async def _send_single_bot_message( from app.models import SendChannelMessageRequest, SendDirectMessageRequest from app.routers.messages import send_channel_message, send_direct_message - from app.websocket import broadcast_event # Serialize bot sends and enforce minimum spacing async with _bot_send_lock: @@ -190,15 +189,11 @@ async def _send_single_bot_message( if is_dm: logger.info("Bot sending DM reply to %s", sender_key[:12]) request = SendDirectMessageRequest(destination=sender_key, text=message_text) - message = await send_direct_message(request) - # Broadcast to WebSocket (endpoint returns to HTTP caller, bot needs explicit broadcast) - broadcast_event("message", message.model_dump()) + await send_direct_message(request) elif channel_key: logger.info("Bot sending channel reply to %s", channel_key[:8]) request = SendChannelMessageRequest(channel_key=channel_key, text=message_text) - message = await send_channel_message(request) - # Broadcast to WebSocket - broadcast_event("message", message.model_dump()) + await send_channel_message(request) else: logger.warning("Cannot send bot response: no destination") return # Don't update timestamp if we didn't send diff --git a/app/radio.py b/app/radio.py index 25b04ab..e0d4ff5 100644 --- a/app/radio.py +++ b/app/radio.py @@ -365,7 +365,7 @@ class RadioManager: self._meshcore = None logger.debug("Radio disconnected") - async def reconnect(self) -> bool: + async def reconnect(self, *, broadcast_on_success: bool = True) -> bool: """Attempt to reconnect to the radio. Returns True if reconnection was successful, False otherwise. @@ -399,7 +399,8 @@ class RadioManager: if self.is_connected: logger.info("Radio reconnected successfully at %s", self._connection_info) - broadcast_health(True, self._connection_info) + if broadcast_on_success: + broadcast_health(True, self._connection_info) return True else: logger.warning("Reconnection failed: not connected after connect()") @@ -435,13 +436,18 @@ class RadioManager: if not current_connected: # Attempt reconnection on every loop while disconnected - if not self.is_reconnecting and await self.reconnect(): + if not self.is_reconnecting and await self.reconnect( + broadcast_on_success=False + ): await self.post_connect_setup() + broadcast_health(True, self._connection_info) self._last_connected = True elif not self._last_connected and current_connected: - # Connection restored (might have reconnected automatically) + # Connection restored (might have reconnected automatically). + # Always run setup before reporting healthy. logger.info("Radio connection restored") + await self.post_connect_setup() broadcast_health(True, self._connection_info) self._last_connected = True diff --git a/app/routers/messages.py b/app/routers/messages.py index 41b701c..6dd4b8b 100644 --- a/app/routers/messages.py +++ b/app/routers/messages.py @@ -10,6 +10,7 @@ from app.event_handlers import track_pending_ack from app.models import Message, SendChannelMessageRequest, SendDirectMessageRequest from app.radio import radio_manager from app.repository import AmbiguousPublicKeyPrefixError, MessageRepository +from app.websocket import broadcast_event logger = logging.getLogger(__name__) router = APIRouter(prefix="/messages", tags=["messages"]) @@ -132,6 +133,9 @@ async def send_direct_message(request: SendDirectMessageRequest) -> Message: acked=0, ) + # Broadcast so all connected clients (not just sender) see the outgoing message immediately. + broadcast_event("message", message.model_dump()) + # Trigger bots for outgoing DMs (runs in background, doesn't block response) from app.bot import run_bot_for_message @@ -281,6 +285,9 @@ async def send_channel_message(request: SendChannelMessageRequest) -> Message: acked=acked_count, ) + # Broadcast so all connected clients (not just sender) see the outgoing message immediately. + broadcast_event("message", message.model_dump()) + # Trigger bots for outgoing channel messages (runs in background, doesn't block response) from app.bot import run_bot_for_message diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index 725394d..da0f1ba 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -1,749 +1,201 @@ # Frontend AGENTS.md -This document provides context for AI assistants and developers working on the React frontend. +This document is the frontend working guide for agents and developers. +Keep it aligned with `frontend/src` source code. -## Technology Stack +## Stack -- **React 18** - UI framework with hooks -- **TypeScript** - Type safety -- **Vite** - Build tool with HMR -- **Vitest** - Testing framework -- **Sonner** - Toast notifications -- **shadcn/ui components** - Sheet, Tabs, Button (in `components/ui/`) -- **meshcore-hashtag-cracker** - WebGPU-accelerated channel key bruteforcing -- **nosleep.js** - Prevents device sleep during cracking -- **leaflet / react-leaflet** - Interactive map for node locations +- React 18 + TypeScript +- Vite +- Vitest + Testing Library +- shadcn/ui primitives +- Tailwind utility classes + local CSS (`index.css`, `styles.css`) +- Sonner (toasts) +- Leaflet / react-leaflet (map) +- `meshcore-hashtag-cracker` + `nosleep.js` (channel cracker) -## Directory Structure +## Frontend Map -``` -frontend/ -├── src/ -│ ├── main.tsx # Entry point, renders App -│ ├── App.tsx # Main component, all state management -│ ├── api.ts # REST API client -│ ├── types.ts # TypeScript interfaces -│ ├── useWebSocket.ts # WebSocket hook with auto-reconnect -│ ├── messageCache.ts # LRU message cache for conversation switching -│ ├── styles.css # Dark theme CSS -│ ├── hooks/ -│ │ ├── index.ts -│ │ ├── useConversationMessages.ts # Message fetching, pagination, cache integration -│ │ ├── useUnreadCounts.ts # Unread count tracking -│ │ └── useRepeaterMode.ts # Repeater login/CLI mode -│ ├── utils/ -│ │ ├── messageParser.ts # Text parsing utilities -│ │ ├── conversationState.ts # localStorage for message times (sidebar sorting) -│ │ ├── pubkey.ts # Public key utilities (prefix matching, display names) -│ │ └── contactAvatar.ts # Avatar generation (colors, initials/emoji) -│ ├── components/ -│ │ ├── ui/ # shadcn/ui components -│ │ │ ├── sonner.tsx # Toast notifications (Sonner wrapper) -│ │ │ ├── sheet.tsx # Slide-out panel -│ │ │ ├── tabs.tsx # Tab navigation -│ │ │ └── button.tsx # Button component -│ │ ├── StatusBar.tsx # Radio status, reconnect button, config button -│ │ ├── Sidebar.tsx # Contacts/channels list, search, unread badges -│ │ ├── MessageList.tsx # Message display, avatars, clickable senders -│ │ ├── MessageInput.tsx # Text input with imperative handle -│ │ ├── ContactAvatar.tsx # Contact profile image component -│ │ ├── RawPacketList.tsx # Raw packet feed (tertiary debug/observation tool) -│ │ ├── MapView.tsx # Leaflet map showing node locations -│ │ ├── CrackerPanel.tsx # WebGPU channel key cracker (lazy-loads wordlist) -│ │ ├── NewMessageModal.tsx -│ │ └── SettingsModal.tsx # Unified settings: radio config, identity, connectivity, database, advertise -│ └── test/ -│ ├── setup.ts # Test setup (jsdom, matchers) -│ ├── messageParser.test.ts -│ ├── unreadCounts.test.ts -│ ├── contactAvatar.test.ts -│ ├── messageDeduplication.test.ts -│ └── websocket.test.ts -├── index.html -├── vite.config.ts # API proxy config -├── tsconfig.json -└── package.json +```text +frontend/src/ +├── App.tsx # App shell and orchestration +├── api.ts # Typed REST client +├── types.ts # Shared TS contracts +├── useWebSocket.ts # WS lifecycle + event dispatch +├── messageCache.ts # Conversation-scoped cache +├── index.css # Global styles/utilities +├── styles.css # Additional global app styles +├── hooks/ +│ ├── useConversationMessages.ts +│ ├── useUnreadCounts.ts +│ ├── useRepeaterMode.ts +│ └── useAirtimeTracking.ts +├── utils/ +│ ├── urlHash.ts +│ ├── conversationState.ts +│ ├── favorites.ts +│ ├── messageParser.ts +│ ├── pathUtils.ts +│ ├── pubkey.ts +│ └── contactAvatar.ts +├── components/ +│ ├── StatusBar.tsx +│ ├── Sidebar.tsx +│ ├── MessageList.tsx +│ ├── MessageInput.tsx +│ ├── NewMessageModal.tsx +│ ├── SettingsModal.tsx +│ ├── RawPacketList.tsx +│ ├── MapView.tsx +│ ├── VisualizerView.tsx +│ ├── PacketVisualizer.tsx +│ ├── PathModal.tsx +│ ├── CrackerPanel.tsx +│ ├── BotCodeEditor.tsx +│ ├── ContactAvatar.tsx +│ └── ui/ +└── test/ + ├── api.test.ts + ├── appFavorites.test.tsx + ├── appStartupHash.test.tsx + ├── contactAvatar.test.ts + ├── integration.test.ts + ├── messageCache.test.ts + ├── messageParser.test.ts + ├── pathUtils.test.ts + ├── radioPresets.test.ts + ├── repeaterMode.test.ts + ├── settingsModal.test.tsx + ├── unreadCounts.test.ts + ├── urlHash.test.ts + ├── useConversationMessages.test.ts + ├── useRepeaterMode.test.ts + ├── useWebSocket.lifecycle.test.ts + ├── websocket.test.ts + └── setup.ts ``` -## Intentional Security Design Decisions +## Architecture Notes -The following are **deliberate design choices**, not bugs. They are documented in the README with appropriate warnings. Do not "fix" these or flag them as vulnerabilities. +### State ownership -1. **No authentication UI**: There is no login page, session management, or auth tokens. The frontend assumes open access to the backend API. The app is designed for trusted networks only (home LAN, VPN). -2. **No CORS restrictions on the backend**: The frontend may be served from a different origin during development (Vite on `:5173` vs backend on `:8000`). The backend allows all origins intentionally. -3. **Arbitrary bot code**: The settings UI lets users write and enable Python bot code that the backend executes via `exec()`. This is a power-user feature, not a vulnerability. +`App.tsx` orchestrates high-level state (health, config, contacts/channels, active conversation, UI flags). +Specialized logic is delegated to hooks: +- `useConversationMessages`: fetch, pagination, dedup/update helpers +- `useUnreadCounts`: unread counters, mention tracking, recent-sort timestamps +- `useRepeaterMode`: repeater login/command workflow -## State Management +### Initial load + realtime -All application state lives in `App.tsx` using React hooks. No external state library. +- Initial data: REST fetches (`api.ts`) for config/settings/channels/contacts/unreads. +- WebSocket: realtime deltas/events. +- On WS connect, backend sends `health` only; contacts/channels still come from REST. -### Core State +### Message behavior -```typescript -const [health, setHealth] = useState(null); -const [config, setConfig] = useState(null); -const [appSettings, setAppSettings] = useState(null); -const [contacts, setContacts] = useState([]); -const [channels, setChannels] = useState([]); -const [messages, setMessages] = useState([]); -const [rawPackets, setRawPackets] = useState([]); -const [activeConversation, setActiveConversation] = useState(null); -const [unreadCounts, setUnreadCounts] = useState>({}); -``` - -### App Settings - -App settings are stored server-side and include: -- `favorites` - List of favorited conversations (channels/contacts) -- `sidebar_sort_order` - 'recent' or 'alpha' -- `auto_decrypt_dm_on_advert` - Auto-decrypt historical DMs on new contact -- `experimental_channel_double_send` - Experimental setting to send a byte-perfect channel resend after 3 seconds -- `last_message_times` - Map of conversation keys to last message timestamps - -**Migration**: On first load, localStorage preferences are migrated to the server. -The `preferences_migrated` flag prevents duplicate migrations. - -### Message Cache (`messageCache.ts`) - -An LRU cache stores messages for recently-visited conversations so switching back is instant -(no spinner, no fetch). On switch-away, the active conversation's messages are saved to cache. -On switch-to, cached messages are restored immediately, then a silent background fetch reconciles -with the backend — only updating state if something differs (missed WS message, stale ack). -The happy path (cache is consistent) causes zero rerenders. - -- Cache capacity: `MAX_CACHED_CONVERSATIONS` (20) entries, `MAX_MESSAGES_PER_ENTRY` (200) messages each -- Uses `Map` insertion-order for LRU semantics (delete + re-insert promotes to MRU) -- WebSocket messages for non-active cached conversations are written directly to the cache -- `reconcile(current, fetched)` compares by message ID + ack count, returns merged array or `null` -- Deleted conversations are evicted from cache via `remove()` - -### State Flow - -1. **REST API** fetches initial data on mount in parallel (config, settings, channels, contacts, unreads) -2. **WebSocket** pushes real-time updates (health, messages, contact changes, raw packets) -3. **Components** receive state as props, call handlers to trigger changes - -**Note:** Contacts and channels are loaded via REST on mount (not from WebSocket initial push). -The WebSocket only sends health on initial connect, then broadcasts real-time updates. - -### Conversation Header - -For contacts, the header shows path information alongside "Last heard": -- `(Last heard: 10:30 AM, direct)` - Direct neighbor (path_len=0) -- `(Last heard: 10:30 AM, 2 hops)` - Routed through repeaters (path_len>0) -- `(Last heard: 10:30 AM, flood)` - No known path (path_len=-1) +- Outgoing sends are optimistic in UI and persisted server-side. +- Backend also emits WS `message` for outgoing sends so other clients stay in sync. +- ACK/repeat updates arrive as `message_acked` events. ## WebSocket (`useWebSocket.ts`) -The `useWebSocket` hook manages real-time connection: +- Auto reconnect (3s) with cleanup guard on unmount. +- Heartbeat ping every 30s. +- Event handlers: `health`, `contacts`, `channels`, `message`, `contact`, `raw_packet`, `message_acked`, `error`, `success`. -```typescript -const wsHandlers = useMemo(() => ({ - onHealth: (data: HealthStatus) => setHealth(data), - onMessage: (msg: Message) => { /* add to list, track unread */ }, - onMessageAcked: (messageId: number, ackCount: number) => { /* update ack count */ }, - // ... -}), []); +## URL Hash Navigation (`utils/urlHash.ts`) -useWebSocket(wsHandlers); -``` +Supported routes: +- `#raw` +- `#map` +- `#map/focus/{pubkey_or_prefix}` +- `#visualizer` +- `#channel/{channelKey}` +- `#channel/{channelKey}/{label}` +- `#contact/{publicKey}` +- `#contact/{publicKey}/{label}` -### Features +Legacy name-based hashes are still accepted for compatibility. -- **Auto-reconnect**: Reconnects after 3 seconds on disconnect -- **Heartbeat**: Sends ping every 30 seconds -- **Event types**: `health`, `contacts`, `channels`, `message`, `contact`, `raw_packet`, `message_acked`, `error` -- **Error handling**: `onError` handler displays toast notifications for backend errors +## Conversation State Keys (`utils/conversationState.ts`) -### URL Detection +`getStateKey(type, id)` produces: +- channels: `channel-{channelKey}` +- contacts: `contact-{publicKey}` -```typescript -const isDev = window.location.port === '5173'; -const wsUrl = isDev - ? 'ws://localhost:8000/api/ws' - : `${protocol}//${window.location.host}/api/ws`; -``` +Use full contact public key here (not 12-char prefix). -## API Client (`api.ts`) +`conversationState.ts` keeps an in-memory cache and localStorage helpers used for migration/compatibility. +Canonical persistence for unread and sort metadata is server-side (`app_settings` + read-state endpoints). -Typed REST client with consistent error handling: +## Utilities -```typescript -import { api } from './api'; +### `utils/pubkey.ts` -// Health -await api.getHealth(); +Current public export: +- `getContactDisplayName(name, pubkey)` -// Radio -await api.getRadioConfig(); -await api.updateRadioConfig({ name: 'MyRadio' }); -await api.sendAdvertisement(); +It falls back to a 12-char prefix when `name` is missing. -// Contacts/Channels -await api.getContacts(); -await api.createContact(publicKey, name, tryHistorical); // Create contact, optionally decrypt historical DMs -await api.getChannels(); -await api.createChannel('#test'); +### `utils/pathUtils.ts` -// Messages -await api.getMessages({ type: 'CHAN', conversation_key: channelKey, limit: 200 }); -await api.sendChannelMessage(channelKey, 'Hello'); -await api.sendDirectMessage(publicKey, 'Hello'); +Distance/validation helpers used by path + map UI. -// Historical decryption -await api.decryptHistoricalPackets({ key_type: 'channel', channel_name: '#test' }); +### `utils/favorites.ts` -// Radio reconnection -await api.reconnectRadio(); // Returns { status, message, connected } +LocalStorage migration helpers for favorites; canonical favorites are server-side. -// Repeater telemetry -await api.requestTelemetry(publicKey, password); // Returns TelemetryResponse +## Types and Contracts (`types.ts`) -// Repeater CLI commands (after login) -await api.sendRepeaterCommand(publicKey, 'ver'); // Returns CommandResponse -``` +`AppSettings` currently includes: +- `max_radio_contacts` +- `experimental_channel_double_send` +- `favorites` +- `auto_decrypt_dm_on_advert` +- `sidebar_sort_order` +- `last_message_times` +- `preferences_migrated` +- `advert_interval` +- `bots` -### API Proxy (Development) +Backend also tracks `last_advert_time` in settings responses. -Vite proxies `/api/*` to backend (backend routes are already prefixed with `/api`): +## Repeater Mode -```typescript -// vite.config.ts -server: { - proxy: { - '/api': { - target: 'http://localhost:8000', - changeOrigin: true, - }, - }, -} -``` +For repeater contacts (`type=2`): +1. Telemetry/login phase (`POST /api/contacts/{key}/telemetry`) +2. Command phase (`POST /api/contacts/{key}/command`) -## Type Definitions (`types.ts`) +CLI responses are rendered as local-only messages (not persisted to DB). -### Key Interfaces +## Styling -```typescript -interface Contact { - public_key: string; // 64-char hex public key - name: string | null; - type: number; // 0=unknown, 1=client, 2=repeater, 3=room - on_radio: boolean; - last_path_len: number; // -1=flood, 0=direct, >0=hops through repeaters - last_path: string | null; // Hex routing path - last_seen: number | null; // Unix timestamp - // ... -} +UI styling is mostly utility-class driven (Tailwind-style classes in JSX) plus shared globals in `index.css` and `styles.css`. +Do not rely on old class-only layout assumptions. -interface Channel { - key: string; // 32-char hex channel key - name: string; - is_hashtag: boolean; - on_radio: boolean; -} +## Security Posture (intentional) -interface Message { - id: number; - type: 'PRIV' | 'CHAN'; - conversation_key: string; // public key for PRIV, channel key for CHAN - text: string; - outgoing: boolean; - acked: number; // 0=not acked, 1+=ack count (flood echoes) - // ... -} - -interface Conversation { - type: 'contact' | 'channel' | 'raw' | 'map' | 'visualizer'; - id: string; // public key for contacts, channel key for channels, 'raw'/'map'/'visualizer' for special views - name: string; -} - -interface Favorite { - type: 'channel' | 'contact'; - id: string; // Channel key or contact public key -} - -interface AppSettings { - max_radio_contacts: number; - experimental_channel_double_send: boolean; - favorites: Favorite[]; - auto_decrypt_dm_on_advert: boolean; - sidebar_sort_order: 'recent' | 'alpha'; - last_message_times: Record; - preferences_migrated: boolean; -} - -// Repeater telemetry types -interface NeighborInfo { - pubkey_prefix: string; - name: string | null; - snr: number; - last_heard_seconds: number; -} - -interface AclEntry { - pubkey_prefix: string; - name: string | null; - permission: number; - permission_name: string; -} - -interface TelemetryResponse { - battery_volts: number; - uptime_seconds: number; - // ... status fields - neighbors: NeighborInfo[]; - acl: AclEntry[]; -} - -interface CommandResponse { - command: string; - response: string; - sender_timestamp: number | null; -} -``` - -## Component Patterns - -### MessageInput with Imperative Handle - -Exposes `appendText` method for click-to-mention: - -```typescript -export interface MessageInputHandle { - appendText: (text: string) => void; -} - -export const MessageInput = forwardRef( - function MessageInput({ onSend, disabled, isRepeaterMode }, ref) { - useImperativeHandle(ref, () => ({ - appendText: (text: string) => { - setText((prev) => prev + text); - inputRef.current?.focus(); - }, - })); - // ... - } -); - -// Usage in App.tsx -const messageInputRef = useRef(null); -messageInputRef.current?.appendText(`@[${sender}] `); -``` - -### Repeater Mode - -Repeater contacts (type=2) have a two-phase interaction: - -**Phase 1: Login (password mode)** -- Input type changes to `password` -- Button shows "Fetch" instead of "Send" -- Enter "." for empty password (converted to empty string) -- Submitting requests telemetry + logs in - -**Phase 2: CLI commands (after login)** -- Input switches back to normal text -- Placeholder shows "Enter CLI command..." -- Commands sent via `/contacts/{key}/command` endpoint -- Responses displayed as local messages (not persisted to database) - -```typescript -// State tracking -const [repeaterLoggedIn, setRepeaterLoggedIn] = useState(false); - -// Reset on conversation change -useEffect(() => { - setRepeaterLoggedIn(false); -}, [activeConversation?.id]); - -// Mode switches after successful telemetry -const isRepeaterMode = activeContactIsRepeater && !repeaterLoggedIn; - - -``` - -Telemetry response is displayed as three local messages (not persisted): -1. **Telemetry** - Battery voltage, uptime, signal quality, packet stats -2. **Neighbors** - Sorted by SNR (highest first), with resolved names -3. **ACL** - Access control list with permission levels - -### Repeater Message Rendering - -Repeater CLI responses often contain colons (e.g., `clock: 12:30:00`). To prevent -incorrect sender parsing, MessageList skips `parseSenderFromText()` for repeater contacts: - -```typescript -const isRepeater = contact?.type === CONTACT_TYPE_REPEATER; -const { sender, content } = isRepeater - ? { sender: null, content: msg.text } // Preserve full text - : parseSenderFromText(msg.text); -``` - -### Unread Count Tracking - -Uses refs to avoid stale closures in memoized handlers: - -```typescript -const activeConversationRef = useRef(null); - -// Keep ref in sync -useEffect(() => { - activeConversationRef.current = activeConversation; -}, [activeConversation]); - -// In WebSocket handler (can safely access current value) -const activeConv = activeConversationRef.current; -``` - -### State Tracking Keys - -State tracking keys (for message times used in sidebar sorting) are generated by `getStateKey()`: - -```typescript -import { getStateKey } from './utils/conversationState'; - -// Channels: "channel-{channelKey}" -getStateKey('channel', channelKey) // e.g., "channel-8B3387E9C5CDEA6AC9E5EDBAA115CD72" - -// Contacts: "contact-{12-char-prefix}" -getStateKey('contact', publicKey) // e.g., "contact-abc123def456" -``` - -**Note:** `getStateKey()` is NOT the same as `Message.conversation_key`. The state key is prefixed -for local state tracking, while `conversation_key` is the raw database field. - -### Read State (Server-Side) - -Unread tracking uses server-side `last_read_at` timestamps for cross-device consistency: - -```typescript -// Fetch aggregated unread counts from server (replaces bulk message fetch + client-side counting) -await api.getUnreads(myName); // Returns { counts, mentions, last_message_times } - -// Mark as read via API (called automatically when viewing conversation) -await api.markContactRead(publicKey); -await api.markChannelRead(channelKey); -await api.markAllRead(); // Bulk mark all as read -``` - -The `useUnreadCounts` hook fetches counts from `GET /api/read-state/unreads` on mount and -when channels/contacts change. Real-time increments are still tracked client-side via WebSocket -message events. The server computes unread counts using `last_read_at` vs `received_at`. - -## Utility Functions - -### Message Parser (`utils/messageParser.ts`) - -```typescript -// Parse "sender: message" format from channel messages -parseSenderFromText(text: string): { sender: string | null; content: string } - -// Format Unix timestamp to time string -formatTime(timestamp: number): string -``` - -### Public Key Utilities (`utils/pubkey.ts`) - -Consistent handling of 64-char full keys and 12-char prefixes: - -```typescript -import { getPubkeyPrefix, pubkeysMatch, getContactDisplayName } from './utils/pubkey'; - -// Extract 12-char prefix (works with full keys or existing prefixes) -getPubkeyPrefix(key) // "abc123def456..." - -// Compare keys by prefix (handles mixed full/prefix comparisons) -pubkeysMatch(key1, key2) // true if prefixes match - -// Get display name with fallback to prefix -getContactDisplayName(name, publicKey) // name or first 12 chars of key -``` - -### Conversation State (`utils/conversationState.ts`) - -```typescript -import { getStateKey, setLastMessageTime, getLastMessageTimes } from './utils/conversationState'; - -// Generate state tracking key (NOT the same as Message.conversation_key) -getStateKey('channel', channelKey) -getStateKey('contact', publicKey) - -// Track message times for sidebar sorting (stored in localStorage) -setLastMessageTime(stateKey, timestamp) -getLastMessageTimes() // Returns all tracked message times -``` - -**Note:** Read state (`last_read_at`) is tracked server-side, not in localStorage. - -### Contact Avatar (`utils/contactAvatar.ts`) - -Generates consistent profile "images" for contacts using hash-based colors: - -```typescript -import { getContactAvatar, CONTACT_TYPE_REPEATER } from './utils/contactAvatar'; - -// Get avatar info for a contact -const avatar = getContactAvatar(name, publicKey, contactType); -// Returns: { text: 'JD', background: 'hsl(180, 60%, 40%)', textColor: '#ffffff' } - -// Repeaters (type=2) always show 🛜 with gray background -const repeaterAvatar = getContactAvatar('Some Repeater', key, CONTACT_TYPE_REPEATER); -// Returns: { text: '🛜', background: '#444444', textColor: '#ffffff' } -``` - -Avatar text priority: -1. First emoji in name -2. Initials (first letter + first letter after space) -3. Single first letter -4. First 2 chars of public key (fallback) - -## CSS Patterns - -The app uses a minimal dark theme in `styles.css`. - -### Key Classes - -```css -.app /* Root container */ -.status-bar /* Top bar with radio info */ -.sidebar /* Left panel with contacts/channels */ -.sidebar-item /* Individual contact/channel row */ -.sidebar-item.unread /* Bold with badge */ -.message-area /* Main content area */ -.message-list /* Scrollable message container */ -.message /* Individual message */ -.message.outgoing /* Right-aligned, different color */ -.message .sender /* Clickable sender name */ -``` - -### Unread Badge - -```css -.sidebar-item.unread .name { - font-weight: 700; - color: #fff; -} -.sidebar-item .unread-badge { - background: #4caf50; - color: #fff; - font-size: 10px; - padding: 2px 6px; - border-radius: 10px; -} -``` +- No authentication UI. +- Frontend assumes trusted network usage. +- Bot editor intentionally allows arbitrary backend bot code configuration. ## Testing -Run tests with: ```bash cd frontend -npm run test:run # Single run -npm run test # Watch mode -``` - -### Test Files - -- `messageParser.test.ts` - Sender extraction, time formatting, conversation keys -- `unreadCounts.test.ts` - Unread tracking logic -- `contactAvatar.test.ts` - Avatar text extraction, color generation, repeater handling -- `useConversationMessages.test.ts` - Message content key generation, ack update logic -- `messageCache.test.ts` - LRU cache: eviction, dedup, ack updates, reconciliation -- `websocket.test.ts` - WebSocket message routing -- `repeaterMode.test.ts` - Repeater CLI parsing, password "." conversion -- `useRepeaterMode.test.ts` - Repeater hook: login flow, CLI commands, state reset -- `integration.test.ts` - Cross-component integration scenarios -- `urlHash.test.ts` - URL hash parsing and generation -- `pathUtils.test.ts` - Path distance calculation utilities -- `radioPresets.test.ts` - Radio preset configuration -- `api.test.ts` - API client request formatting - -### Test Setup - -Tests use jsdom environment with `@testing-library/react`: - -```typescript -// src/test/setup.ts -import '@testing-library/jest-dom'; -``` - -## Common Tasks - -### Adding a New Component - -1. Create component in `src/components/` -2. Add TypeScript props interface -3. Import and use in `App.tsx` or parent component -4. Add styles to `styles.css` - -### Adding a New API Endpoint - -1. Add method to `api.ts` -2. Add/update types in `types.ts` -3. Call from `App.tsx` or component - -### Adding New WebSocket Event - -1. Add handler option to `UseWebSocketOptions` interface in `useWebSocket.ts` -2. Add case to `onmessage` switch -3. Provide handler in `wsHandlers` object in `App.tsx` - -### Adding State - -1. Add `useState` in `App.tsx` -2. Pass down as props to components -3. If needed in WebSocket handler, also use a ref to avoid stale closures - -## Development Workflow - -```bash -# Start dev server (hot reload) -npm run dev - -# Build for production -npm run build - -# Preview production build -npm run preview - -# Run tests npm run test:run +npm run build ``` -The dev server runs on port 5173 and proxies API requests to `localhost:8000`. - -### Production Build - -In production, the FastAPI backend serves the compiled frontend from `frontend/dist`: +When touching cross-layer contracts, also run backend tests from repo root: ```bash -npm run build -# Then run backend: uv run uvicorn app.main:app --host 0.0.0.0 --port 8000 +PYTHONPATH=. uv run pytest tests/ -v ``` -## URL Hash Navigation +## Editing Checklist -Deep linking to conversations via URL hash: - -- `#channel/RoomName` - Opens a channel (leading `#` stripped from name for cleaner URLs) -- `#contact/ContactName` - Opens a DM -- `#raw` - Opens the raw packet feed -- `#map` - Opens the node map - -```typescript -// Parse hash on initial load -const hashConv = parseHashConversation(); - -// Update hash when conversation changes (uses replaceState to avoid history pollution) -window.history.replaceState(null, '', newHash); -``` - -## CrackerPanel - -The `CrackerPanel` component provides WebGPU-accelerated brute-forcing of channel keys for undecrypted GROUP_TEXT packets. - -### Features - -- **Dictionary attack first**: Uses `words.txt` wordlist -- **GPU bruteforce**: Falls back to character-by-character search -- **Queue management**: Automatically processes new packets as they arrive -- **Auto-channel creation**: Cracked channels are automatically added to the channel list -- **Configurable max length**: Adjustable while running (default: 6) -- **Retry failed**: Option to retry failed packets at increasing lengths -- **NoSleep integration**: Prevents device sleep during cracking via `nosleep.js` -- **Global collapsible panel**: Toggle from sidebar, runs in background when hidden - -### Key Implementation Patterns - -Uses refs to avoid stale closures in async callbacks: - -```typescript -const isRunningRef = useRef(false); -const isProcessingRef = useRef(false); // Prevents concurrent GPU operations -const queueRef = useRef>(new Map()); -const retryFailedRef = useRef(false); -const maxLengthRef = useRef(6); -``` - -Progress reporting shows rate in Mkeys/s or Gkeys/s depending on speed. - -## MapView - -The `MapView` component displays contacts with GPS coordinates on an interactive Leaflet map. - -### Features - -- **Location filtering**: Only shows contacts with lat/lon that were heard within the last 7 days -- **Freshness coloring**: Markers colored by how recently the contact was heard: - - Bright green (`#22c55e`) - less than 1 hour ago - - Light green (`#4ade80`) - less than 1 day ago - - Yellow-green (`#a3e635`) - less than 3 days ago - - Gray (`#9ca3af`) - older (up to 7 days) -- **Node/repeater distinction**: Regular nodes have black outlines, repeaters are larger with no outline -- **Geolocation**: Tries browser geolocation first, falls back to fitting all markers in view -- **Popups**: Click a marker to see contact name, last heard time, and coordinates - -### Data Source - -Contact location data (`lat`, `lon`) is extracted from advertisement packets in the backend (`decoder.py`). -The `last_seen` timestamp determines marker freshness. - -## Sidebar Features - -- **Sort toggle**: Default is 'recent' (most recent message first), can toggle to alphabetical -- **Mark all as read**: Button appears when there are unread messages, clears all unread counts -- **Cracker toggle**: Shows/hides the global cracker panel with running status indicator - -## Toast Notifications - -The app uses Sonner for toast notifications via a custom wrapper at `components/ui/sonner.tsx`: - -```typescript -import { toast } from './components/ui/sonner'; - -// Success toast (use sparingly - only for significant/destructive actions) -toast.success('Channel deleted'); - -// Error toast with details -toast.error('Failed to send message', { - description: err instanceof Error ? err.message : 'Check radio connection', -}); -``` - -### Error Handling Pattern - -All async operations that can fail should show error toasts. Keep console.error for debugging: - -```typescript -try { - await api.someOperation(); -} catch (err) { - console.error('Failed to do X:', err); - toast.error('Failed to do X', { - description: err instanceof Error ? err.message : 'Check radio connection', - }); -} -``` - -### Where Toasts Are Used - -**Error toasts** (shown when operations fail): -- `App.tsx`: Advertisement, channel delete, contact delete -- `useConversationMessages.ts`: Message loading (initial and pagination) -- `MessageInput.tsx`: Message send, telemetry request -- `CrackerPanel.tsx`: Channel save after cracking, WebGPU unavailable -- `StatusBar.tsx`: Manual reconnection failure -- `useWebSocket.ts`: Backend errors via WebSocket `error` events - -**Success toasts** (used sparingly for significant actions): -- Radio connection/disconnection status changes -- Manual reconnection success -- Advertisement sent, channel/contact deleted (confirmation of intentional actions) - -**Avoid success toasts** for routine operations like sending messages - only show errors. - -The `` component is rendered in `App.tsx` with `position="top-right"`. +1. If API/WS payloads change, update `types.ts`, handlers, and tests. +2. If URL/hash behavior changes, update `utils/urlHash.ts` tests. +3. If read/unread semantics change, update `useUnreadCounts` tests. +4. Keep this file concise; prefer source links over speculative detail. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index d808059..3eeee68 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -78,6 +78,7 @@ export function App() { const [config, setConfig] = useState(null); const [appSettings, setAppSettings] = useState(null); const [contacts, setContacts] = useState([]); + const [contactsLoaded, setContactsLoaded] = useState(false); const [channels, setChannels] = useState([]); const [rawPackets, setRawPackets] = useState([]); const [activeConversation, setActiveConversation] = useState(null); @@ -178,7 +179,10 @@ export function App() { description: success.details, }); }, - onContacts: (data: Contact[]) => setContacts(data), + onContacts: (data: Contact[]) => { + setContacts(data); + setContactsLoaded(true); + }, onChannels: (data: Channel[]) => setChannels(data), onMessage: (msg: Message) => { const activeConv = activeConversationRef.current; @@ -339,7 +343,15 @@ export function App() { // Fetch contacts and channels via REST (parallel, faster than WS serial push) api.getChannels().then(setChannels).catch(console.error); - fetchAllContacts().then(setContacts).catch(console.error); + fetchAllContacts() + .then((data) => { + setContacts(data); + setContactsLoaded(true); + }) + .catch((err) => { + console.error(err); + setContactsLoaded(true); + }); }, [fetchConfig, fetchAppSettings, fetchUndecryptedCount, fetchAllContacts]); // One-time migration of localStorage preferences to server @@ -467,10 +479,12 @@ export function App() { // Phase 2: Resolve contact hash (only if phase 1 didn't set a conversation) useEffect(() => { if (hasSetDefaultConversation.current || activeConversation) return; - if (contacts.length === 0) return; const hashConv = parseHashConversation(); if (hashConv?.type === 'contact') { + // Wait until the initial contacts load finishes so we don't fall back early. + if (!contactsLoaded) return; + const contact = resolveContactFromHashToken(hashConv.name, contacts); if (contact) { setActiveConversation({ @@ -481,21 +495,21 @@ export function App() { hasSetDefaultConversation.current = true; return; } - } - // Contact hash didn't match — fall back to Public if channels loaded - if (channels.length > 0) { - const publicChannel = channels.find((c) => c.name === 'Public'); - if (publicChannel) { - setActiveConversation({ - type: 'channel', - id: publicChannel.key, - name: publicChannel.name, - }); - hasSetDefaultConversation.current = true; + // Contact hash didn't match — fall back to Public if channels loaded. + if (channels.length > 0) { + const publicChannel = channels.find((c) => c.name === 'Public'); + if (publicChannel) { + setActiveConversation({ + type: 'channel', + id: publicChannel.key, + name: publicChannel.name, + }); + hasSetDefaultConversation.current = true; + } } } - }, [contacts, channels, activeConversation]); + }, [contacts, channels, activeConversation, contactsLoaded]); // Keep ref in sync and update URL hash useEffect(() => { diff --git a/frontend/src/components/PacketVisualizer.tsx b/frontend/src/components/PacketVisualizer.tsx index 112fc5a..8de7316 100644 --- a/frontend/src/components/PacketVisualizer.tsx +++ b/frontend/src/components/PacketVisualizer.tsx @@ -1584,7 +1584,7 @@ export function PacketVisualizer({ title="Split ambiguous repeaters into separate nodes based on traffic patterns (prev→next). Helps identify colliding prefixes representing different physical nodes." className={!showAmbiguousPaths ? 'text-muted-foreground' : ''} > - Hueristically group repeaters by traffic pattern + Heuristically group repeaters by traffic pattern