Traceroute Return Path logged and displayed (#97)

* traceroute returns are now logged and /packetlist now graphs the correct data for a return route
* now using alembic to update schema
* HOWTO - Alembic

---------

Co-authored-by: Joel Krauska <jkrauska@gmail.com>

README.md

@@ -14,6 +14,7 @@ The project serves as a real-time monitoring and diagnostic tool for the Meshtas
 * New API /api/edges (See API documentation)
 * Adds edges to the map (click to see traceroute and neighbours)
 
+
 ### Version 2.0.4 update - August 2025
 * New statistic page with more data.
 * New API /api/stats (See API documentation).

alembic/versions/ac311b3782a1_add_route_return_to_traceroute.py

@@ -0,0 +1,31 @@
+"""add route_return to traceroute
+
+Revision ID: ac311b3782a1
+Revises: 1717fa5c6545
+Create Date: 2025-11-04 20:28:33.174137
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = 'ac311b3782a1'
+down_revision: str | None = '1717fa5c6545'
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # Add route_return column to traceroute table
+    with op.batch_alter_table('traceroute', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('route_return', sa.LargeBinary(), nullable=True))
+
+
+def downgrade() -> None:
+    # Remove route_return column from traceroute table
+    with op.batch_alter_table('traceroute', schema=None) as batch_op:
+        batch_op.drop_column('route_return')

docs/Database-Changes-With-Alembic.md

@@ -0,0 +1,146 @@
+# Database Changes With Alembic
+
+This guide explains how to make database schema changes in MeshView using Alembic migrations.
+
+## Overview
+
+When you need to add, modify, or remove columns from database tables, you must:
+1. Update the SQLAlchemy model
+2. Create an Alembic migration
+3. Let the system automatically apply the migration
+
+## Step-by-Step Process
+
+### 1. Update the Model
+
+Edit `meshview/models.py` to add/modify the column in the appropriate model class:
+
+```python
+class Traceroute(Base):
+    __tablename__ = "traceroute"
+
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    # ... existing columns ...
+    route_return: Mapped[bytes] = mapped_column(nullable=True)  # New column
+```
+
+### 2. Create an Alembic Migration
+
+Generate a new migration file with a descriptive message:
+
+```bash
+./env/bin/alembic revision -m "add route_return to traceroute"
+```
+
+This creates a new file in `alembic/versions/` with a unique revision ID.
+
+### 3. Fill in the Migration
+
+Edit the generated migration file to implement the actual database changes:
+
+```python
+def upgrade() -> None:
+    # Add route_return column to traceroute table
+    with op.batch_alter_table('traceroute', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('route_return', sa.LargeBinary(), nullable=True))
+
+
+def downgrade() -> None:
+    # Remove route_return column from traceroute table
+    with op.batch_alter_table('traceroute', schema=None) as batch_op:
+        batch_op.drop_column('route_return')
+```
+
+### 4. Migration Runs Automatically
+
+When you restart the application with `mvrun.py`:
+
+1. The writer process (`startdb.py`) starts up
+2. It checks if the database schema is up to date
+3. If new migrations are pending, it runs them automatically
+4. The reader process (web server) waits for migrations to complete before starting
+
+**No manual migration command is needed** - the application handles this automatically on startup.
+
+### 5. Commit Both Files
+
+Add both files to git:
+
+```bash
+git add meshview/models.py
+git add alembic/versions/ac311b3782a1_add_route_return_to_traceroute.py
+git commit -m "Add route_return column to traceroute table"
+```
+
+## Important Notes
+
+### SQLite Compatibility
+
+Always use `batch_alter_table` for SQLite compatibility:
+
+```python
+with op.batch_alter_table('table_name', schema=None) as batch_op:
+    batch_op.add_column(...)
+```
+
+SQLite has limited ALTER TABLE support, and `batch_alter_table` works around these limitations.
+
+### Migration Process
+
+- **Writer process** (`startdb.py`): Runs migrations on startup
+- **Reader process** (web server in `main.py`): Waits for migrations to complete
+- Migrations are checked and applied every time the application starts
+- The system uses a migration status table to coordinate between processes
+
+### Common Column Types
+
+```python
+# Integer
+column: Mapped[int] = mapped_column(BigInteger, nullable=True)
+
+# String
+column: Mapped[str] = mapped_column(nullable=True)
+
+# Bytes/Binary
+column: Mapped[bytes] = mapped_column(nullable=True)
+
+# DateTime
+column: Mapped[datetime] = mapped_column(nullable=True)
+
+# Boolean
+column: Mapped[bool] = mapped_column(nullable=True)
+
+# Float
+column: Mapped[float] = mapped_column(nullable=True)
+```
+
+### Migration File Location
+
+Migrations are stored in: `alembic/versions/`
+
+Each migration file includes:
+- Revision ID (unique identifier)
+- Down revision (previous migration in chain)
+- Create date
+- `upgrade()` function (applies changes)
+- `downgrade()` function (reverts changes)
+
+## Troubleshooting
+
+### Migration Not Running
+
+If migrations don't run automatically:
+
+1. Check that the database is writable
+2. Look for errors in the startup logs
+3. Verify the migration chain is correct (each migration references the previous one)
+
+### Manual Migration (Not Recommended)
+
+If you need to manually run migrations for debugging:
+
+```bash
+./env/bin/alembic upgrade head
+```
+
+However, the application normally handles this automatically.

meshview/models.py

@@ -103,6 +103,7 @@ class Traceroute(Base):
     done: Mapped[bool] = mapped_column(nullable=True)
     route: Mapped[bytes] = mapped_column(nullable=True)
     import_time: Mapped[datetime] = mapped_column(nullable=True)
+    route_return: Mapped[bytes] = mapped_column(nullable=True)
     import_time_us: Mapped[int] = mapped_column(BigInteger, nullable=True)
 
     __table_args__ = (

meshview/mqtt_store.py

@@ -199,15 +199,7 @@ async def process_envelope(topic, env):
 
         # --- TRACEROUTE_APP (no conflict handling, normal insert)
         if env.packet.decoded.portnum == PortNum.TRACEROUTE_APP:
-            packet_id = None
-            if env.packet.decoded.want_response:
-                packet_id = env.packet.id
-            else:
-                result = await session.execute(
-                    select(Packet).where(Packet.id == env.packet.decoded.request_id)
-                )
-                if result.scalar_one_or_none():
-                    packet_id = env.packet.decoded.request_id
+            packet_id = env.packet.id
             if packet_id is not None:
                 now = datetime.datetime.now(datetime.UTC)
                 now_us = int(now.timestamp() * 1_000_000)

meshview/templates/packet.html

@@ -56,9 +56,10 @@
                             {% endfor %}
                         </ul>
                         {% if packet.raw_mesh_packet.decoded.want_response %}
-                        <a href="/graph/traceroute/{{packet.id}}">graph</a>
+                        <a href="/graph/traceroute/{{packet.id}}">graph route</a>
+
                         {% else %}
-                        <a href="/graph/traceroute/{{packet.raw_mesh_packet.decoded.request_id}}">graph</a>
+                        <a href="/graph/traceroute/{{packet.id}}">graph route return</a>
                         {% endif %}
                     {% endif %}
                     <pre>{{packet.payload}}</pre>