From a65de73b3ab0dd22de73fdb96b18dfebf6ed0822 Mon Sep 17 00:00:00 2001 From: Jim Schrempp Date: Tue, 4 Nov 2025 20:36:24 -0800 Subject: [PATCH] Traceroute Return Path logged and displayed (#97) * traceroute returns are now logged and /packetlist now graphs the correct data for a return route * now using alembic to update schema * HOWTO - Alembic --------- Co-authored-by: Joel Krauska --- README.md | 1 + ...1b3782a1_add_route_return_to_traceroute.py | 31 ++++ docs/Database-Changes-With-Alembic.md | 146 ++++++++++++++++++ meshview/models.py | 1 + meshview/mqtt_store.py | 10 +- meshview/templates/packet.html | 5 +- 6 files changed, 183 insertions(+), 11 deletions(-) create mode 100644 alembic/versions/ac311b3782a1_add_route_return_to_traceroute.py create mode 100644 docs/Database-Changes-With-Alembic.md diff --git a/README.md b/README.md index 8893933..5655499 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ The project serves as a real-time monitoring and diagnostic tool for the Meshtas * New API /api/edges (See API documentation) * Adds edges to the map (click to see traceroute and neighbours) + ### Version 2.0.4 update - August 2025 * New statistic page with more data. * New API /api/stats (See API documentation). diff --git a/alembic/versions/ac311b3782a1_add_route_return_to_traceroute.py b/alembic/versions/ac311b3782a1_add_route_return_to_traceroute.py new file mode 100644 index 0000000..046e8a5 --- /dev/null +++ b/alembic/versions/ac311b3782a1_add_route_return_to_traceroute.py @@ -0,0 +1,31 @@ +"""add route_return to traceroute + +Revision ID: ac311b3782a1 +Revises: 1717fa5c6545 +Create Date: 2025-11-04 20:28:33.174137 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = 'ac311b3782a1' +down_revision: str | None = '1717fa5c6545' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # Add route_return column to traceroute table + with op.batch_alter_table('traceroute', schema=None) as batch_op: + batch_op.add_column(sa.Column('route_return', sa.LargeBinary(), nullable=True)) + + +def downgrade() -> None: + # Remove route_return column from traceroute table + with op.batch_alter_table('traceroute', schema=None) as batch_op: + batch_op.drop_column('route_return') diff --git a/docs/Database-Changes-With-Alembic.md b/docs/Database-Changes-With-Alembic.md new file mode 100644 index 0000000..453dfa5 --- /dev/null +++ b/docs/Database-Changes-With-Alembic.md @@ -0,0 +1,146 @@ +# Database Changes With Alembic + +This guide explains how to make database schema changes in MeshView using Alembic migrations. + +## Overview + +When you need to add, modify, or remove columns from database tables, you must: +1. Update the SQLAlchemy model +2. Create an Alembic migration +3. Let the system automatically apply the migration + +## Step-by-Step Process + +### 1. Update the Model + +Edit `meshview/models.py` to add/modify the column in the appropriate model class: + +```python +class Traceroute(Base): + __tablename__ = "traceroute" + + id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) + # ... existing columns ... + route_return: Mapped[bytes] = mapped_column(nullable=True) # New column +``` + +### 2. Create an Alembic Migration + +Generate a new migration file with a descriptive message: + +```bash +./env/bin/alembic revision -m "add route_return to traceroute" +``` + +This creates a new file in `alembic/versions/` with a unique revision ID. + +### 3. Fill in the Migration + +Edit the generated migration file to implement the actual database changes: + +```python +def upgrade() -> None: + # Add route_return column to traceroute table + with op.batch_alter_table('traceroute', schema=None) as batch_op: + batch_op.add_column(sa.Column('route_return', sa.LargeBinary(), nullable=True)) + + +def downgrade() -> None: + # Remove route_return column from traceroute table + with op.batch_alter_table('traceroute', schema=None) as batch_op: + batch_op.drop_column('route_return') +``` + +### 4. Migration Runs Automatically + +When you restart the application with `mvrun.py`: + +1. The writer process (`startdb.py`) starts up +2. It checks if the database schema is up to date +3. If new migrations are pending, it runs them automatically +4. The reader process (web server) waits for migrations to complete before starting + +**No manual migration command is needed** - the application handles this automatically on startup. + +### 5. Commit Both Files + +Add both files to git: + +```bash +git add meshview/models.py +git add alembic/versions/ac311b3782a1_add_route_return_to_traceroute.py +git commit -m "Add route_return column to traceroute table" +``` + +## Important Notes + +### SQLite Compatibility + +Always use `batch_alter_table` for SQLite compatibility: + +```python +with op.batch_alter_table('table_name', schema=None) as batch_op: + batch_op.add_column(...) +``` + +SQLite has limited ALTER TABLE support, and `batch_alter_table` works around these limitations. + +### Migration Process + +- **Writer process** (`startdb.py`): Runs migrations on startup +- **Reader process** (web server in `main.py`): Waits for migrations to complete +- Migrations are checked and applied every time the application starts +- The system uses a migration status table to coordinate between processes + +### Common Column Types + +```python +# Integer +column: Mapped[int] = mapped_column(BigInteger, nullable=True) + +# String +column: Mapped[str] = mapped_column(nullable=True) + +# Bytes/Binary +column: Mapped[bytes] = mapped_column(nullable=True) + +# DateTime +column: Mapped[datetime] = mapped_column(nullable=True) + +# Boolean +column: Mapped[bool] = mapped_column(nullable=True) + +# Float +column: Mapped[float] = mapped_column(nullable=True) +``` + +### Migration File Location + +Migrations are stored in: `alembic/versions/` + +Each migration file includes: +- Revision ID (unique identifier) +- Down revision (previous migration in chain) +- Create date +- `upgrade()` function (applies changes) +- `downgrade()` function (reverts changes) + +## Troubleshooting + +### Migration Not Running + +If migrations don't run automatically: + +1. Check that the database is writable +2. Look for errors in the startup logs +3. Verify the migration chain is correct (each migration references the previous one) + +### Manual Migration (Not Recommended) + +If you need to manually run migrations for debugging: + +```bash +./env/bin/alembic upgrade head +``` + +However, the application normally handles this automatically. diff --git a/meshview/models.py b/meshview/models.py index 2937a30..2825dba 100644 --- a/meshview/models.py +++ b/meshview/models.py @@ -103,6 +103,7 @@ class Traceroute(Base): done: Mapped[bool] = mapped_column(nullable=True) route: Mapped[bytes] = mapped_column(nullable=True) import_time: Mapped[datetime] = mapped_column(nullable=True) + route_return: Mapped[bytes] = mapped_column(nullable=True) import_time_us: Mapped[int] = mapped_column(BigInteger, nullable=True) __table_args__ = ( diff --git a/meshview/mqtt_store.py b/meshview/mqtt_store.py index 8ebcfb5..baca633 100644 --- a/meshview/mqtt_store.py +++ b/meshview/mqtt_store.py @@ -199,15 +199,7 @@ async def process_envelope(topic, env): # --- TRACEROUTE_APP (no conflict handling, normal insert) if env.packet.decoded.portnum == PortNum.TRACEROUTE_APP: - packet_id = None - if env.packet.decoded.want_response: - packet_id = env.packet.id - else: - result = await session.execute( - select(Packet).where(Packet.id == env.packet.decoded.request_id) - ) - if result.scalar_one_or_none(): - packet_id = env.packet.decoded.request_id + packet_id = env.packet.id if packet_id is not None: now = datetime.datetime.now(datetime.UTC) now_us = int(now.timestamp() * 1_000_000) diff --git a/meshview/templates/packet.html b/meshview/templates/packet.html index dfcfe3a..b3de808 100644 --- a/meshview/templates/packet.html +++ b/meshview/templates/packet.html @@ -56,9 +56,10 @@ {% endfor %} {% if packet.raw_mesh_packet.decoded.want_response %} - graph + graph route + {% else %} - graph + graph route return {% endif %} {% endif %} 
{{packet.payload}}