From 57f51c741cbca99cf73c27eb94d4791a75d41a06 Mon Sep 17 00:00:00 2001 From: Louis King Date: Mon, 8 Dec 2025 15:13:24 +0000 Subject: [PATCH] Fixed Member model --- ...d9_add_member_id_field_to_members_table.py | 111 ++++++++++++ ..._aa1162502616_remove_member_nodes_table.py | 57 ++++++ docker-compose.yml | 12 ++ example/seed/members.yaml | 18 +- src/meshcore_hub/api/routes/members.py | 167 +++--------------- src/meshcore_hub/api/routes/nodes.py | 28 ++- src/meshcore_hub/collector/cli.py | 29 +-- src/meshcore_hub/collector/member_import.py | 90 +++------- src/meshcore_hub/common/models/__init__.py | 2 - src/meshcore_hub/common/models/member.py | 25 ++- src/meshcore_hub/common/models/member_node.py | 56 ------ src/meshcore_hub/common/schemas/members.py | 75 +++----- src/meshcore_hub/common/schemas/nodes.py | 6 +- src/meshcore_hub/web/routes/members.py | 60 ++++++- src/meshcore_hub/web/templates/members.html | 22 ++- src/meshcore_hub/web/templates/nodes.html | 2 +- tests/test_web/conftest.py | 55 ++++-- 17 files changed, 422 insertions(+), 393 deletions(-) create mode 100644 alembic/versions/20251208_1434_03b9b2451bd9_add_member_id_field_to_members_table.py create mode 100644 alembic/versions/20251208_1504_aa1162502616_remove_member_nodes_table.py delete mode 100644 src/meshcore_hub/common/models/member_node.py diff --git a/alembic/versions/20251208_1434_03b9b2451bd9_add_member_id_field_to_members_table.py b/alembic/versions/20251208_1434_03b9b2451bd9_add_member_id_field_to_members_table.py new file mode 100644 index 0000000..da396af --- /dev/null +++ b/alembic/versions/20251208_1434_03b9b2451bd9_add_member_id_field_to_members_table.py @@ -0,0 +1,111 @@ +"""Add member_id field to members table + +Revision ID: 03b9b2451bd9 +Revises: 0b944542ccd8 +Create Date: 2025-12-08 14:34:30.337799+00:00 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "03b9b2451bd9" +down_revision: Union[str, None] = "0b944542ccd8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("advertisements", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_advertisements_event_hash_unique")) + batch_op.create_unique_constraint( + "uq_advertisements_event_hash", ["event_hash"] + ) + + with op.batch_alter_table("members", schema=None) as batch_op: + # Add member_id as nullable first to handle existing data + batch_op.add_column( + sa.Column("member_id", sa.String(length=100), nullable=True) + ) + + # Generate member_id for existing members based on their name + # Convert name to lowercase and replace spaces with underscores + connection = op.get_bind() + connection.execute( + sa.text( + "UPDATE members SET member_id = LOWER(REPLACE(name, ' ', '_')) WHERE member_id IS NULL" + ) + ) + + with op.batch_alter_table("members", schema=None) as batch_op: + # Now make it non-nullable and add unique index + batch_op.alter_column("member_id", nullable=False) + batch_op.drop_index(batch_op.f("ix_members_name")) + batch_op.create_index( + batch_op.f("ix_members_member_id"), ["member_id"], unique=True + ) + + with op.batch_alter_table("messages", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_messages_event_hash_unique")) + batch_op.create_unique_constraint("uq_messages_event_hash", ["event_hash"]) + + with op.batch_alter_table("nodes", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_nodes_public_key")) + batch_op.create_index( + batch_op.f("ix_nodes_public_key"), ["public_key"], unique=True + ) + + with op.batch_alter_table("telemetry", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_telemetry_event_hash_unique")) + batch_op.create_unique_constraint("uq_telemetry_event_hash", ["event_hash"]) + + with op.batch_alter_table("trace_paths", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_trace_paths_event_hash_unique")) + batch_op.create_unique_constraint("uq_trace_paths_event_hash", ["event_hash"]) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("trace_paths", schema=None) as batch_op: + batch_op.drop_constraint("uq_trace_paths_event_hash", type_="unique") + batch_op.create_index( + batch_op.f("ix_trace_paths_event_hash_unique"), ["event_hash"], unique=1 + ) + + with op.batch_alter_table("telemetry", schema=None) as batch_op: + batch_op.drop_constraint("uq_telemetry_event_hash", type_="unique") + batch_op.create_index( + batch_op.f("ix_telemetry_event_hash_unique"), ["event_hash"], unique=1 + ) + + with op.batch_alter_table("nodes", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_nodes_public_key")) + batch_op.create_index( + batch_op.f("ix_nodes_public_key"), ["public_key"], unique=False + ) + + with op.batch_alter_table("messages", schema=None) as batch_op: + batch_op.drop_constraint("uq_messages_event_hash", type_="unique") + batch_op.create_index( + batch_op.f("ix_messages_event_hash_unique"), ["event_hash"], unique=1 + ) + + with op.batch_alter_table("members", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_members_member_id")) + batch_op.create_index(batch_op.f("ix_members_name"), ["name"], unique=False) + batch_op.drop_column("member_id") + + with op.batch_alter_table("advertisements", schema=None) as batch_op: + batch_op.drop_constraint("uq_advertisements_event_hash", type_="unique") + batch_op.create_index( + batch_op.f("ix_advertisements_event_hash_unique"), ["event_hash"], unique=1 + ) + + # ### end Alembic commands ### diff --git a/alembic/versions/20251208_1504_aa1162502616_remove_member_nodes_table.py b/alembic/versions/20251208_1504_aa1162502616_remove_member_nodes_table.py new file mode 100644 index 0000000..5f98aef --- /dev/null +++ b/alembic/versions/20251208_1504_aa1162502616_remove_member_nodes_table.py @@ -0,0 +1,57 @@ +"""Remove member_nodes table + +Revision ID: aa1162502616 +Revises: 03b9b2451bd9 +Create Date: 2025-12-08 15:04:37.260923+00:00 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "aa1162502616" +down_revision: Union[str, None] = "03b9b2451bd9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Drop the member_nodes table + # Nodes are now associated with members via a 'member_id' tag on the node + op.drop_table("member_nodes") + + +def downgrade() -> None: + # Recreate the member_nodes table if needed for rollback + op.create_table( + "member_nodes", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("member_id", sa.String(length=36), nullable=False), + sa.Column("public_key", sa.String(length=64), nullable=False), + sa.Column("node_role", sa.String(length=50), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["member_id"], + ["members.id"], + name=op.f("fk_member_nodes_member_id_members"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_member_nodes")), + ) + op.create_index( + op.f("ix_member_nodes_member_id"), "member_nodes", ["member_id"], unique=False + ) + op.create_index( + op.f("ix_member_nodes_public_key"), "member_nodes", ["public_key"], unique=False + ) + op.create_index( + "ix_member_nodes_member_public_key", + "member_nodes", + ["member_id", "public_key"], + unique=False, + ) diff --git a/docker-compose.yml b/docker-compose.yml index 612e988..4692227 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -138,6 +138,9 @@ services: - all - core restart: unless-stopped + depends_on: + seed: + condition: service_completed_successfully volumes: - ${DATA_HOME:-./data}:/data - ${SEED_HOME:-./seed}:/seed @@ -193,6 +196,8 @@ services: - core restart: unless-stopped depends_on: + seed: + condition: service_completed_successfully collector: condition: service_started ports: @@ -274,7 +279,9 @@ services: container_name: meshcore-db-migrate profiles: - all + - core - migrate + restart: "no" volumes: # Mount data directory (uses collector/meshcore.db) - ${DATA_HOME:-./data}:/data @@ -295,7 +302,12 @@ services: container_name: meshcore-seed profiles: - all + - core - seed + restart: "no" + depends_on: + db-migrate: + condition: service_completed_successfully volumes: # Mount data directory for database (read-write) - ${DATA_HOME:-./data}:/data diff --git a/example/seed/members.yaml b/example/seed/members.yaml index 087c9fc..da16900 100644 --- a/example/seed/members.yaml +++ b/example/seed/members.yaml @@ -1,16 +1,14 @@ # Example members seed file -# Each member can have multiple nodes with different roles (chat, repeater, etc.) +# Note: Nodes are associated with members via a 'member_id' tag on the node. +# Use node_tags.yaml to set member_id tags on nodes. members: - - name: Example Member + - member_id: example_member + name: Example Member callsign: N0CALL role: Network Operator - description: Example member entry with multiple nodes - nodes: - - public_key: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef - node_role: chat - - public_key: fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210 - node_role: repeater - - name: Simple Member + description: Example network operator member + - member_id: simple_member + name: Simple Member callsign: N0CALL2 role: Observer - description: Member without any nodes + description: Example observer member diff --git a/src/meshcore_hub/api/routes/members.py b/src/meshcore_hub/api/routes/members.py index f817afd..e1c544c 100644 --- a/src/meshcore_hub/api/routes/members.py +++ b/src/meshcore_hub/api/routes/members.py @@ -2,15 +2,13 @@ from fastapi import APIRouter, HTTPException, Query from sqlalchemy import func, select -from sqlalchemy.orm import selectinload from meshcore_hub.api.auth import RequireAdmin, RequireRead from meshcore_hub.api.dependencies import DbSession -from meshcore_hub.common.models import Member, MemberNode, Node +from meshcore_hub.common.models import Member from meshcore_hub.common.schemas.members import ( MemberCreate, MemberList, - MemberNodeRead, MemberRead, MemberUpdate, ) @@ -18,50 +16,6 @@ from meshcore_hub.common.schemas.members import ( router = APIRouter() -def _enrich_member_nodes( - member: Member, node_info: dict[str, dict] -) -> list[MemberNodeRead]: - """Enrich member nodes with node details from the database. - - Args: - member: The member with nodes to enrich - node_info: Dict mapping public_key to node details - - Returns: - List of MemberNodeRead with node details populated - """ - enriched_nodes = [] - for mn in member.nodes: - info = node_info.get(mn.public_key, {}) - enriched_nodes.append( - MemberNodeRead( - public_key=mn.public_key, - node_role=mn.node_role, - created_at=mn.created_at, - updated_at=mn.updated_at, - node_name=info.get("name"), - node_adv_type=info.get("adv_type"), - tag_name=info.get("tag_name"), - ) - ) - return enriched_nodes - - -def _member_to_read(member: Member, node_info: dict[str, dict]) -> MemberRead: - """Convert a Member model to MemberRead with enriched node data.""" - return MemberRead( - id=member.id, - name=member.name, - callsign=member.callsign, - role=member.role, - description=member.description, - contact=member.contact, - nodes=_enrich_member_nodes(member, node_info), - created_at=member.created_at, - updated_at=member.updated_at, - ) - - @router.get("", response_model=MemberList) async def list_members( _: RequireRead, @@ -74,45 +28,12 @@ async def list_members( count_query = select(func.count()).select_from(Member) total = session.execute(count_query).scalar() or 0 - # Get members with nodes eagerly loaded - query = ( - select(Member) - .options(selectinload(Member.nodes)) - .order_by(Member.name) - .limit(limit) - .offset(offset) - ) + # Get members ordered by name + query = select(Member).order_by(Member.name).limit(limit).offset(offset) members = list(session.execute(query).scalars().all()) - # Collect all public keys from member nodes - all_public_keys = set() - for m in members: - for mn in m.nodes: - all_public_keys.add(mn.public_key) - - # Fetch node info for all public keys in one query - node_info: dict[str, dict] = {} - if all_public_keys: - node_query = ( - select(Node) - .options(selectinload(Node.tags)) - .where(Node.public_key.in_(all_public_keys)) - ) - nodes = session.execute(node_query).scalars().all() - for node in nodes: - tag_name = None - for tag in node.tags: - if tag.key == "name": - tag_name = tag.value - break - node_info[node.public_key] = { - "name": node.name, - "adv_type": node.adv_type, - "tag_name": tag_name, - } - return MemberList( - items=[_member_to_read(m, node_info) for m in members], + items=[MemberRead.model_validate(m) for m in members], total=total, limit=limit, offset=offset, @@ -126,37 +47,13 @@ async def get_member( member_id: str, ) -> MemberRead: """Get a specific member by ID.""" - query = ( - select(Member).options(selectinload(Member.nodes)).where(Member.id == member_id) - ) + query = select(Member).where(Member.id == member_id) member = session.execute(query).scalar_one_or_none() if not member: raise HTTPException(status_code=404, detail="Member not found") - # Fetch node info for member's nodes - node_info: dict[str, dict] = {} - public_keys = [mn.public_key for mn in member.nodes] - if public_keys: - node_query = ( - select(Node) - .options(selectinload(Node.tags)) - .where(Node.public_key.in_(public_keys)) - ) - nodes = session.execute(node_query).scalars().all() - for node in nodes: - tag_name = None - for tag in node.tags: - if tag.key == "name": - tag_name = tag.value - break - node_info[node.public_key] = { - "name": node.name, - "adv_type": node.adv_type, - "tag_name": tag_name, - } - - return _member_to_read(member, node_info) + return MemberRead.model_validate(member) @router.post("", response_model=MemberRead, status_code=201) @@ -166,8 +63,18 @@ async def create_member( member: MemberCreate, ) -> MemberRead: """Create a new member.""" + # Check if member_id already exists + query = select(Member).where(Member.member_id == member.member_id) + existing = session.execute(query).scalar_one_or_none() + if existing: + raise HTTPException( + status_code=400, + detail=f"Member with member_id '{member.member_id}' already exists", + ) + # Create member new_member = Member( + member_id=member.member_id, name=member.name, callsign=member.callsign, role=member.role, @@ -175,18 +82,6 @@ async def create_member( contact=member.contact, ) session.add(new_member) - session.flush() # Get the ID for the member - - # Add nodes if provided - if member.nodes: - for node_data in member.nodes: - node = MemberNode( - member_id=new_member.id, - public_key=node_data.public_key.lower(), - node_role=node_data.node_role, - ) - session.add(node) - session.commit() session.refresh(new_member) @@ -201,15 +96,25 @@ async def update_member( member: MemberUpdate, ) -> MemberRead: """Update a member.""" - query = ( - select(Member).options(selectinload(Member.nodes)).where(Member.id == member_id) - ) + query = select(Member).where(Member.id == member_id) existing = session.execute(query).scalar_one_or_none() if not existing: raise HTTPException(status_code=404, detail="Member not found") # Update fields + if member.member_id is not None: + # Check if new member_id is already taken by another member + check_query = select(Member).where( + Member.member_id == member.member_id, Member.id != member_id + ) + collision = session.execute(check_query).scalar_one_or_none() + if collision: + raise HTTPException( + status_code=400, + detail=f"Member with member_id '{member.member_id}' already exists", + ) + existing.member_id = member.member_id if member.name is not None: existing.name = member.name if member.callsign is not None: @@ -221,20 +126,6 @@ async def update_member( if member.contact is not None: existing.contact = member.contact - # Update nodes if provided (replaces existing nodes) - if member.nodes is not None: - # Clear existing nodes - existing.nodes.clear() - - # Add new nodes - for node_data in member.nodes: - node = MemberNode( - member_id=existing.id, - public_key=node_data.public_key.lower(), - node_role=node_data.node_role, - ) - existing.nodes.append(node) - session.commit() session.refresh(existing) diff --git a/src/meshcore_hub/api/routes/nodes.py b/src/meshcore_hub/api/routes/nodes.py index d82e874..a8ca922 100644 --- a/src/meshcore_hub/api/routes/nodes.py +++ b/src/meshcore_hub/api/routes/nodes.py @@ -3,11 +3,12 @@ from typing import Optional from fastapi import APIRouter, HTTPException, Query -from sqlalchemy import func, select +from sqlalchemy import func, or_, select +from sqlalchemy.orm import selectinload from meshcore_hub.api.auth import RequireRead from meshcore_hub.api.dependencies import DbSession -from meshcore_hub.common.models import Node +from meshcore_hub.common.models import Node, NodeTag from meshcore_hub.common.schemas.nodes import NodeList, NodeRead router = APIRouter() @@ -17,18 +18,31 @@ router = APIRouter() async def list_nodes( _: RequireRead, session: DbSession, - search: Optional[str] = Query(None, description="Search in name or public key"), + search: Optional[str] = Query( + None, description="Search in name tag, node name, or public key" + ), adv_type: Optional[str] = Query(None, description="Filter by advertisement type"), limit: int = Query(50, ge=1, le=500, description="Page size"), offset: int = Query(0, ge=0, description="Page offset"), ) -> NodeList: """List all nodes with pagination and filtering.""" - # Build query - query = select(Node) + # Build base query with tags loaded + query = select(Node).options(selectinload(Node.tags)) if search: + # Search in public key, node name, or name tag + # For name tag search, we need to join with NodeTag + search_pattern = f"%{search}%" query = query.where( - (Node.name.ilike(f"%{search}%")) | (Node.public_key.ilike(f"%{search}%")) + or_( + Node.public_key.ilike(search_pattern), + Node.name.ilike(search_pattern), + Node.id.in_( + select(NodeTag.node_id).where( + NodeTag.key == "name", NodeTag.value.ilike(search_pattern) + ) + ), + ) ) if adv_type: @@ -38,7 +52,7 @@ async def list_nodes( count_query = select(func.count()).select_from(query.subquery()) total = session.execute(count_query).scalar() or 0 - # Apply pagination + # Apply pagination and ordering query = query.order_by(Node.last_seen.desc()).offset(offset).limit(limit) # Execute diff --git a/src/meshcore_hub/collector/cli.py b/src/meshcore_hub/collector/cli.py index 9a61e14..e8ac94d 100644 --- a/src/meshcore_hub/collector/cli.py +++ b/src/meshcore_hub/collector/cli.py @@ -170,8 +170,8 @@ def _run_collector_service( ) -> None: """Run the collector service. - On startup, automatically seeds the database from YAML files in seed_home - if they exist. + Note: Seed data import should be done using the 'meshcore-hub collector seed' + command or the dedicated seed container before starting the collector service. Webhooks can be configured via environment variables: - WEBHOOK_ADVERTISEMENT_URL: Webhook for advertisement events @@ -193,31 +193,6 @@ def _run_collector_service( click.echo(f"MQTT: {mqtt_host}:{mqtt_port} (prefix: {prefix})") click.echo(f"Database: {database_url}") - # Initialize database (schema managed by Alembic migrations) - from meshcore_hub.common.database import DatabaseManager - - db = DatabaseManager(database_url) - - # Auto-seed from seed files on startup - click.echo("") - click.echo("Checking for seed files...") - seed_home_path = Path(seed_home) - node_tags_exists = (seed_home_path / "node_tags.yaml").exists() - members_exists = (seed_home_path / "members.yaml").exists() - - if node_tags_exists or members_exists: - click.echo("Running seed import...") - _run_seed_import( - seed_home=seed_home, - db=db, - create_nodes=True, - verbose=True, - ) - else: - click.echo(f"No seed files found in {seed_home}") - - db.dispose() - # Load webhook configuration from settings from meshcore_hub.collector.webhook import ( WebhookDispatcher, diff --git a/src/meshcore_hub/collector/member_import.py b/src/meshcore_hub/collector/member_import.py index 7898150..d0c0cf9 100644 --- a/src/meshcore_hub/collector/member_import.py +++ b/src/meshcore_hub/collector/member_import.py @@ -5,41 +5,28 @@ from pathlib import Path from typing import Any, Optional import yaml -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field from sqlalchemy import select from meshcore_hub.common.database import DatabaseManager -from meshcore_hub.common.models import Member, MemberNode +from meshcore_hub.common.models import Member logger = logging.getLogger(__name__) -class NodeData(BaseModel): - """Schema for a node entry in the member import file.""" - - public_key: str = Field(..., min_length=64, max_length=64) - node_role: Optional[str] = Field(default=None, max_length=50) - - @field_validator("public_key") - @classmethod - def validate_public_key(cls, v: str) -> str: - """Validate and normalize public key.""" - if len(v) != 64: - raise ValueError(f"public_key must be 64 characters, got {len(v)}") - if not all(c in "0123456789abcdefABCDEF" for c in v): - raise ValueError("public_key must be a valid hex string") - return v.lower() - - class MemberData(BaseModel): - """Schema for a member entry in the import file.""" + """Schema for a member entry in the import file. + Note: Nodes are associated with members via a 'member_id' tag on the node, + not through this schema. + """ + + member_id: str = Field(..., min_length=1, max_length=100) name: str = Field(..., min_length=1, max_length=255) callsign: Optional[str] = Field(default=None, max_length=20) role: Optional[str] = Field(default=None, max_length=100) description: Optional[str] = Field(default=None) contact: Optional[str] = Field(default=None, max_length=255) - nodes: Optional[list[NodeData]] = Field(default=None) def load_members_file(file_path: str | Path) -> list[dict[str, Any]]: @@ -48,20 +35,16 @@ def load_members_file(file_path: str | Path) -> list[dict[str, Any]]: Supports two formats: 1. List of member objects: - - name: Member 1 + - member_id: member1 + name: Member 1 callsign: M1 - nodes: - - public_key: abc123... - node_role: chat 2. Object with "members" key: members: - - name: Member 1 + - member_id: member1 + name: Member 1 callsign: M1 - nodes: - - public_key: abc123... - node_role: chat Args: file_path: Path to the members YAML file @@ -96,6 +79,8 @@ def load_members_file(file_path: str | Path) -> list[dict[str, Any]]: for i, member in enumerate(members_list): if not isinstance(member, dict): raise ValueError(f"Member at index {i} must be an object") + if "member_id" not in member: + raise ValueError(f"Member at index {i} must have a 'member_id' field") if "name" not in member: raise ValueError(f"Member at index {i} must have a 'name' field") @@ -115,9 +100,11 @@ def import_members( ) -> dict[str, Any]: """Import members from a YAML file into the database. - Performs upsert operations based on name - existing members are updated, - new members are created. Nodes are synced (existing nodes removed and - replaced with new ones from the file). + Performs upsert operations based on member_id - existing members are updated, + new members are created. + + Note: Nodes are associated with members via a 'member_id' tag on the node. + This import does not manage node associations. Args: file_path: Path to the members YAML file @@ -149,14 +136,17 @@ def import_members( with db.session_scope() as session: for member_data in members_data: try: + member_id = member_data["member_id"] name = member_data["name"] - # Find existing member by name - query = select(Member).where(Member.name == name) + # Find existing member by member_id + query = select(Member).where(Member.member_id == member_id) existing = session.execute(query).scalar_one_or_none() if existing: # Update existing member + if member_data.get("name") is not None: + existing.name = member_data["name"] if member_data.get("callsign") is not None: existing.callsign = member_data["callsign"] if member_data.get("role") is not None: @@ -166,25 +156,12 @@ def import_members( if member_data.get("contact") is not None: existing.contact = member_data["contact"] - # Sync nodes if provided - if member_data.get("nodes") is not None: - # Remove existing nodes - existing.nodes.clear() - - # Add new nodes - for node_data in member_data["nodes"]: - node = MemberNode( - member_id=existing.id, - public_key=node_data["public_key"], - node_role=node_data.get("node_role"), - ) - existing.nodes.append(node) - stats["updated"] += 1 - logger.debug(f"Updated member: {name}") + logger.debug(f"Updated member: {member_id} ({name})") else: # Create new member new_member = Member( + member_id=member_id, name=name, callsign=member_data.get("callsign"), role=member_data.get("role"), @@ -192,23 +169,12 @@ def import_members( contact=member_data.get("contact"), ) session.add(new_member) - session.flush() # Get the ID for the member - - # Add nodes if provided - if member_data.get("nodes"): - for node_data in member_data["nodes"]: - node = MemberNode( - member_id=new_member.id, - public_key=node_data["public_key"], - node_role=node_data.get("node_role"), - ) - session.add(node) stats["created"] += 1 - logger.debug(f"Created member: {name}") + logger.debug(f"Created member: {member_id} ({name})") except Exception as e: - error_msg = f"Error processing member '{member_data.get('name', 'unknown')}': {e}" + error_msg = f"Error processing member '{member_data.get('member_id', 'unknown')}' ({member_data.get('name', 'unknown')}): {e}" stats["errors"].append(error_msg) logger.error(error_msg) diff --git a/src/meshcore_hub/common/models/__init__.py b/src/meshcore_hub/common/models/__init__.py index 69eaeb9..f0f1dd9 100644 --- a/src/meshcore_hub/common/models/__init__.py +++ b/src/meshcore_hub/common/models/__init__.py @@ -9,7 +9,6 @@ from meshcore_hub.common.models.trace_path import TracePath from meshcore_hub.common.models.telemetry import Telemetry from meshcore_hub.common.models.event_log import EventLog from meshcore_hub.common.models.member import Member -from meshcore_hub.common.models.member_node import MemberNode from meshcore_hub.common.models.event_receiver import EventReceiver, add_event_receiver __all__ = [ @@ -23,7 +22,6 @@ __all__ = [ "Telemetry", "EventLog", "Member", - "MemberNode", "EventReceiver", "add_event_receiver", ] diff --git a/src/meshcore_hub/common/models/member.py b/src/meshcore_hub/common/models/member.py index a33a329..fdf7c91 100644 --- a/src/meshcore_hub/common/models/member.py +++ b/src/meshcore_hub/common/models/member.py @@ -1,36 +1,39 @@ """Member model for network member information.""" -from typing import TYPE_CHECKING, Optional +from typing import Optional from sqlalchemy import String, Text -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column from meshcore_hub.common.models.base import Base, TimestampMixin, UUIDMixin -if TYPE_CHECKING: - from meshcore_hub.common.models.member_node import MemberNode - class Member(Base, UUIDMixin, TimestampMixin): """Member model for network member information. Stores information about network members/operators. - Members can have multiple associated nodes (chat, repeater, etc.). + Nodes are associated with members via a 'member_id' tag on the node. Attributes: id: UUID primary key + member_id: Unique member identifier (e.g., 'walshie86') name: Member's display name callsign: Amateur radio callsign (optional) role: Member's role in the network (optional) description: Additional description (optional) contact: Contact information (optional) - nodes: List of associated MemberNode records created_at: Record creation timestamp updated_at: Record update timestamp """ __tablename__ = "members" + member_id: Mapped[str] = mapped_column( + String(100), + nullable=False, + unique=True, + index=True, + ) name: Mapped[str] = mapped_column( String(255), nullable=False, @@ -52,11 +55,5 @@ class Member(Base, UUIDMixin, TimestampMixin): nullable=True, ) - # Relationship to member nodes - nodes: Mapped[list["MemberNode"]] = relationship( - back_populates="member", - cascade="all, delete-orphan", - ) - def __repr__(self) -> str: - return f"" + return f"" diff --git a/src/meshcore_hub/common/models/member_node.py b/src/meshcore_hub/common/models/member_node.py deleted file mode 100644 index 665d775..0000000 --- a/src/meshcore_hub/common/models/member_node.py +++ /dev/null @@ -1,56 +0,0 @@ -"""MemberNode model for associating nodes with members.""" - -from typing import TYPE_CHECKING, Optional - -from sqlalchemy import ForeignKey, String, Index -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from meshcore_hub.common.models.base import Base, TimestampMixin, UUIDMixin - -if TYPE_CHECKING: - from meshcore_hub.common.models.member import Member - - -class MemberNode(Base, UUIDMixin, TimestampMixin): - """Association model linking members to their nodes. - - A member can have multiple nodes (e.g., chat node, repeater). - Each node is identified by its public_key and has a role. - - Attributes: - id: UUID primary key - member_id: Foreign key to the member - public_key: Node's public key (64-char hex) - node_role: Role of the node (e.g., 'chat', 'repeater') - created_at: Record creation timestamp - updated_at: Record update timestamp - """ - - __tablename__ = "member_nodes" - - member_id: Mapped[str] = mapped_column( - String(36), - ForeignKey("members.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - public_key: Mapped[str] = mapped_column( - String(64), - nullable=False, - index=True, - ) - node_role: Mapped[Optional[str]] = mapped_column( - String(50), - nullable=True, - ) - - # Relationship back to member - member: Mapped["Member"] = relationship(back_populates="nodes") - - # Composite index for efficient lookups - __table_args__ = ( - Index("ix_member_nodes_member_public_key", "member_id", "public_key"), - ) - - def __repr__(self) -> str: - return f"" diff --git a/src/meshcore_hub/common/schemas/members.py b/src/meshcore_hub/common/schemas/members.py index 1748f28..a444127 100644 --- a/src/meshcore_hub/common/schemas/members.py +++ b/src/meshcore_hub/common/schemas/members.py @@ -6,44 +6,19 @@ from typing import Optional from pydantic import BaseModel, Field -class MemberNodeCreate(BaseModel): - """Schema for creating a member node association.""" - - public_key: str = Field( - ..., - min_length=64, - max_length=64, - pattern=r"^[0-9a-fA-F]{64}$", - description="Node's public key (64-char hex)", - ) - node_role: Optional[str] = Field( - default=None, - max_length=50, - description="Role of the node (e.g., 'chat', 'repeater')", - ) - - -class MemberNodeRead(BaseModel): - """Schema for reading a member node association.""" - - public_key: str = Field(..., description="Node's public key") - node_role: Optional[str] = Field(default=None, description="Role of the node") - created_at: datetime = Field(..., description="Creation timestamp") - updated_at: datetime = Field(..., description="Last update timestamp") - # Node details (populated from nodes table if available) - node_name: Optional[str] = Field(default=None, description="Node's name from DB") - node_adv_type: Optional[str] = Field( - default=None, description="Node's advertisement type" - ) - tag_name: Optional[str] = Field(default=None, description="Node's name tag") - - class Config: - from_attributes = True - - class MemberCreate(BaseModel): - """Schema for creating a member.""" + """Schema for creating a member. + Note: Nodes are associated with members via a 'member_id' tag on the node, + not through this schema. + """ + + member_id: str = Field( + ..., + min_length=1, + max_length=100, + description="Unique member identifier (e.g., 'walshie86')", + ) name: str = Field( ..., min_length=1, @@ -69,15 +44,21 @@ class MemberCreate(BaseModel): max_length=255, description="Contact information", ) - nodes: Optional[list[MemberNodeCreate]] = Field( - default=None, - description="List of associated nodes", - ) class MemberUpdate(BaseModel): - """Schema for updating a member.""" + """Schema for updating a member. + Note: Nodes are associated with members via a 'member_id' tag on the node, + not through this schema. + """ + + member_id: Optional[str] = Field( + default=None, + min_length=1, + max_length=100, + description="Unique member identifier (e.g., 'walshie86')", + ) name: Optional[str] = Field( default=None, min_length=1, @@ -103,22 +84,22 @@ class MemberUpdate(BaseModel): max_length=255, description="Contact information", ) - nodes: Optional[list[MemberNodeCreate]] = Field( - default=None, - description="List of associated nodes (replaces existing nodes)", - ) class MemberRead(BaseModel): - """Schema for reading a member.""" + """Schema for reading a member. + + Note: Nodes are associated with members via a 'member_id' tag on the node. + To find nodes for a member, query nodes with a 'member_id' tag matching this member. + """ id: str = Field(..., description="Member UUID") + member_id: str = Field(..., description="Unique member identifier") name: str = Field(..., description="Member's display name") callsign: Optional[str] = Field(default=None, description="Amateur radio callsign") role: Optional[str] = Field(default=None, description="Member's role") description: Optional[str] = Field(default=None, description="Description") contact: Optional[str] = Field(default=None, description="Contact information") - nodes: list[MemberNodeRead] = Field(default=[], description="Associated nodes") created_at: datetime = Field(..., description="Creation timestamp") updated_at: datetime = Field(..., description="Last update timestamp") diff --git a/src/meshcore_hub/common/schemas/nodes.py b/src/meshcore_hub/common/schemas/nodes.py index 7844083..c5fed41 100644 --- a/src/meshcore_hub/common/schemas/nodes.py +++ b/src/meshcore_hub/common/schemas/nodes.py @@ -59,7 +59,9 @@ class NodeRead(BaseModel): adv_type: Optional[str] = Field(default=None, description="Advertisement type") flags: Optional[int] = Field(default=None, description="Capability flags") first_seen: datetime = Field(..., description="First advertisement timestamp") - last_seen: datetime = Field(..., description="Last activity timestamp") + last_seen: Optional[datetime] = Field( + default=None, description="Last activity timestamp" + ) created_at: datetime = Field(..., description="Record creation timestamp") updated_at: datetime = Field(..., description="Record update timestamp") tags: list[NodeTagRead] = Field(default_factory=list, description="Node tags") @@ -82,7 +84,7 @@ class NodeFilters(BaseModel): search: Optional[str] = Field( default=None, - description="Search in name or public key", + description="Search in name tag, node name, or public key", ) adv_type: Optional[str] = Field( default=None, diff --git a/src/meshcore_hub/web/routes/members.py b/src/meshcore_hub/web/routes/members.py index 1731d52..2c146ba 100644 --- a/src/meshcore_hub/web/routes/members.py +++ b/src/meshcore_hub/web/routes/members.py @@ -23,24 +23,72 @@ async def members_page(request: Request) -> HTMLResponse: def node_sort_key(node: dict) -> int: """Sort nodes: repeater first, then chat, then others.""" - role = (node.get("node_role") or "").lower() - if role == "repeater": + adv_type = (node.get("adv_type") or "").lower() + if adv_type == "repeater": return 0 - if role == "chat": + if adv_type == "chat": return 1 return 2 try: + # Fetch all members response = await request.app.state.http_client.get( "/api/v1/members", params={"limit": 100} ) if response.status_code == 200: data = response.json() members = data.get("items", []) - # Sort nodes within each member (repeater first, then chat) + + # Fetch all nodes with member_id tags in one query + nodes_response = await request.app.state.http_client.get( + "/api/v1/nodes", params={"has_tag": "member_id", "limit": 500} + ) + + # Build a map of member_id -> nodes + member_nodes_map: dict[str, list] = {} + if nodes_response.status_code == 200: + nodes_data = nodes_response.json() + all_nodes = nodes_data.get("items", []) + + for node in all_nodes: + # Find member_id tag + for tag in node.get("tags", []): + if tag.get("key") == "member_id": + member_id_value = tag.get("value") + if member_id_value: + if member_id_value not in member_nodes_map: + member_nodes_map[member_id_value] = [] + member_nodes_map[member_id_value].append(node) + break + + # Assign nodes to members and sort for member in members: - if member.get("nodes"): - member["nodes"] = sorted(member["nodes"], key=node_sort_key) + member_id = member.get("member_id") + if member_id and member_id in member_nodes_map: + # Sort nodes (repeater first, then chat, then by name tag) + nodes = member_nodes_map[member_id] + + # Sort by advertisement type first, then by name + def full_sort_key(node: dict) -> tuple: + adv_type = (node.get("adv_type") or "").lower() + type_priority = ( + 0 + if adv_type == "repeater" + else (1 if adv_type == "chat" else 2) + ) + + # Get name from tags + node_name = node.get("name") or "" + for tag in node.get("tags", []): + if tag.get("key") == "name": + node_name = tag.get("value") or node_name + break + + return (type_priority, node_name.lower()) + + member["nodes"] = sorted(nodes, key=full_sort_key) + else: + member["nodes"] = [] except Exception as e: logger.warning(f"Failed to fetch members from API: {e}") context["api_error"] = str(e) diff --git a/src/meshcore_hub/web/templates/members.html b/src/meshcore_hub/web/templates/members.html index f375a70..ab64314 100644 --- a/src/meshcore_hub/web/templates/members.html +++ b/src/meshcore_hub/web/templates/members.html @@ -33,7 +33,9 @@ {% if member.nodes %}
{% for node in member.nodes %} - {% set adv_type = node.node_adv_type or node.node_role %} + {% set adv_type = node.adv_type %} + {% set node_tag_name = node.tags|selectattr('key', 'equalto', 'name')|map(attribute='value')|first %} + {% set display_name = node_tag_name or node.name %} {% if adv_type and adv_type|lower == 'chat' %} @@ -49,8 +51,8 @@ {% endif %}
- {% if node.friendly_name or node.node_name %} -
{{ node.friendly_name or node.node_name }}
+ {% if display_name %} +
{{ display_name }}
{{ node.public_key[:12] }}...
{% else %}
{{ node.public_key[:12] }}...
@@ -80,18 +82,20 @@

Members File Format

Create a YAML file at $SEED_HOME/members.yaml with the following structure:

members:
-  - name: John Doe
+  - member_id: johndoe
+    name: John Doe
     callsign: AB1CD
     role: Network Admin
     description: Manages the main repeater node.
     contact: john@example.com
-    nodes:
-      - public_key: abc123def456...  # 64-char hex
-        node_role: repeater
-  - name: Jane Smith
+  - member_id: janesmith
+    name: Jane Smith
     role: Member
     description: Regular user in the downtown area.
-

Run meshcore-hub collector seed to import members, or they will be imported automatically on collector startup.

+

+ Run meshcore-hub collector seed to import members.
+ To associate nodes with members, add a member_id tag to nodes in node_tags.yaml. +

{% endif %} diff --git a/src/meshcore_hub/web/templates/nodes.html b/src/meshcore_hub/web/templates/nodes.html index 7221f5f..3ab0e22 100644 --- a/src/meshcore_hub/web/templates/nodes.html +++ b/src/meshcore_hub/web/templates/nodes.html @@ -25,7 +25,7 @@ - +