From df05c3a46212e20fdd54570b4e4fb5417a1ab25f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 4 Dec 2025 01:27:03 +0000 Subject: [PATCH] Convert collector seed mechanism from JSON to YAML - Replace JSON seed files with YAML format for better readability - Auto-detect YAML primitive types (number, boolean, string) from values - Add automatic seed import on collector startup - Split lat/lon into separate tags instead of combined coordinate string - Add PyYAML dependency and types-PyYAML for type checking - Update example/seed and contrib/seed/ipnet with clean YAML format - Update tests to verify YAML primitive type detection --- .pre-commit-config.yaml | 1 + contrib/seed/ipnet/members.json | 10 - contrib/seed/ipnet/members.yaml | 6 + contrib/seed/ipnet/node_tags.json | 613 -------------------- contrib/seed/ipnet/node_tags.yaml | 311 ++++++++++ example/seed/members.json | 10 - example/seed/members.yaml | 6 + example/seed/node_tags.json | 16 - example/seed/node_tags.yaml | 29 + pyproject.toml | 2 + src/meshcore_hub/collector/cli.py | 239 +++++--- src/meshcore_hub/collector/member_import.py | 29 +- src/meshcore_hub/collector/tag_import.py | 60 +- src/meshcore_hub/common/config.py | 10 +- tests/test_collector/test_tag_import.py | 96 +-- tests/test_common/test_config.py | 12 +- 16 files changed, 638 insertions(+), 812 deletions(-) delete mode 100755 contrib/seed/ipnet/members.json create mode 100644 contrib/seed/ipnet/members.yaml delete mode 100755 contrib/seed/ipnet/node_tags.json create mode 100644 contrib/seed/ipnet/node_tags.yaml delete mode 100644 example/seed/members.json create mode 100644 example/seed/members.yaml delete mode 100644 example/seed/node_tags.json create mode 100644 example/seed/node_tags.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5e8574c..43d24ac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,3 +38,4 @@ repos: - fastapi>=0.100.0 - alembic>=1.7.0 - types-paho-mqtt>=1.6.0 + - types-PyYAML>=6.0.0 diff --git a/contrib/seed/ipnet/members.json b/contrib/seed/ipnet/members.json deleted file mode 100755 index b91ed10..0000000 --- a/contrib/seed/ipnet/members.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "members": [ - { - "name": "Louis", - "callsign": "Louis", - "role": "admin", - "description": "IPNet Founder" - } - ] -} diff --git a/contrib/seed/ipnet/members.yaml b/contrib/seed/ipnet/members.yaml new file mode 100644 index 0000000..15207ee --- /dev/null +++ b/contrib/seed/ipnet/members.yaml @@ -0,0 +1,6 @@ +# IPNet Network Members +members: + - name: Louis + callsign: Louis + role: admin + description: IPNet Founder diff --git a/contrib/seed/ipnet/node_tags.json b/contrib/seed/ipnet/node_tags.json deleted file mode 100755 index 3a43d38..0000000 --- a/contrib/seed/ipnet/node_tags.json +++ /dev/null @@ -1,613 +0,0 @@ -{ - "2337484665ced7e210007e9fd9db98ced0a24a6eab8b4cbe3a06b3a1cea33ca1": { - "friendly_name": "IP2 Repeater 1", - "node_id": "ip2-rep01.ipnt.uk", - "member_id": "louis", - "area": "IP2", - "location": { - "value": "52.0357627,1.132079", - "type": "coordinate" - }, - "location_description": "Fountains Road", - "hardware": "Heltec V3", - "antenna": "Paradar 8.5dBi Omni", - "elevation": { - "value": "31", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "8cb01fff1afc099055af418ce5fc5e60384df9ff763c25dd7e6a5e0922e8df90": { - "friendly_name": "IP2 Repeater 2", - "node_id": "ip2-rep02.ipnt.uk", - "member_id": "louis", - "area": "IP2", - "location": { - "value": "52.0390682,1.1304141", - "type": "coordinate" - }, - "location_description": "Belstead Road", - "hardware": "Heltec V3", - "antenna": "McGill 6dBi Omni", - "elevation": { - "value": "44", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "5b565df747913358e24d890b2227de9c35d09763746b6ec326c15ebbf9b8be3b": { - "friendly_name": "IP2 Repeater 3", - "node_id": "ip2-rep03.ipnt.uk", - "member_id": "louis", - "area": "IP2", - "location": { - "value": "52.046356,1.134661", - "type": "coordinate" - }, - "location_description": "Birkfield Drive", - "hardware": "Heltec V3", - "antenna": "Paradar 8.5dBi Omni", - "elevation": { - "value": "52", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "780d0939f90b22d3bd7cbedcaf4e8d468a12c01886ab24b8cfa11eab2f5516c5": { - "friendly_name": "IP2 Integration 1", - "node_id": "ip2-int01.ipnt.uk", - "member_id": "louis", - "area": "IP2", - "location": { - "value": "52.0354539,1.1295338", - "type": "coordinate" - }, - "location_description": "Fountains Road", - "hardware": "Heltec V3", - "antenna": "Generic 5dBi Whip", - "elevation": { - "value": "25", - "type": "number" - }, - "show_on_map": { - "value": "false", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "integration" - }, - "30121dc60362c633c457ffa18f49b3e1d6823402c33709f32d7df70612250b96": { - "friendly_name": "MeshBot", - "node_id": "bot.ipnt.uk", - "member_id": "louis", - "area": "IP2", - "location": { - "value": "52.0354539,1.1295338", - "type": "coordinate" - }, - "location_description": "Fountains Road", - "hardware": "Heltec V3", - "antenna": "Generic 5dBi Whip", - "elevation": { - "value": "25", - "type": "number" - }, - "show_on_map": { - "value": "false", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "integration" - }, - "9135986b83815ada92883358435cc6528c7db60cb647f9b6547739a1ce5eb1c8": { - "friendly_name": "IP3 Repeater 1", - "node_id": "ip3-rep01.ipnt.uk", - "member_id": "markab", - "area": "IP3", - "location": { - "value": "52.045803,1.204416", - "type": "coordinate" - }, - "location_description": "Brokehall", - "hardware": "Heltec V3", - "antenna": "Paradar 8.5dBi Omni", - "elevation": { - "value": "42", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "e334ec5475789d542ed9e692fbeef7444a371fcc05adcbda1f47ba6a3191b459": { - "friendly_name": "IP3 Repeater 2", - "node_id": "ip3-rep02.ipnt.uk", - "member_id": "ccz", - "area": "IP3", - "location": { - "value": "52.03297,1.17543", - "type": "coordinate" - }, - "location_description": "Morland Road Allotments", - "hardware": "Heltec T114", - "antenna": "Unknown", - "elevation": { - "value": "39", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "cc15fb33e98f2e098a543f516f770dc3061a1a6b30f79b84780663bf68ae6b53": { - "friendly_name": "IP3 Repeater 3", - "node_id": "ip3-rep03.ipnt.uk", - "member_id": "ccz", - "area": "IP3", - "location": { - "value": "52.04499,1.18149", - "type": "coordinate" - }, - "location_description": "Hatfield Road", - "hardware": "Heltec V3", - "antenna": "Unknown", - "elevation": { - "value": "39", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "22309435fbd9dd1f14870a1895dc854779f6b2af72b08542f6105d264a493ebe": { - "friendly_name": "IP3 Integration 1", - "node_id": "ip3-int01.ipnt.uk", - "member_id": "markab", - "area": "IP3", - "location": { - "value": "52.045773,1.212808", - "type": "coordinate" - }, - "location_description": "Brokehall", - "hardware": "Heltec V3", - "antenna": "Generic 3dBi Whip", - "elevation": { - "value": "37", - "type": "number" - }, - "show_on_map": { - "value": "false", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "false", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "integration" - }, - "2a4f89e766dfa1758e35a69962c1f6d352b206a5e3562a589155a3ebfe7fc2bb": { - "friendly_name": "IP3 Repeater 4", - "node_id": "ip3-rep04.ipnt.uk", - "member_id": "markab", - "area": "IP3", - "location": { - "value": "52.046383,1.174542", - "type": "coordinate" - }, - "location_description": "Holywells", - "hardware": "Sensecap Solar", - "antenna": "Paradar 6.5dbi Omni", - "elevation": { - "value": "21", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "e790b73b2d6e377dd0f575c847f3ef42232f610eb9a19af57083fc4f647309ac": { - "friendly_name": "IP3 Repeater 5", - "node_id": "ip3-rep05.ipnt.uk", - "member_id": "markab", - "area": "IP3", - "location": { - "value": "52.05252,1.17034", - "type": "coordinate" - }, - "location_description": "Back Hamlet", - "hardware": "Heltec T114", - "antenna": "Paradar 6.5dBi Omni", - "elevation": { - "value": "38", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "20ed75ffc0f9777951716bb3d308d7f041fd2ad32fe2e998e600d0361e1fe2ac": { - "friendly_name": "IP3 Repeater 6", - "node_id": "ip3-rep06.ipnt.uk", - "member_id": "ccz", - "area": "IP3", - "location": { - "value": "52.04893,1.18965", - "type": "coordinate" - }, - "location_description": "Dover Road", - "hardware": "Unknown", - "antenna": "Generic 5dBi Whip", - "elevation": { - "value": "38", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "bd7b5ac75f660675b39f368e1dbb6d1dbcefd8bd7a170e21a942954f67c8bf52": { - "friendly_name": "IP8 Repeater 1", - "node_id": "rep01.ip8.ipnt.uk", - "member_id": "walshie86", - "area": "IP8", - "location": { - "value": "52.033684,1.118384", - "type": "coordinate" - }, - "location_description": "Grove Hill", - "hardware": "Heltec V3", - "antenna": "McGill 3dBi Omni", - "elevation": { - "value": "13", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "9cf300c40112ea34d0a59858270948b27ab6cd87e840de338f3ca782c17537b2": { - "friendly_name": "IP8 Repeater 2", - "node_id": "rep02.ip8.ipnt.uk", - "member_id": "walshie86", - "area": "IP8", - "location": { - "value": "52.035648,1.073271", - "type": "coordinate" - }, - "location_description": "Washbrook", - "hardware": "Sensecap Solar", - "elevation": { - "value": "13", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "d3c20d962f7384c111fbafad6fbc1c1dc0e5c3ce802fb3ee11020e8d8207ed3a": { - "friendly_name": "IP4 Repeater 1", - "node_id": "ip4-rep01.ipnt.uk", - "member_id": "markab", - "area": "IP4", - "location": { - "value": "52.052445,1.156882", - "type": "coordinate" - }, - "location_description": "Wine Rack", - "hardware": "Heltec T114", - "antenna": "Generic 5dbi Whip", - "elevation": { - "value": "50", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "b00ce9d218203e96d8557a4d59e06f5de59bbc4dcc4df9c870079d2cb8b5bd80": { - "friendly_name": "IP4 Repeater 2", - "node_id": "ip4-rep02.ipnt.uk", - "member_id": "markab", - "area": "IP4", - "location": { - "value": "52.06217,1.18332", - "type": "coordinate" - }, - "location_description": "Rushmere Road", - "hardware": "Heltec V3", - "antenna": "Paradar 5dbi Whip", - "elevation": { - "value": "35", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "false", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "8accb6d0189ccaffb745ba54793e7fe3edd515edb45554325d957e48c1b9f3b3": { - "friendly_name": "IP4 Repeater 3", - "node_id": "ip4-rep03.ipnt.uk", - "member_id": "craig", - "area": "IP4", - "location": { - "value": "52.058,1.165", - "type": "coordinate" - }, - "location_description": "IP4 Area", - "hardware": "Heltec v3", - "antenna": "Generic Whip", - "elevation": { - "value": "30", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "true", - "type": "boolean" - }, - "is_testing": { - "value": "false", - "type": "boolean" - }, - "mesh_role": "repeater" - }, - "69fb8431e7ab307513797544fab99ce53ce24c46ec2d3a11767fe70f2ca37b23": { - "friendly_name": "IP3 Test Repeater 1", - "node_id": "ip3-tst01.ipnt.uk", - "member_id": "markab", - "area": "IP3", - "location": { - "value": "52.041869,1.204789", - "type": "coordinate" - }, - "location_description": "Brokehall", - "hardware": "Station G2", - "antenna": "McGill 10dBi Panel", - "elevation": { - "value": "37", - "type": "number" - }, - "show_on_map": { - "value": "true", - "type": "boolean" - }, - "is_public": { - "value": "true", - "type": "boolean" - }, - "is_online": { - "value": "false", - "type": "boolean" - }, - "is_testing": { - "value": "true", - "type": "boolean" - }, - "mesh_role": "repeater" - } -} diff --git a/contrib/seed/ipnet/node_tags.yaml b/contrib/seed/ipnet/node_tags.yaml new file mode 100644 index 0000000..03ef582 --- /dev/null +++ b/contrib/seed/ipnet/node_tags.yaml @@ -0,0 +1,311 @@ +# IPNet Network Node Tags +# Uses YAML primitives: numbers, booleans, and strings are auto-detected + +# IP2 Area Nodes +2337484665ced7e210007e9fd9db98ced0a24a6eab8b4cbe3a06b3a1cea33ca1: + friendly_name: IP2 Repeater 1 + node_id: ip2-rep01.ipnt.uk + member_id: louis + area: IP2 + lat: 52.0357627 + lon: 1.132079 + location_description: Fountains Road + hardware: Heltec V3 + antenna: Paradar 8.5dBi Omni + elevation: 31 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +8cb01fff1afc099055af418ce5fc5e60384df9ff763c25dd7e6a5e0922e8df90: + friendly_name: IP2 Repeater 2 + node_id: ip2-rep02.ipnt.uk + member_id: louis + area: IP2 + lat: 52.0390682 + lon: 1.1304141 + location_description: Belstead Road + hardware: Heltec V3 + antenna: McGill 6dBi Omni + elevation: 44 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +5b565df747913358e24d890b2227de9c35d09763746b6ec326c15ebbf9b8be3b: + friendly_name: IP2 Repeater 3 + node_id: ip2-rep03.ipnt.uk + member_id: louis + area: IP2 + lat: 52.046356 + lon: 1.134661 + location_description: Birkfield Drive + hardware: Heltec V3 + antenna: Paradar 8.5dBi Omni + elevation: 52 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +780d0939f90b22d3bd7cbedcaf4e8d468a12c01886ab24b8cfa11eab2f5516c5: + friendly_name: IP2 Integration 1 + node_id: ip2-int01.ipnt.uk + member_id: louis + area: IP2 + lat: 52.0354539 + lon: 1.1295338 + location_description: Fountains Road + hardware: Heltec V3 + antenna: Generic 5dBi Whip + elevation: 25 + show_on_map: false + is_public: true + is_online: true + is_testing: false + mesh_role: integration + +30121dc60362c633c457ffa18f49b3e1d6823402c33709f32d7df70612250b96: + friendly_name: MeshBot + node_id: bot.ipnt.uk + member_id: louis + area: IP2 + lat: 52.0354539 + lon: 1.1295338 + location_description: Fountains Road + hardware: Heltec V3 + antenna: Generic 5dBi Whip + elevation: 25 + show_on_map: false + is_public: true + is_online: true + is_testing: false + mesh_role: integration + +# IP3 Area Nodes +9135986b83815ada92883358435cc6528c7db60cb647f9b6547739a1ce5eb1c8: + friendly_name: IP3 Repeater 1 + node_id: ip3-rep01.ipnt.uk + member_id: markab + area: IP3 + lat: 52.045803 + lon: 1.204416 + location_description: Brokehall + hardware: Heltec V3 + antenna: Paradar 8.5dBi Omni + elevation: 42 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +e334ec5475789d542ed9e692fbeef7444a371fcc05adcbda1f47ba6a3191b459: + friendly_name: IP3 Repeater 2 + node_id: ip3-rep02.ipnt.uk + member_id: ccz + area: IP3 + lat: 52.03297 + lon: 1.17543 + location_description: Morland Road Allotments + hardware: Heltec T114 + antenna: Unknown + elevation: 39 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +cc15fb33e98f2e098a543f516f770dc3061a1a6b30f79b84780663bf68ae6b53: + friendly_name: IP3 Repeater 3 + node_id: ip3-rep03.ipnt.uk + member_id: ccz + area: IP3 + lat: 52.04499 + lon: 1.18149 + location_description: Hatfield Road + hardware: Heltec V3 + antenna: Unknown + elevation: 39 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +22309435fbd9dd1f14870a1895dc854779f6b2af72b08542f6105d264a493ebe: + friendly_name: IP3 Integration 1 + node_id: ip3-int01.ipnt.uk + member_id: markab + area: IP3 + lat: 52.045773 + lon: 1.212808 + location_description: Brokehall + hardware: Heltec V3 + antenna: Generic 3dBi Whip + elevation: 37 + show_on_map: false + is_public: true + is_online: false + is_testing: false + mesh_role: integration + +2a4f89e766dfa1758e35a69962c1f6d352b206a5e3562a589155a3ebfe7fc2bb: + friendly_name: IP3 Repeater 4 + node_id: ip3-rep04.ipnt.uk + member_id: markab + area: IP3 + lat: 52.046383 + lon: 1.174542 + location_description: Holywells + hardware: Sensecap Solar + antenna: Paradar 6.5dbi Omni + elevation: 21 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +e790b73b2d6e377dd0f575c847f3ef42232f610eb9a19af57083fc4f647309ac: + friendly_name: IP3 Repeater 5 + node_id: ip3-rep05.ipnt.uk + member_id: markab + area: IP3 + lat: 52.05252 + lon: 1.17034 + location_description: Back Hamlet + hardware: Heltec T114 + antenna: Paradar 6.5dBi Omni + elevation: 38 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +20ed75ffc0f9777951716bb3d308d7f041fd2ad32fe2e998e600d0361e1fe2ac: + friendly_name: IP3 Repeater 6 + node_id: ip3-rep06.ipnt.uk + member_id: ccz + area: IP3 + lat: 52.04893 + lon: 1.18965 + location_description: Dover Road + hardware: Unknown + antenna: Generic 5dBi Whip + elevation: 38 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +69fb8431e7ab307513797544fab99ce53ce24c46ec2d3a11767fe70f2ca37b23: + friendly_name: IP3 Test Repeater 1 + node_id: ip3-tst01.ipnt.uk + member_id: markab + area: IP3 + lat: 52.041869 + lon: 1.204789 + location_description: Brokehall + hardware: Station G2 + antenna: McGill 10dBi Panel + elevation: 37 + show_on_map: true + is_public: true + is_online: false + is_testing: true + mesh_role: repeater + +# IP4 Area Nodes +d3c20d962f7384c111fbafad6fbc1c1dc0e5c3ce802fb3ee11020e8d8207ed3a: + friendly_name: IP4 Repeater 1 + node_id: ip4-rep01.ipnt.uk + member_id: markab + area: IP4 + lat: 52.052445 + lon: 1.156882 + location_description: Wine Rack + hardware: Heltec T114 + antenna: Generic 5dbi Whip + elevation: 50 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +b00ce9d218203e96d8557a4d59e06f5de59bbc4dcc4df9c870079d2cb8b5bd80: + friendly_name: IP4 Repeater 2 + node_id: ip4-rep02.ipnt.uk + member_id: markab + area: IP4 + lat: 52.06217 + lon: 1.18332 + location_description: Rushmere Road + hardware: Heltec V3 + antenna: Paradar 5dbi Whip + elevation: 35 + show_on_map: true + is_public: true + is_online: false + is_testing: false + mesh_role: repeater + +8accb6d0189ccaffb745ba54793e7fe3edd515edb45554325d957e48c1b9f3b3: + friendly_name: IP4 Repeater 3 + node_id: ip4-rep03.ipnt.uk + member_id: craig + area: IP4 + lat: 52.058 + lon: 1.165 + location_description: IP4 Area + hardware: Heltec v3 + antenna: Generic Whip + elevation: 30 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +# IP8 Area Nodes +bd7b5ac75f660675b39f368e1dbb6d1dbcefd8bd7a170e21a942954f67c8bf52: + friendly_name: IP8 Repeater 1 + node_id: rep01.ip8.ipnt.uk + member_id: walshie86 + area: IP8 + lat: 52.033684 + lon: 1.118384 + location_description: Grove Hill + hardware: Heltec V3 + antenna: McGill 3dBi Omni + elevation: 13 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater + +9cf300c40112ea34d0a59858270948b27ab6cd87e840de338f3ca782c17537b2: + friendly_name: IP8 Repeater 2 + node_id: rep02.ip8.ipnt.uk + member_id: walshie86 + area: IP8 + lat: 52.035648 + lon: 1.073271 + location_description: Washbrook + hardware: Sensecap Solar + elevation: 13 + show_on_map: true + is_public: true + is_online: true + is_testing: false + mesh_role: repeater diff --git a/example/seed/members.json b/example/seed/members.json deleted file mode 100644 index 89857ff..0000000 --- a/example/seed/members.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "members": [ - { - "name": "Example Member", - "callsign": "N0CALL", - "role": "Network Operator", - "description": "Example member entry" - } - ] -} diff --git a/example/seed/members.yaml b/example/seed/members.yaml new file mode 100644 index 0000000..c8670fe --- /dev/null +++ b/example/seed/members.yaml @@ -0,0 +1,6 @@ +# Example members seed file +members: + - name: Example Member + callsign: N0CALL + role: Network Operator + description: Example member entry diff --git a/example/seed/node_tags.json b/example/seed/node_tags.json deleted file mode 100644 index d4b753c..0000000 --- a/example/seed/node_tags.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef": { - "friendly_name": "Gateway Node", - "location": {"value": "37.7749,-122.4194", "type": "coordinate"}, - "lat": {"value": "37.7749", "type": "number"}, - "lon": {"value": "-122.4194", "type": "number"}, - "role": "gateway" - }, - "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210": { - "friendly_name": "Oakland Repeater", - "location": {"value": "37.8044,-122.2712", "type": "coordinate"}, - "lat": {"value": "37.8044", "type": "number"}, - "lon": {"value": "-122.2712", "type": "number"}, - "altitude": {"value": "150", "type": "number"} - } -} diff --git a/example/seed/node_tags.yaml b/example/seed/node_tags.yaml new file mode 100644 index 0000000..7fb256a --- /dev/null +++ b/example/seed/node_tags.yaml @@ -0,0 +1,29 @@ +# Example node tags seed file +# Each key is a 64-character hex public key +# +# Tag values can be: +# - YAML primitives (auto-detected type): +# friendly_name: Gateway Node # string +# elevation: 150 # number +# is_online: true # boolean +# +# - Explicit type (for special types like coordinate): +# location: +# value: "37.7749,-122.4194" +# type: coordinate +# +# Supported types: string, number, boolean, coordinate + +0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef: + friendly_name: Gateway Node + role: gateway + lat: 37.7749 + lon: -122.4194 + is_online: true + +fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210: + friendly_name: Oakland Repeater + lat: 37.8044 + lon: -122.2712 + altitude: 150 + is_online: false diff --git a/pyproject.toml b/pyproject.toml index 3f10fc7..871fc6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "httpx>=0.25.0", "aiosqlite>=0.19.0", "meshcore>=2.2.0", + "pyyaml>=6.0.0", ] [project.optional-dependencies] @@ -51,6 +52,7 @@ dev = [ "mypy>=1.5.0", "pre-commit>=3.4.0", "types-paho-mqtt>=1.6.0", + "types-PyYAML>=6.0.0", ] postgres = [ "asyncpg>=0.28.0", diff --git a/src/meshcore_hub/collector/cli.py b/src/meshcore_hub/collector/cli.py index 789e40c..c501de9 100644 --- a/src/meshcore_hub/collector/cli.py +++ b/src/meshcore_hub/collector/cli.py @@ -1,9 +1,14 @@ """CLI for the Collector component.""" +from typing import TYPE_CHECKING + import click from meshcore_hub.common.logging import configure_logging +if TYPE_CHECKING: + from meshcore_hub.common.database import DatabaseManager + @click.group(invoke_without_command=True) @click.pass_context @@ -137,6 +142,7 @@ def collector( database_url=effective_db_url, log_level=log_level, data_home=data_home or settings.data_home, + seed_home=settings.effective_seed_home, ) @@ -149,9 +155,13 @@ def _run_collector_service( database_url: str, log_level: str, data_home: str, + seed_home: str, ) -> None: """Run the collector service. + On startup, automatically seeds the database from YAML files in seed_home + if they exist. + Webhooks can be configured via environment variables: - WEBHOOK_ADVERTISEMENT_URL: Webhook for advertisement events - WEBHOOK_MESSAGE_URL: Webhook for all message events @@ -168,20 +178,48 @@ def _run_collector_service( click.echo("Starting MeshCore Collector") click.echo(f"Data home: {data_home}") + click.echo(f"Seed home: {seed_home}") click.echo(f"MQTT: {mqtt_host}:{mqtt_port} (prefix: {prefix})") click.echo(f"Database: {database_url}") + # Initialize database and run seed import on startup + from meshcore_hub.common.database import DatabaseManager + + db = DatabaseManager(database_url) + db.create_tables() + + # Auto-seed from seed files on startup + click.echo("") + click.echo("Checking for seed files...") + seed_home_path = Path(seed_home) + node_tags_exists = (seed_home_path / "node_tags.yaml").exists() + members_exists = (seed_home_path / "members.yaml").exists() + + if node_tags_exists or members_exists: + click.echo("Running seed import...") + _run_seed_import( + seed_home=seed_home, + db=db, + create_nodes=True, + verbose=True, + ) + else: + click.echo(f"No seed files found in {seed_home}") + + db.dispose() + # Load webhook configuration from settings - from meshcore_hub.common.config import get_collector_settings from meshcore_hub.collector.webhook import ( WebhookDispatcher, create_webhooks_from_settings, ) + from meshcore_hub.common.config import get_collector_settings settings = get_collector_settings() webhooks = create_webhooks_from_settings(settings) webhook_dispatcher = WebhookDispatcher(webhooks) if webhooks else None + click.echo("") if webhook_dispatcher and webhook_dispatcher.webhooks: click.echo(f"Webhooks configured: {len(webhooks)}") for wh in webhooks: @@ -191,6 +229,8 @@ def _run_collector_service( from meshcore_hub.collector.subscriber import run_collector + click.echo("") + click.echo("Starting MQTT subscriber...") run_collector( mqtt_host=mqtt_host, mqtt_port=mqtt_port, @@ -218,6 +258,7 @@ def run_cmd(ctx: click.Context) -> None: database_url=ctx.obj["database_url"], log_level=ctx.obj["log_level"], data_home=ctx.obj["data_home"], + seed_home=ctx.obj["seed_home"], ) @@ -236,17 +277,15 @@ def seed_cmd( """Import seed data from SEED_HOME directory. Looks for the following files in SEED_HOME: - - node_tags.json: Node tag definitions (keyed by public_key) - - members.json: Network member definitions + - node_tags.yaml: Node tag definitions (keyed by public_key) + - members.yaml: Network member definitions Files that don't exist are skipped. This command is idempotent - existing records are updated, new records are created. - SEED_HOME defaults to {DATA_HOME}/collector but can be overridden + SEED_HOME defaults to ./seed but can be overridden with the --seed-home option or SEED_HOME environment variable. """ - from pathlib import Path - configure_logging(level=ctx.obj["log_level"]) seed_home = ctx.obj["seed_home"] @@ -254,50 +293,18 @@ def seed_cmd( click.echo(f"Database: {ctx.obj['database_url']}") from meshcore_hub.common.database import DatabaseManager - from meshcore_hub.collector.tag_import import import_tags - from meshcore_hub.collector.member_import import import_members # Initialize database db = DatabaseManager(ctx.obj["database_url"]) db.create_tables() - # Track what was imported - imported_any = False - - # Import node tags if file exists - node_tags_file = Path(seed_home) / "node_tags.json" - if node_tags_file.exists(): - click.echo(f"\nImporting node tags from: {node_tags_file}") - stats = import_tags( - file_path=str(node_tags_file), - db=db, - create_nodes=not no_create_nodes, - ) - click.echo(f" Tags: {stats['created']} created, {stats['updated']} updated") - if stats["nodes_created"]: - click.echo(f" Nodes created: {stats['nodes_created']}") - if stats["errors"]: - for error in stats["errors"]: - click.echo(f" Error: {error}", err=True) - imported_any = True - else: - click.echo(f"\nNo node_tags.json found in {seed_home}") - - # Import members if file exists - members_file = Path(seed_home) / "members.json" - if members_file.exists(): - click.echo(f"\nImporting members from: {members_file}") - stats = import_members( - file_path=str(members_file), - db=db, - ) - click.echo(f" Members: {stats['created']} created, {stats['updated']} updated") - if stats["errors"]: - for error in stats["errors"]: - click.echo(f" Error: {error}", err=True) - imported_any = True - else: - click.echo(f"\nNo members.json found in {seed_home}") + # Run seed import + imported_any = _run_seed_import( + seed_home=seed_home, + db=db, + create_nodes=not no_create_nodes, + verbose=True, + ) if not imported_any: click.echo("\nNo seed files found. Nothing to import.") @@ -307,6 +314,76 @@ def seed_cmd( db.dispose() +def _run_seed_import( + seed_home: str, + db: "DatabaseManager", + create_nodes: bool = True, + verbose: bool = False, +) -> bool: + """Run seed import from SEED_HOME directory. + + Args: + seed_home: Path to seed home directory + db: Database manager instance + create_nodes: If True, create nodes that don't exist + verbose: If True, output progress messages + + Returns: + True if any files were imported, False otherwise + """ + from pathlib import Path + + from meshcore_hub.collector.member_import import import_members + from meshcore_hub.collector.tag_import import import_tags + + imported_any = False + + # Import node tags if file exists + node_tags_file = Path(seed_home) / "node_tags.yaml" + if node_tags_file.exists(): + if verbose: + click.echo(f"\nImporting node tags from: {node_tags_file}") + stats = import_tags( + file_path=str(node_tags_file), + db=db, + create_nodes=create_nodes, + ) + if verbose: + click.echo( + f" Tags: {stats['created']} created, {stats['updated']} updated" + ) + if stats["nodes_created"]: + click.echo(f" Nodes created: {stats['nodes_created']}") + if stats["errors"]: + for error in stats["errors"]: + click.echo(f" Error: {error}", err=True) + imported_any = True + elif verbose: + click.echo(f"\nNo node_tags.yaml found in {seed_home}") + + # Import members if file exists + members_file = Path(seed_home) / "members.yaml" + if members_file.exists(): + if verbose: + click.echo(f"\nImporting members from: {members_file}") + stats = import_members( + file_path=str(members_file), + db=db, + ) + if verbose: + click.echo( + f" Members: {stats['created']} created, {stats['updated']} updated" + ) + if stats["errors"]: + for error in stats["errors"]: + click.echo(f" Error: {error}", err=True) + imported_any = True + elif verbose: + click.echo(f"\nNo members.yaml found in {seed_home}") + + return imported_any + + @collector.command("import-tags") @click.argument("file", type=click.Path(), required=False, default=None) @click.option( @@ -321,32 +398,32 @@ def import_tags_cmd( file: str | None, no_create_nodes: bool, ) -> None: - """Import node tags from a JSON file. + """Import node tags from a YAML file. - Reads a JSON file containing tag definitions and upserts them + Reads a YAML file containing tag definitions and upserts them into the database. Existing tags are updated, new tags are created. - FILE is the path to the JSON file containing tags. - If not provided, defaults to {SEED_HOME}/node_tags.json. + FILE is the path to the YAML file containing tags. + If not provided, defaults to {SEED_HOME}/node_tags.yaml. + + Expected YAML format (keyed by public_key): - Expected JSON format (keyed by public_key): \b - { - "0123456789abcdef...": { - "friendly_name": "My Node", - "location": {"value": "52.0,1.0", "type": "coordinate"}, - "altitude": {"value": "150", "type": "number"} - } - } + 0123456789abcdef...: + friendly_name: My Node + location: + value: "52.0,1.0" + type: coordinate + altitude: + value: "150" + type: number Shorthand is also supported (string values with default type): + \b - { - "0123456789abcdef...": { - "friendly_name": "My Node", - "role": "gateway" - } - } + 0123456789abcdef...: + friendly_name: My Node + role: gateway Supported types: string, number, boolean, coordinate """ @@ -362,7 +439,7 @@ def import_tags_cmd( if not Path(tags_file).exists(): click.echo(f"Tags file not found: {tags_file}") if not file: - click.echo("Specify a file path or create the default node_tags.json.") + click.echo("Specify a file path or create the default node_tags.yaml.") return click.echo(f"Importing tags from: {tags_file}") @@ -407,33 +484,29 @@ def import_members_cmd( ctx: click.Context, file: str | None, ) -> None: - """Import network members from a JSON file. + """Import network members from a YAML file. - Reads a JSON file containing member definitions and upserts them + Reads a YAML file containing member definitions and upserts them into the database. Existing members (matched by name) are updated, new members are created. - FILE is the path to the JSON file containing members. - If not provided, defaults to {SEED_HOME}/members.json. + FILE is the path to the YAML file containing members. + If not provided, defaults to {SEED_HOME}/members.yaml. + + Expected YAML format (list): - Expected JSON format (list): \b - [ - { - "name": "John Doe", - "callsign": "N0CALL", - "role": "Network Operator", - "description": "Example member" - } - ] + - name: John Doe + callsign: N0CALL + role: Network Operator + description: Example member Or with "members" key: + \b - { - "members": [ - {"name": "John Doe", "callsign": "N0CALL", ...} - ] - } + members: + - name: John Doe + callsign: N0CALL """ from pathlib import Path @@ -447,7 +520,7 @@ def import_members_cmd( if not Path(members_file).exists(): click.echo(f"Members file not found: {members_file}") if not file: - click.echo("Specify a file path or create the default members.json.") + click.echo("Specify a file path or create the default members.yaml.") return click.echo(f"Importing members from: {members_file}") diff --git a/src/meshcore_hub/collector/member_import.py b/src/meshcore_hub/collector/member_import.py index fdbfaee..4ecaf2f 100644 --- a/src/meshcore_hub/collector/member_import.py +++ b/src/meshcore_hub/collector/member_import.py @@ -1,10 +1,10 @@ -"""Import members from JSON file.""" +"""Import members from YAML file.""" -import json import logging from pathlib import Path from typing import Any, Optional +import yaml from pydantic import BaseModel, Field, field_validator from sqlalchemy import select @@ -38,24 +38,31 @@ class MemberData(BaseModel): def load_members_file(file_path: str | Path) -> list[dict[str, Any]]: - """Load and validate members from a JSON file. + """Load and validate members from a YAML file. Supports two formats: 1. List of member objects: - [{"name": "Member 1", ...}, {"name": "Member 2", ...}] + + - name: Member 1 + callsign: M1 + - name: Member 2 + callsign: M2 2. Object with "members" key: - {"members": [{"name": "Member 1", ...}, ...]} + + members: + - name: Member 1 + callsign: M1 Args: - file_path: Path to the members JSON file + file_path: Path to the members YAML file Returns: List of validated member dictionaries Raises: FileNotFoundError: If file does not exist - json.JSONDecodeError: If file is not valid JSON + yaml.YAMLError: If file is not valid YAML ValueError: If file content is invalid """ path = Path(file_path) @@ -63,7 +70,7 @@ def load_members_file(file_path: str | Path) -> list[dict[str, Any]]: raise FileNotFoundError(f"Members file not found: {file_path}") with open(path, "r") as f: - data = json.load(f) + data = yaml.safe_load(f) # Handle both formats if isinstance(data, list): @@ -73,7 +80,7 @@ def load_members_file(file_path: str | Path) -> list[dict[str, Any]]: if not isinstance(members_list, list): raise ValueError("'members' key must contain a list") else: - raise ValueError("Members file must be a list or an object with 'members' key") + raise ValueError("Members file must be a list or a mapping with 'members' key") # Validate each member validated: list[dict[str, Any]] = [] @@ -97,13 +104,13 @@ def import_members( file_path: str | Path, db: DatabaseManager, ) -> dict[str, Any]: - """Import members from a JSON file into the database. + """Import members from a YAML file into the database. Performs upsert operations based on name - existing members are updated, new members are created. Args: - file_path: Path to the members JSON file + file_path: Path to the members YAML file db: Database manager instance Returns: diff --git a/src/meshcore_hub/collector/tag_import.py b/src/meshcore_hub/collector/tag_import.py index ac4ff8d..791c288 100644 --- a/src/meshcore_hub/collector/tag_import.py +++ b/src/meshcore_hub/collector/tag_import.py @@ -1,11 +1,11 @@ -"""Import node tags from JSON file.""" +"""Import node tags from YAML file.""" -import json import logging from datetime import datetime, timezone from pathlib import Path from typing import Any +import yaml from pydantic import BaseModel, Field, model_validator from sqlalchemy import select @@ -64,33 +64,33 @@ def validate_public_key(public_key: str) -> str: def load_tags_file(file_path: str | Path) -> dict[str, dict[str, Any]]: - """Load and validate tags from a JSON file. + """Load and validate tags from a YAML file. - New format - dictionary keyed by public_key: - { - "0123456789abcdef...": { - "friendly_name": "My Node", - "location": {"value": "52.0,1.0", "type": "coordinate"}, - "altitude": {"value": "150", "type": "number"} - } - } + YAML format - dictionary keyed by public_key: + + 0123456789abcdef...: + friendly_name: My Node + location: + value: "52.0,1.0" + type: coordinate + altitude: + value: "150" + type: number Shorthand is allowed - string values are auto-converted: - { - "0123456789abcdef...": { - "friendly_name": "My Node" - } - } + + 0123456789abcdef...: + friendly_name: My Node Args: - file_path: Path to the tags JSON file + file_path: Path to the tags YAML file Returns: Dictionary mapping public_key to tag dictionary Raises: FileNotFoundError: If file does not exist - json.JSONDecodeError: If file is not valid JSON + yaml.YAMLError: If file is not valid YAML ValueError: If file content is invalid """ path = Path(file_path) @@ -98,10 +98,10 @@ def load_tags_file(file_path: str | Path) -> dict[str, dict[str, Any]]: raise FileNotFoundError(f"Tags file not found: {file_path}") with open(path, "r") as f: - data = json.load(f) + data = yaml.safe_load(f) if not isinstance(data, dict): - raise ValueError("Tags file must contain a JSON object") + raise ValueError("Tags file must contain a YAML mapping") # Validate each entry validated: dict[str, dict[str, Any]] = {} @@ -117,12 +117,24 @@ def load_tags_file(file_path: str | Path) -> dict[str, dict[str, Any]]: for tag_key, tag_value in tags.items(): if isinstance(tag_value, dict): # Full format with value and type + raw_value = tag_value.get("value") + # Convert value to string if it's not None + str_value = str(raw_value) if raw_value is not None else None validated_tags[tag_key] = { - "value": tag_value.get("value"), + "value": str_value, "type": tag_value.get("type", "string"), } + elif isinstance(tag_value, bool): + # YAML boolean - must check before int since bool is subclass of int + validated_tags[tag_key] = { + "value": str(tag_value).lower(), + "type": "boolean", + } + elif isinstance(tag_value, (int, float)): + # YAML number (int or float) + validated_tags[tag_key] = {"value": str(tag_value), "type": "number"} elif isinstance(tag_value, str): - # Shorthand: just a string value + # String value validated_tags[tag_key] = {"value": tag_value, "type": "string"} elif tag_value is None: validated_tags[tag_key] = {"value": None, "type": "string"} @@ -140,12 +152,12 @@ def import_tags( db: DatabaseManager, create_nodes: bool = True, ) -> dict[str, Any]: - """Import tags from a JSON file into the database. + """Import tags from a YAML file into the database. Performs upsert operations - existing tags are updated, new tags are created. Args: - file_path: Path to the tags JSON file + file_path: Path to the tags YAML file db: Database manager instance create_nodes: If True, create nodes that don't exist. If False, skip tags for non-existent nodes. diff --git a/src/meshcore_hub/common/config.py b/src/meshcore_hub/common/config.py index 7e4b179..6f19623 100644 --- a/src/meshcore_hub/common/config.py +++ b/src/meshcore_hub/common/config.py @@ -80,7 +80,7 @@ class CollectorSettings(CommonSettings): description="SQLAlchemy database URL (default: sqlite:///{data_home}/collector/meshcore.db)", ) - # Seed home directory - contains initial data files (node_tags.json, members.json) + # Seed home directory - contains initial data files (node_tags.yaml, members.yaml) seed_home: str = Field( default="./seed", description="Directory containing seed data files (default: ./seed)", @@ -147,17 +147,17 @@ class CollectorSettings(CommonSettings): @property def node_tags_file(self) -> str: - """Get the path to node_tags.json in seed_home.""" + """Get the path to node_tags.yaml in seed_home.""" from pathlib import Path - return str(Path(self.effective_seed_home) / "node_tags.json") + return str(Path(self.effective_seed_home) / "node_tags.yaml") @property def members_file(self) -> str: - """Get the path to members.json in seed_home.""" + """Get the path to members.yaml in seed_home.""" from pathlib import Path - return str(Path(self.effective_seed_home) / "members.json") + return str(Path(self.effective_seed_home) / "members.yaml") @field_validator("database_url") @classmethod diff --git a/tests/test_collector/test_tag_import.py b/tests/test_collector/test_tag_import.py index 412f1be..8d0c7e0 100644 --- a/tests/test_collector/test_tag_import.py +++ b/tests/test_collector/test_tag_import.py @@ -1,10 +1,10 @@ """Tests for tag import functionality.""" -import json import tempfile from pathlib import Path import pytest +import yaml from sqlalchemy import select from meshcore_hub.collector.tag_import import ( @@ -62,8 +62,8 @@ class TestLoadTagsFile: } } - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() result = load_tags_file(f.name) @@ -84,8 +84,8 @@ class TestLoadTagsFile: } } - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() result = load_tags_file(f.name) @@ -104,8 +104,8 @@ class TestLoadTagsFile: } } - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() result = load_tags_file(f.name) @@ -118,15 +118,15 @@ class TestLoadTagsFile: def test_file_not_found(self): """Test loading non-existent file.""" with pytest.raises(FileNotFoundError): - load_tags_file("/nonexistent/path/tags.json") + load_tags_file("/nonexistent/path/tags.yaml") - def test_invalid_json(self): - """Test loading invalid JSON file.""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - f.write("invalid json {{{") + def test_invalid_yaml(self): + """Test loading invalid YAML file.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write("invalid: yaml: content: [unclosed") f.flush() - with pytest.raises(json.JSONDecodeError): + with pytest.raises(yaml.YAMLError): load_tags_file(f.name) Path(f.name).unlink() @@ -135,11 +135,11 @@ class TestLoadTagsFile: """Test loading file with invalid schema (not a dict).""" data = [{"public_key": "abc"}] - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() - with pytest.raises(ValueError, match="must contain a JSON object"): + with pytest.raises(ValueError, match="must contain a YAML mapping"): load_tags_file(f.name) Path(f.name).unlink() @@ -148,8 +148,8 @@ class TestLoadTagsFile: """Test loading file with invalid public key.""" data = {"invalid_key": {"tag": "value"}} - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() with pytest.raises(ValueError, match="must be 64 characters"): @@ -161,8 +161,8 @@ class TestLoadTagsFile: """Test loading empty tags file.""" data: dict[str, dict[str, str]] = {} - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() result = load_tags_file(f.name) @@ -195,8 +195,8 @@ class TestImportTags: }, } - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() yield f.name @@ -270,7 +270,7 @@ class TestImportTags: def test_import_nonexistent_file(self, db_manager): """Test import with non-existent file.""" - stats = import_tags("/nonexistent/tags.json", db_manager) + stats = import_tags("/nonexistent/tags.yaml", db_manager) assert stats["total"] == 0 assert len(stats["errors"]) == 1 @@ -280,8 +280,8 @@ class TestImportTags: """Test import with empty tags object.""" data: dict[str, dict[str, str]] = {} - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() stats = import_tags(f.name, db_manager) @@ -300,8 +300,8 @@ class TestImportTags: } } - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() import_tags(f.name, db_manager, create_nodes=True) @@ -323,8 +323,8 @@ class TestImportTags: } } - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() stats = import_tags(f.name, db_manager, create_nodes=True) @@ -339,16 +339,16 @@ class TestImportTags: Path(f.name).unlink() - def test_import_numeric_value_converted(self, db_manager): - """Test that numeric values are converted to strings.""" + def test_import_numeric_value_detected(self, db_manager): + """Test that YAML numeric values are detected and stored with number type.""" data = { "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef": { "num_val": 42, } } - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(data, f) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) f.flush() stats = import_tags(f.name, db_manager, create_nodes=True) @@ -359,6 +359,34 @@ class TestImportTags: tag = session.execute(select(NodeTag)).scalar_one() assert tag.key == "num_val" assert tag.value == "42" - assert tag.value_type == "string" + assert tag.value_type == "number" + + Path(f.name).unlink() + + def test_import_boolean_value_detected(self, db_manager): + """Test that YAML boolean values are detected and stored with boolean type.""" + data = { + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef": { + "is_active": True, + "is_disabled": False, + } + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) + f.flush() + + stats = import_tags(f.name, db_manager, create_nodes=True) + + assert stats["created"] == 2 + + with db_manager.session_scope() as session: + tags = session.execute(select(NodeTag)).scalars().all() + tag_dict = {t.key: t for t in tags} + + assert tag_dict["is_active"].value == "true" + assert tag_dict["is_active"].value_type == "boolean" + assert tag_dict["is_disabled"].value == "false" + assert tag_dict["is_disabled"].value_type == "boolean" Path(f.name).unlink() diff --git a/tests/test_common/test_config.py b/tests/test_common/test_config.py index ad3c50d..ea6c1b3 100644 --- a/tests/test_common/test_config.py +++ b/tests/test_common/test_config.py @@ -64,8 +64,8 @@ class TestCollectorSettings: assert settings.seed_home == "./seed" assert settings.effective_seed_home == "seed" # node_tags_file and members_file are derived from effective_seed_home - assert settings.node_tags_file == "seed/node_tags.json" - assert settings.members_file == "seed/members.json" + assert settings.node_tags_file == "seed/node_tags.yaml" + assert settings.members_file == "seed/members.yaml" def test_custom_data_home(self) -> None: """Test that custom data_home affects effective paths.""" @@ -78,8 +78,8 @@ class TestCollectorSettings: assert settings.collector_data_dir == "/custom/data/collector" # seed_home is independent of data_home assert settings.effective_seed_home == "seed" - assert settings.node_tags_file == "seed/node_tags.json" - assert settings.members_file == "seed/members.json" + assert settings.node_tags_file == "seed/node_tags.yaml" + assert settings.members_file == "seed/members.yaml" def test_explicit_database_url_overrides(self) -> None: """Test that explicit database_url overrides the default.""" @@ -96,8 +96,8 @@ class TestCollectorSettings: assert settings.seed_home == "/seed/data" assert settings.effective_seed_home == "/seed/data" - assert settings.node_tags_file == "/seed/data/node_tags.json" - assert settings.members_file == "/seed/data/members.json" + assert settings.node_tags_file == "/seed/data/node_tags.yaml" + assert settings.members_file == "/seed/data/members.yaml" class TestAPISettings: