diff --git a/data/mesh_ingestor/handlers.py b/data/mesh_ingestor/handlers.py index efba9f0..d229935 100644 --- a/data/mesh_ingestor/handlers.py +++ b/data/mesh_ingestor/handlers.py @@ -460,6 +460,189 @@ def store_telemetry_packet(packet: Mapping, decoded: Mapping) -> None: ) ) + current = _coerce_float( + _first( + telemetry_section, + "current", + "deviceMetrics.current", + "deviceMetrics.current_ma", + "deviceMetrics.currentMa", + "environmentMetrics.current", + default=None, + ) + ) + gas_resistance = _coerce_float( + _first( + telemetry_section, + "gasResistance", + "gas_resistance", + "environmentMetrics.gasResistance", + "environmentMetrics.gas_resistance", + default=None, + ) + ) + iaq = _coerce_int( + _first( + telemetry_section, + "iaq", + "environmentMetrics.iaq", + "environmentMetrics.iaqIndex", + "environmentMetrics.iaq_index", + default=None, + ) + ) + distance = _coerce_float( + _first( + telemetry_section, + "distance", + "environmentMetrics.distance", + "environmentMetrics.range", + "environmentMetrics.rangeMeters", + default=None, + ) + ) + lux = _coerce_float( + _first( + telemetry_section, + "lux", + "environmentMetrics.lux", + "environmentMetrics.illuminance", + default=None, + ) + ) + white_lux = _coerce_float( + _first( + telemetry_section, + "whiteLux", + "white_lux", + "environmentMetrics.whiteLux", + "environmentMetrics.white_lux", + default=None, + ) + ) + ir_lux = _coerce_float( + _first( + telemetry_section, + "irLux", + "ir_lux", + "environmentMetrics.irLux", + "environmentMetrics.ir_lux", + default=None, + ) + ) + uv_lux = _coerce_float( + _first( + telemetry_section, + "uvLux", + "uv_lux", + "environmentMetrics.uvLux", + "environmentMetrics.uv_lux", + "environmentMetrics.uvIndex", + default=None, + ) + ) + wind_direction = _coerce_int( + _first( + telemetry_section, + "windDirection", + "wind_direction", + "environmentMetrics.windDirection", + "environmentMetrics.wind_direction", + default=None, + ) + ) + wind_speed = _coerce_float( + _first( + telemetry_section, + "windSpeed", + "wind_speed", + "environmentMetrics.windSpeed", + "environmentMetrics.wind_speed", + "environmentMetrics.windSpeedMps", + default=None, + ) + ) + wind_gust = _coerce_float( + _first( + telemetry_section, + "windGust", + "wind_gust", + "environmentMetrics.windGust", + "environmentMetrics.wind_gust", + default=None, + ) + ) + wind_lull = _coerce_float( + _first( + telemetry_section, + "windLull", + "wind_lull", + "environmentMetrics.windLull", + "environmentMetrics.wind_lull", + default=None, + ) + ) + weight = _coerce_float( + _first( + telemetry_section, + "weight", + "environmentMetrics.weight", + "environmentMetrics.mass", + default=None, + ) + ) + radiation = _coerce_float( + _first( + telemetry_section, + "radiation", + "environmentMetrics.radiation", + "environmentMetrics.radiationLevel", + default=None, + ) + ) + rainfall_1h = _coerce_float( + _first( + telemetry_section, + "rainfall1h", + "rainfall_1h", + "environmentMetrics.rainfall1h", + "environmentMetrics.rainfall_1h", + "environmentMetrics.rainfallOneHour", + default=None, + ) + ) + rainfall_24h = _coerce_float( + _first( + telemetry_section, + "rainfall24h", + "rainfall_24h", + "environmentMetrics.rainfall24h", + "environmentMetrics.rainfall_24h", + "environmentMetrics.rainfallTwentyFourHour", + default=None, + ) + ) + soil_moisture = _coerce_int( + _first( + telemetry_section, + "soilMoisture", + "soil_moisture", + "environmentMetrics.soilMoisture", + "environmentMetrics.soil_moisture", + default=None, + ) + ) + soil_temperature = _coerce_float( + _first( + telemetry_section, + "soilTemperature", + "soil_temperature", + "environmentMetrics.soilTemperature", + "environmentMetrics.soil_temperature", + default=None, + ) + ) + telemetry_payload = { "id": pkt_id, "node_id": node_id, @@ -494,6 +677,42 @@ def store_telemetry_packet(packet: Mapping, decoded: Mapping) -> None: telemetry_payload["relative_humidity"] = relative_humidity if barometric_pressure is not None: telemetry_payload["barometric_pressure"] = barometric_pressure + if current is not None: + telemetry_payload["current"] = current + if gas_resistance is not None: + telemetry_payload["gas_resistance"] = gas_resistance + if iaq is not None: + telemetry_payload["iaq"] = iaq + if distance is not None: + telemetry_payload["distance"] = distance + if lux is not None: + telemetry_payload["lux"] = lux + if white_lux is not None: + telemetry_payload["white_lux"] = white_lux + if ir_lux is not None: + telemetry_payload["ir_lux"] = ir_lux + if uv_lux is not None: + telemetry_payload["uv_lux"] = uv_lux + if wind_direction is not None: + telemetry_payload["wind_direction"] = wind_direction + if wind_speed is not None: + telemetry_payload["wind_speed"] = wind_speed + if wind_gust is not None: + telemetry_payload["wind_gust"] = wind_gust + if wind_lull is not None: + telemetry_payload["wind_lull"] = wind_lull + if weight is not None: + telemetry_payload["weight"] = weight + if radiation is not None: + telemetry_payload["radiation"] = radiation + if rainfall_1h is not None: + telemetry_payload["rainfall_1h"] = rainfall_1h + if rainfall_24h is not None: + telemetry_payload["rainfall_24h"] = rainfall_24h + if soil_moisture is not None: + telemetry_payload["soil_moisture"] = soil_moisture + if soil_temperature is not None: + telemetry_payload["soil_temperature"] = soil_temperature _queue_post_json( "/api/telemetry", diff --git a/data/migrations/20250305_extend_environment_telemetry.sql b/data/migrations/20250305_extend_environment_telemetry.sql new file mode 100644 index 0000000..dd2d4ac --- /dev/null +++ b/data/migrations/20250305_extend_environment_telemetry.sql @@ -0,0 +1,35 @@ +-- Copyright (C) 2025 l5yth +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Extend the telemetry table with additional environment metrics. +BEGIN; +ALTER TABLE telemetry ADD COLUMN gas_resistance REAL; +ALTER TABLE telemetry ADD COLUMN current REAL; +ALTER TABLE telemetry ADD COLUMN iaq INTEGER; +ALTER TABLE telemetry ADD COLUMN distance REAL; +ALTER TABLE telemetry ADD COLUMN lux REAL; +ALTER TABLE telemetry ADD COLUMN white_lux REAL; +ALTER TABLE telemetry ADD COLUMN ir_lux REAL; +ALTER TABLE telemetry ADD COLUMN uv_lux REAL; +ALTER TABLE telemetry ADD COLUMN wind_direction INTEGER; +ALTER TABLE telemetry ADD COLUMN wind_speed REAL; +ALTER TABLE telemetry ADD COLUMN weight REAL; +ALTER TABLE telemetry ADD COLUMN wind_gust REAL; +ALTER TABLE telemetry ADD COLUMN wind_lull REAL; +ALTER TABLE telemetry ADD COLUMN radiation REAL; +ALTER TABLE telemetry ADD COLUMN rainfall_1h REAL; +ALTER TABLE telemetry ADD COLUMN rainfall_24h REAL; +ALTER TABLE telemetry ADD COLUMN soil_moisture INTEGER; +ALTER TABLE telemetry ADD COLUMN soil_temperature REAL; +COMMIT; diff --git a/data/telemetry.sql b/data/telemetry.sql index 37c2ecc..93b04c9 100644 --- a/data/telemetry.sql +++ b/data/telemetry.sql @@ -35,7 +35,25 @@ CREATE TABLE IF NOT EXISTS telemetry ( uptime_seconds INTEGER, temperature REAL, relative_humidity REAL, - barometric_pressure REAL + barometric_pressure REAL, + gas_resistance REAL, + current REAL, + iaq INTEGER, + distance REAL, + lux REAL, + white_lux REAL, + ir_lux REAL, + uv_lux REAL, + wind_direction INTEGER, + wind_speed REAL, + weight REAL, + wind_gust REAL, + wind_lull REAL, + radiation REAL, + rainfall_1h REAL, + rainfall_24h REAL, + soil_moisture INTEGER, + soil_temperature REAL ); CREATE INDEX IF NOT EXISTS idx_telemetry_rx_time ON telemetry(rx_time); diff --git a/tests/telemetry.json b/tests/telemetry.json index bc84727..3ae4c4e 100644 --- a/tests/telemetry.json +++ b/tests/telemetry.json @@ -12,12 +12,31 @@ "battery_level": 101, "bitfield": 1, "payload_b64": "DTVr0mgSFQhlFQIrh0AdJb8YPyXYFSA9KJTPEg==", + "current": 0.0715, + "gas_resistance": 1456.0, + "iaq": 83, + "distance": 12.5, + "lux": 100.25, + "white_lux": 64.5, + "ir_lux": 12.75, + "uv_lux": 1.6, + "wind_direction": 270, + "wind_speed": 5.9, + "wind_gust": 7.4, + "wind_lull": 4.8, + "weight": 32.7, + "radiation": 0.45, + "rainfall_1h": 0.18, + "rainfall_24h": 1.42, + "soil_moisture": 3100, + "soil_temperature": 18.9, "device_metrics": { "batteryLevel": 101, "voltage": 4.224, "channelUtilization": 0.59666663, "airUtilTx": 0.03908333, - "uptimeSeconds": 305044 + "uptimeSeconds": 305044, + "current": 0.0715 }, "raw": { "device_metrics": { @@ -43,7 +62,24 @@ "environment_metrics": { "temperature": 21.98, "relativeHumidity": 39.475586, - "barometricPressure": 1017.8353 + "barometricPressure": 1017.8353, + "gasResistance": 1456.0, + "iaq": 83, + "distance": 12.5, + "lux": 100.25, + "whiteLux": 64.5, + "irLux": 12.75, + "uvLux": 1.6, + "windDirection": 270, + "windSpeed": 5.9, + "windGust": 7.4, + "windLull": 4.8, + "weight": 32.7, + "radiation": 0.45, + "rainfall1h": 0.18, + "rainfall24h": 1.42, + "soilMoisture": 3100, + "soilTemperature": 18.9 }, "raw": { "environment_metrics": { @@ -70,7 +106,22 @@ "voltage": 3.92, "channel_utilization": 0.284, "air_util_tx": 0.051, - "uptime_seconds": 86400 + "uptime_seconds": 86400, + "current": 0.033 + }, + "environment_metrics": { + "temperature": 19.5, + "relative_humidity": 48.2, + "barometric_pressure": 1013.1, + "distance": 7.25, + "lux": 75.5, + "whiteLux": 40.0, + "windDirection": 180, + "windSpeed": 4.3, + "weight": 28.4, + "rainfall24h": 0.75, + "soilMoisture": 2850, + "soilTemperature": 17.1 }, "local_stats": { "numPacketsTx": 1280, diff --git a/tests/test_mesh.py b/tests/test_mesh.py index 507071a..c0b8e99 100644 --- a/tests/test_mesh.py +++ b/tests/test_mesh.py @@ -1575,6 +1575,7 @@ def test_store_packet_dict_handles_telemetry_packet(mesh_module, monkeypatch): "channelUtilization": 0.59666663, "airUtilTx": 0.03908333, "uptimeSeconds": 305044, + "current": 0.0715, }, "localStats": { "numPacketsTx": 1280, @@ -1606,6 +1607,7 @@ def test_store_packet_dict_handles_telemetry_packet(mesh_module, monkeypatch): assert payload["channel_utilization"] == pytest.approx(0.59666663) assert payload["air_util_tx"] == pytest.approx(0.03908333) assert payload["uptime_seconds"] == 305044 + assert payload["current"] == pytest.approx(0.0715) assert payload["lora_freq"] == 868 assert payload["modem_preset"] == "MediumFast" @@ -1634,6 +1636,23 @@ def test_store_packet_dict_handles_environment_telemetry(mesh_module, monkeypatc "temperature": 21.98, "relativeHumidity": 39.475586, "barometricPressure": 1017.8353, + "gasResistance": 1456.0, + "iaq": 83, + "distance": 12.5, + "lux": 100.25, + "whiteLux": 64.5, + "irLux": 12.75, + "uvLux": 1.6, + "windDirection": 270, + "windSpeed": 5.9, + "windGust": 7.4, + "windLull": 4.8, + "weight": 32.7, + "radiation": 0.45, + "rainfall1h": 0.18, + "rainfall24h": 1.42, + "soilMoisture": 3100, + "soilTemperature": 18.9, }, }, }, @@ -1651,6 +1670,23 @@ def test_store_packet_dict_handles_environment_telemetry(mesh_module, monkeypatc assert payload["temperature"] == pytest.approx(21.98) assert payload["relative_humidity"] == pytest.approx(39.475586) assert payload["barometric_pressure"] == pytest.approx(1017.8353) + assert payload["gas_resistance"] == pytest.approx(1456.0) + assert payload["iaq"] == 83 + assert payload["distance"] == pytest.approx(12.5) + assert payload["lux"] == pytest.approx(100.25) + assert payload["white_lux"] == pytest.approx(64.5) + assert payload["ir_lux"] == pytest.approx(12.75) + assert payload["uv_lux"] == pytest.approx(1.6) + assert payload["wind_direction"] == 270 + assert payload["wind_speed"] == pytest.approx(5.9) + assert payload["wind_gust"] == pytest.approx(7.4) + assert payload["wind_lull"] == pytest.approx(4.8) + assert payload["weight"] == pytest.approx(32.7) + assert payload["radiation"] == pytest.approx(0.45) + assert payload["rainfall_1h"] == pytest.approx(0.18) + assert payload["rainfall_24h"] == pytest.approx(1.42) + assert payload["soil_moisture"] == 3100 + assert payload["soil_temperature"] == pytest.approx(18.9) assert payload["lora_freq"] == 868 assert payload["modem_preset"] == "MediumFast" diff --git a/web/lib/potato_mesh/application/data_processing.rb b/web/lib/potato_mesh/application/data_processing.rb index 6c6b82c..508f9b1 100644 --- a/web/lib/potato_mesh/application/data_processing.rb +++ b/web/lib/potato_mesh/application/data_processing.rb @@ -731,6 +731,50 @@ module PotatoMesh end end + # Resolve a telemetry metric from the provided data sources. + # + # @param key_map [Hash{Symbol=>Array}] ordered mapping of source names to candidate keys. + # @param sources [Hash{Symbol=>Hash}] data structures to search for metric values. + # @param type [Symbol] coercion strategy, ``:float`` or ``:integer``. + # @return [Numeric, nil] coerced metric value or nil when no candidates exist. + def resolve_numeric_metric(key_map, sources, type) + key_map.each do |source, keys| + next if keys.nil? || keys.empty? + + data = sources[source] + next unless data.is_a?(Hash) + + keys.each do |name| + next if name.nil? + + key = name.to_s + value = if data.key?(key) + data[key] + else + sym_key = key.to_sym + data.key?(sym_key) ? data[sym_key] : nil + end + + next if value.nil? + + coerced = case type + when :float + coerce_float(value) + when :integer + coerce_integer(value) + else + value + end + + return coerced unless coerced.nil? + end + end + + nil + end + + private :resolve_numeric_metric + def insert_telemetry(db, payload) return unless payload.is_a?(Hash) @@ -776,54 +820,285 @@ module PotatoMesh environment_metrics = normalize_json_object(payload["environment_metrics"] || payload["environmentMetrics"]) environment_metrics ||= normalize_json_object(telemetry_section["environmentMetrics"]) if telemetry_section&.key?("environmentMetrics") - fetch_metric = lambda do |map, *names| - next nil unless map.is_a?(Hash) - names.each do |name| - next unless name - key = name.to_s - return map[key] if map.key?(key) - end - nil + sources = { + payload: payload, + telemetry: telemetry_section, + device: device_metrics, + environment: environment_metrics, + } + + metric_definitions = [ + [ + "battery_level", + :float, + { + payload: %w[battery_level batteryLevel], + telemetry: %w[batteryLevel], + device: %w[battery_level batteryLevel], + environment: %w[battery_level batteryLevel], + }, + ], + [ + "voltage", + :float, + { + payload: %w[voltage], + telemetry: %w[voltage], + device: %w[voltage], + environment: %w[voltage], + }, + ], + [ + "channel_utilization", + :float, + { + payload: %w[channel_utilization channelUtilization], + telemetry: %w[channelUtilization], + device: %w[channel_utilization channelUtilization], + }, + ], + [ + "air_util_tx", + :float, + { + payload: %w[air_util_tx airUtilTx], + telemetry: %w[airUtilTx], + device: %w[air_util_tx airUtilTx], + }, + ], + [ + "uptime_seconds", + :integer, + { + payload: %w[uptime_seconds uptimeSeconds], + telemetry: %w[uptimeSeconds], + device: %w[uptime_seconds uptimeSeconds], + }, + ], + [ + "temperature", + :float, + { + payload: %w[temperature temperatureC tempC], + telemetry: %w[temperature temperatureC tempC], + environment: %w[temperature temperatureC temperature_c tempC], + }, + ], + [ + "relative_humidity", + :float, + { + payload: %w[relative_humidity relativeHumidity humidity], + telemetry: %w[relative_humidity relativeHumidity humidity], + environment: %w[relative_humidity relativeHumidity humidity], + }, + ], + [ + "barometric_pressure", + :float, + { + payload: %w[barometric_pressure barometricPressure pressure], + telemetry: %w[barometric_pressure barometricPressure pressure], + environment: %w[barometric_pressure barometricPressure pressure], + }, + ], + [ + "gas_resistance", + :float, + { + payload: %w[gas_resistance gasResistance], + telemetry: %w[gas_resistance gasResistance], + environment: %w[gas_resistance gasResistance], + }, + ], + [ + "current", + :float, + { + payload: %w[current current_ma currentMa], + telemetry: %w[current current_ma currentMa], + device: %w[current current_ma currentMa], + environment: %w[current], + }, + ], + [ + "iaq", + :integer, + { + payload: %w[iaq iaqIndex iaq_index], + telemetry: %w[iaq iaqIndex iaq_index], + environment: %w[iaq iaqIndex iaq_index], + }, + ], + [ + "distance", + :float, + { + payload: %w[distance range rangeMeters], + telemetry: %w[distance range rangeMeters], + environment: %w[distance range rangeMeters], + }, + ], + [ + "lux", + :float, + { + payload: %w[lux illuminance lightLux], + telemetry: %w[lux illuminance lightLux], + environment: %w[lux illuminance lightLux], + }, + ], + [ + "white_lux", + :float, + { + payload: %w[white_lux whiteLux], + telemetry: %w[white_lux whiteLux], + environment: %w[white_lux whiteLux], + }, + ], + [ + "ir_lux", + :float, + { + payload: %w[ir_lux irLux], + telemetry: %w[ir_lux irLux], + environment: %w[ir_lux irLux], + }, + ], + [ + "uv_lux", + :float, + { + payload: %w[uv_lux uvLux uvIndex], + telemetry: %w[uv_lux uvLux uvIndex], + environment: %w[uv_lux uvLux uvIndex], + }, + ], + [ + "wind_direction", + :integer, + { + payload: %w[wind_direction windDirection], + telemetry: %w[wind_direction windDirection], + environment: %w[wind_direction windDirection], + }, + ], + [ + "wind_speed", + :float, + { + payload: %w[wind_speed windSpeed windSpeedMps], + telemetry: %w[wind_speed windSpeed windSpeedMps], + environment: %w[wind_speed windSpeed windSpeedMps], + }, + ], + [ + "weight", + :float, + { + payload: %w[weight mass], + telemetry: %w[weight mass], + environment: %w[weight mass], + }, + ], + [ + "wind_gust", + :float, + { + payload: %w[wind_gust windGust], + telemetry: %w[wind_gust windGust], + environment: %w[wind_gust windGust], + }, + ], + [ + "wind_lull", + :float, + { + payload: %w[wind_lull windLull], + telemetry: %w[wind_lull windLull], + environment: %w[wind_lull windLull], + }, + ], + [ + "radiation", + :float, + { + payload: %w[radiation radiationLevel], + telemetry: %w[radiation radiationLevel], + environment: %w[radiation radiationLevel], + }, + ], + [ + "rainfall_1h", + :float, + { + payload: %w[rainfall_1h rainfall1h rainfallOneHour], + telemetry: %w[rainfall_1h rainfall1h rainfallOneHour], + environment: %w[rainfall_1h rainfall1h rainfallOneHour], + }, + ], + [ + "rainfall_24h", + :float, + { + payload: %w[rainfall_24h rainfall24h rainfallTwentyFourHour], + telemetry: %w[rainfall_24h rainfall24h rainfallTwentyFourHour], + environment: %w[rainfall_24h rainfall24h rainfallTwentyFourHour], + }, + ], + [ + "soil_moisture", + :integer, + { + payload: %w[soil_moisture soilMoisture], + telemetry: %w[soil_moisture soilMoisture], + environment: %w[soil_moisture soilMoisture], + }, + ], + [ + "soil_temperature", + :float, + { + payload: %w[soil_temperature soilTemperature], + telemetry: %w[soil_temperature soilTemperature], + environment: %w[soil_temperature soilTemperature], + }, + ], + ] + + metric_values = {} + metric_definitions.each do |column, type, key_map| + value = resolve_numeric_metric(key_map, sources, type) + metric_values[column] = value unless value.nil? end - battery_level = payload.key?("battery_level") ? payload["battery_level"] : nil - battery_level = coerce_float(battery_level) - battery_level ||= coerce_float(fetch_metric.call(device_metrics, :battery_level, :batteryLevel)) - - voltage = payload.key?("voltage") ? payload["voltage"] : nil - voltage = coerce_float(voltage) - voltage ||= coerce_float(fetch_metric.call(device_metrics, :voltage)) - - channel_utilization = payload.key?("channel_utilization") ? payload["channel_utilization"] : nil - channel_utilization ||= payload["channelUtilization"] if payload.key?("channelUtilization") - channel_utilization = coerce_float(channel_utilization) - channel_utilization ||= coerce_float(fetch_metric.call(device_metrics, :channel_utilization, :channelUtilization)) - - air_util_tx = payload.key?("air_util_tx") ? payload["air_util_tx"] : nil - air_util_tx ||= payload["airUtilTx"] if payload.key?("airUtilTx") - air_util_tx = coerce_float(air_util_tx) - air_util_tx ||= coerce_float(fetch_metric.call(device_metrics, :air_util_tx, :airUtilTx)) - - uptime_seconds = payload.key?("uptime_seconds") ? payload["uptime_seconds"] : nil - uptime_seconds ||= payload["uptimeSeconds"] if payload.key?("uptimeSeconds") - uptime_seconds = coerce_integer(uptime_seconds) - uptime_seconds ||= coerce_integer(fetch_metric.call(device_metrics, :uptime_seconds, :uptimeSeconds)) - - temperature = payload.key?("temperature") ? payload["temperature"] : nil - temperature = coerce_float(temperature) - temperature ||= coerce_float(fetch_metric.call(environment_metrics, :temperature, :temperatureC, :temperature_c, :tempC)) - - relative_humidity = payload.key?("relative_humidity") ? payload["relative_humidity"] : nil - relative_humidity ||= payload["relativeHumidity"] if payload.key?("relativeHumidity") - relative_humidity ||= payload["humidity"] if payload.key?("humidity") - relative_humidity = coerce_float(relative_humidity) - relative_humidity ||= coerce_float(fetch_metric.call(environment_metrics, :relative_humidity, :relativeHumidity, :humidity)) - - barometric_pressure = payload.key?("barometric_pressure") ? payload["barometric_pressure"] : nil - barometric_pressure ||= payload["barometricPressure"] if payload.key?("barometricPressure") - barometric_pressure ||= payload["pressure"] if payload.key?("pressure") - barometric_pressure = coerce_float(barometric_pressure) - barometric_pressure ||= coerce_float(fetch_metric.call(environment_metrics, :barometric_pressure, :barometricPressure, :pressure)) + battery_level = metric_values["battery_level"] + voltage = metric_values["voltage"] + channel_utilization = metric_values["channel_utilization"] + air_util_tx = metric_values["air_util_tx"] + uptime_seconds = metric_values["uptime_seconds"] + temperature = metric_values["temperature"] + relative_humidity = metric_values["relative_humidity"] + barometric_pressure = metric_values["barometric_pressure"] + gas_resistance = metric_values["gas_resistance"] + current = metric_values["current"] + iaq = metric_values["iaq"] + distance = metric_values["distance"] + lux = metric_values["lux"] + white_lux = metric_values["white_lux"] + ir_lux = metric_values["ir_lux"] + uv_lux = metric_values["uv_lux"] + wind_direction = metric_values["wind_direction"] + wind_speed = metric_values["wind_speed"] + weight = metric_values["weight"] + wind_gust = metric_values["wind_gust"] + wind_lull = metric_values["wind_lull"] + radiation = metric_values["radiation"] + rainfall_1h = metric_values["rainfall_1h"] + rainfall_24h = metric_values["rainfall_24h"] + soil_moisture = metric_values["soil_moisture"] + soil_temperature = metric_values["soil_temperature"] row = [ telemetry_id, @@ -849,13 +1124,33 @@ module PotatoMesh temperature, relative_humidity, barometric_pressure, + gas_resistance, + current, + iaq, + distance, + lux, + white_lux, + ir_lux, + uv_lux, + wind_direction, + wind_speed, + weight, + wind_gust, + wind_lull, + radiation, + rainfall_1h, + rainfall_24h, + soil_moisture, + soil_temperature, ] + placeholders = Array.new(row.length, "?").join(",") + with_busy_retry do db.execute <<~SQL, row INSERT INTO telemetry(id,node_id,node_num,from_id,to_id,rx_time,rx_iso,telemetry_time,channel,portnum,hop_limit,snr,rssi,bitfield,payload_b64, - battery_level,voltage,channel_utilization,air_util_tx,uptime_seconds,temperature,relative_humidity,barometric_pressure) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + battery_level,voltage,channel_utilization,air_util_tx,uptime_seconds,temperature,relative_humidity,barometric_pressure,gas_resistance,current,iaq,distance,lux,white_lux,ir_lux,uv_lux,wind_direction,wind_speed,weight,wind_gust,wind_lull,radiation,rainfall_1h,rainfall_24h,soil_moisture,soil_temperature) + VALUES (#{placeholders}) ON CONFLICT(id) DO UPDATE SET node_id=COALESCE(excluded.node_id,telemetry.node_id), node_num=COALESCE(excluded.node_num,telemetry.node_num), @@ -878,7 +1173,25 @@ module PotatoMesh uptime_seconds=COALESCE(excluded.uptime_seconds,telemetry.uptime_seconds), temperature=COALESCE(excluded.temperature,telemetry.temperature), relative_humidity=COALESCE(excluded.relative_humidity,telemetry.relative_humidity), - barometric_pressure=COALESCE(excluded.barometric_pressure,telemetry.barometric_pressure) + barometric_pressure=COALESCE(excluded.barometric_pressure,telemetry.barometric_pressure), + gas_resistance=COALESCE(excluded.gas_resistance,telemetry.gas_resistance), + current=COALESCE(excluded.current,telemetry.current), + iaq=COALESCE(excluded.iaq,telemetry.iaq), + distance=COALESCE(excluded.distance,telemetry.distance), + lux=COALESCE(excluded.lux,telemetry.lux), + white_lux=COALESCE(excluded.white_lux,telemetry.white_lux), + ir_lux=COALESCE(excluded.ir_lux,telemetry.ir_lux), + uv_lux=COALESCE(excluded.uv_lux,telemetry.uv_lux), + wind_direction=COALESCE(excluded.wind_direction,telemetry.wind_direction), + wind_speed=COALESCE(excluded.wind_speed,telemetry.wind_speed), + weight=COALESCE(excluded.weight,telemetry.weight), + wind_gust=COALESCE(excluded.wind_gust,telemetry.wind_gust), + wind_lull=COALESCE(excluded.wind_lull,telemetry.wind_lull), + radiation=COALESCE(excluded.radiation,telemetry.radiation), + rainfall_1h=COALESCE(excluded.rainfall_1h,telemetry.rainfall_1h), + rainfall_24h=COALESCE(excluded.rainfall_24h,telemetry.rainfall_24h), + soil_moisture=COALESCE(excluded.soil_moisture,telemetry.soil_moisture), + soil_temperature=COALESCE(excluded.soil_temperature,telemetry.soil_temperature) SQL end diff --git a/web/lib/potato_mesh/application/database.rb b/web/lib/potato_mesh/application/database.rb index b82b8c3..7589fd2 100644 --- a/web/lib/potato_mesh/application/database.rb +++ b/web/lib/potato_mesh/application/database.rb @@ -15,6 +15,30 @@ module PotatoMesh module App module Database + # Column definitions required for environment telemetry support. Each + # entry pairs the column name with the SQL type used when backfilling + # legacy databases that pre-date the extended telemetry schema. + TELEMETRY_COLUMN_DEFINITIONS = [ + ["gas_resistance", "REAL"], + ["current", "REAL"], + ["iaq", "INTEGER"], + ["distance", "REAL"], + ["lux", "REAL"], + ["white_lux", "REAL"], + ["ir_lux", "REAL"], + ["uv_lux", "REAL"], + ["wind_direction", "INTEGER"], + ["wind_speed", "REAL"], + ["weight", "REAL"], + ["wind_gust", "REAL"], + ["wind_lull", "REAL"], + ["radiation", "REAL"], + ["rainfall_1h", "REAL"], + ["rainfall_24h", "REAL"], + ["soil_moisture", "INTEGER"], + ["soil_temperature", "REAL"], + ].freeze + # Open a connection to the application database applying common pragmas. # # @param readonly [Boolean] whether to open the database in read-only mode. @@ -119,6 +143,21 @@ module PotatoMesh sql_file = File.expand_path("../../../../data/instances.sql", __dir__) db.execute_batch(File.read(sql_file)) end + + telemetry_tables = + db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='telemetry'").flatten + if telemetry_tables.empty? + telemetry_schema = File.expand_path("../../../../data/telemetry.sql", __dir__) + db.execute_batch(File.read(telemetry_schema)) + end + + telemetry_columns = db.execute("PRAGMA table_info(telemetry)").map { |row| row[1] } + TELEMETRY_COLUMN_DEFINITIONS.each do |name, type| + next if telemetry_columns.include?(name) + + db.execute("ALTER TABLE telemetry ADD COLUMN #{name} #{type}") + telemetry_columns << name + end rescue SQLite3::SQLException, Errno::ENOENT => e warn_log( "Failed to apply schema upgrade", diff --git a/web/lib/potato_mesh/application/queries.rb b/web/lib/potato_mesh/application/queries.rb index fbf9f82..12458fd 100644 --- a/web/lib/potato_mesh/application/queries.rb +++ b/web/lib/potato_mesh/application/queries.rb @@ -400,6 +400,24 @@ module PotatoMesh r["temperature"] = coerce_float(r["temperature"]) r["relative_humidity"] = coerce_float(r["relative_humidity"]) r["barometric_pressure"] = coerce_float(r["barometric_pressure"]) + r["gas_resistance"] = coerce_float(r["gas_resistance"]) + r["current"] = coerce_float(r["current"]) + r["iaq"] = coerce_integer(r["iaq"]) + r["distance"] = coerce_float(r["distance"]) + r["lux"] = coerce_float(r["lux"]) + r["white_lux"] = coerce_float(r["white_lux"]) + r["ir_lux"] = coerce_float(r["ir_lux"]) + r["uv_lux"] = coerce_float(r["uv_lux"]) + r["wind_direction"] = coerce_integer(r["wind_direction"]) + r["wind_speed"] = coerce_float(r["wind_speed"]) + r["weight"] = coerce_float(r["weight"]) + r["wind_gust"] = coerce_float(r["wind_gust"]) + r["wind_lull"] = coerce_float(r["wind_lull"]) + r["radiation"] = coerce_float(r["radiation"]) + r["rainfall_1h"] = coerce_float(r["rainfall_1h"]) + r["rainfall_24h"] = coerce_float(r["rainfall_24h"]) + r["soil_moisture"] = coerce_integer(r["soil_moisture"]) + r["soil_temperature"] = coerce_float(r["soil_temperature"]) end rows ensure diff --git a/web/spec/app_spec.rb b/web/spec/app_spec.rb index 25ca06a..7f50440 100644 --- a/web/spec/app_spec.rb +++ b/web/spec/app_spec.rb @@ -2875,11 +2875,57 @@ RSpec.describe "Potato Mesh Sinatra app" do expect_same_value(first["channel_utilization"], payload[0].dig("device_metrics", "channelUtilization")) expect_same_value(first["air_util_tx"], payload[0].dig("device_metrics", "airUtilTx")) expect(first["uptime_seconds"]).to eq(payload[0].dig("device_metrics", "uptimeSeconds")) + expect_same_value(first["current"], payload[0]["current"]) + expect_same_value(first["gas_resistance"], payload[0]["gas_resistance"]) + expect_same_value(first["iaq"], payload[0]["iaq"]) + expect_same_value(first["distance"], payload[0]["distance"]) + expect_same_value(first["lux"], payload[0]["lux"]) + expect_same_value(first["white_lux"], payload[0]["white_lux"]) + expect_same_value(first["ir_lux"], payload[0]["ir_lux"]) + expect_same_value(first["uv_lux"], payload[0]["uv_lux"]) + expect_same_value(first["wind_direction"], payload[0]["wind_direction"]) + expect_same_value(first["wind_speed"], payload[0]["wind_speed"]) + expect_same_value(first["wind_gust"], payload[0]["wind_gust"]) + expect_same_value(first["wind_lull"], payload[0]["wind_lull"]) + expect_same_value(first["weight"], payload[0]["weight"]) + expect_same_value(first["radiation"], payload[0]["radiation"]) + expect_same_value(first["rainfall_1h"], payload[0]["rainfall_1h"]) + expect_same_value(first["rainfall_24h"], payload[0]["rainfall_24h"]) + expect_same_value(first["soil_moisture"], payload[0]["soil_moisture"]) + expect_same_value(first["soil_temperature"], payload[0]["soil_temperature"]) environment_row = rows.find { |row| row["id"] == payload[1]["id"] } expect(environment_row["temperature"]).to be_within(1e-6).of(payload[1].dig("environment_metrics", "temperature")) expect(environment_row["relative_humidity"]).to be_within(1e-6).of(payload[1].dig("environment_metrics", "relativeHumidity")) expect(environment_row["barometric_pressure"]).to be_within(1e-6).of(payload[1].dig("environment_metrics", "barometricPressure")) + expect_same_value(environment_row["gas_resistance"], payload[1].dig("environment_metrics", "gasResistance")) + expect_same_value(environment_row["iaq"], payload[1].dig("environment_metrics", "iaq")) + expect_same_value(environment_row["distance"], payload[1].dig("environment_metrics", "distance")) + expect_same_value(environment_row["lux"], payload[1].dig("environment_metrics", "lux")) + expect_same_value(environment_row["white_lux"], payload[1].dig("environment_metrics", "whiteLux")) + expect_same_value(environment_row["ir_lux"], payload[1].dig("environment_metrics", "irLux")) + expect_same_value(environment_row["uv_lux"], payload[1].dig("environment_metrics", "uvLux")) + expect_same_value(environment_row["wind_direction"], payload[1].dig("environment_metrics", "windDirection")) + expect_same_value(environment_row["wind_speed"], payload[1].dig("environment_metrics", "windSpeed")) + expect_same_value(environment_row["wind_gust"], payload[1].dig("environment_metrics", "windGust")) + expect_same_value(environment_row["wind_lull"], payload[1].dig("environment_metrics", "windLull")) + expect_same_value(environment_row["weight"], payload[1].dig("environment_metrics", "weight")) + expect_same_value(environment_row["radiation"], payload[1].dig("environment_metrics", "radiation")) + expect_same_value(environment_row["rainfall_1h"], payload[1].dig("environment_metrics", "rainfall1h")) + expect_same_value(environment_row["rainfall_24h"], payload[1].dig("environment_metrics", "rainfall24h")) + expect_same_value(environment_row["soil_moisture"], payload[1].dig("environment_metrics", "soilMoisture")) + expect_same_value(environment_row["soil_temperature"], payload[1].dig("environment_metrics", "soilTemperature")) + + third_row = rows.find { |row| row["id"] == payload[2]["id"] } + expect_same_value(third_row["current"], payload[2].dig("device_metrics", "current")) + expect_same_value(third_row["distance"], payload[2].dig("environment_metrics", "distance")) + expect_same_value(third_row["lux"], payload[2].dig("environment_metrics", "lux")) + expect_same_value(third_row["wind_direction"], payload[2].dig("environment_metrics", "windDirection")) + expect_same_value(third_row["wind_speed"], payload[2].dig("environment_metrics", "windSpeed")) + expect_same_value(third_row["weight"], payload[2].dig("environment_metrics", "weight")) + expect_same_value(third_row["rainfall_24h"], payload[2].dig("environment_metrics", "rainfall24h")) + expect_same_value(third_row["soil_moisture"], payload[2].dig("environment_metrics", "soilMoisture")) + expect_same_value(third_row["soil_temperature"], payload[2].dig("environment_metrics", "soilTemperature")) end with_db(readonly: true) do |db| @@ -3707,6 +3753,15 @@ RSpec.describe "Potato Mesh Sinatra app" do expect(first_entry["telemetry_time_iso"]).to eq(Time.at(latest["telemetry_time"]).utc.iso8601) expect(first_entry).not_to have_key("device_metrics") expect_same_value(first_entry["battery_level"], latest.dig("device_metrics", "battery_level") || latest.dig("device_metrics", "batteryLevel")) + expect_same_value(first_entry["current"], latest.dig("device_metrics", "current")) + expect_same_value(first_entry["distance"], latest.dig("environment_metrics", "distance")) + expect_same_value(first_entry["lux"], latest.dig("environment_metrics", "lux")) + expect_same_value(first_entry["wind_direction"], latest.dig("environment_metrics", "windDirection")) + expect_same_value(first_entry["wind_speed"], latest.dig("environment_metrics", "windSpeed")) + expect_same_value(first_entry["weight"], latest.dig("environment_metrics", "weight")) + expect_same_value(first_entry["rainfall_24h"], latest.dig("environment_metrics", "rainfall24h")) + expect_same_value(first_entry["soil_moisture"], latest.dig("environment_metrics", "soilMoisture")) + expect_same_value(first_entry["soil_temperature"], latest.dig("environment_metrics", "soilTemperature")) second_entry = data.last expect(second_entry["id"]).to eq(second_latest["id"]) @@ -3714,6 +3769,23 @@ RSpec.describe "Potato Mesh Sinatra app" do expect(second_entry["temperature"]).to be_within(1e-6).of(second_latest["environment_metrics"]["temperature"]) expect(second_entry["relative_humidity"]).to be_within(1e-6).of(second_latest["environment_metrics"]["relativeHumidity"]) expect(second_entry["barometric_pressure"]).to be_within(1e-6).of(second_latest["environment_metrics"]["barometricPressure"]) + expect_same_value(second_entry["gas_resistance"], second_latest.dig("environment_metrics", "gasResistance")) + expect_same_value(second_entry["iaq"], second_latest.dig("environment_metrics", "iaq")) + expect_same_value(second_entry["distance"], second_latest.dig("environment_metrics", "distance")) + expect_same_value(second_entry["lux"], second_latest.dig("environment_metrics", "lux")) + expect_same_value(second_entry["white_lux"], second_latest.dig("environment_metrics", "whiteLux")) + expect_same_value(second_entry["ir_lux"], second_latest.dig("environment_metrics", "irLux")) + expect_same_value(second_entry["uv_lux"], second_latest.dig("environment_metrics", "uvLux")) + expect_same_value(second_entry["wind_direction"], second_latest.dig("environment_metrics", "windDirection")) + expect_same_value(second_entry["wind_speed"], second_latest.dig("environment_metrics", "windSpeed")) + expect_same_value(second_entry["wind_gust"], second_latest.dig("environment_metrics", "windGust")) + expect_same_value(second_entry["wind_lull"], second_latest.dig("environment_metrics", "windLull")) + expect_same_value(second_entry["weight"], second_latest.dig("environment_metrics", "weight")) + expect_same_value(second_entry["radiation"], second_latest.dig("environment_metrics", "radiation")) + expect_same_value(second_entry["rainfall_1h"], second_latest.dig("environment_metrics", "rainfall1h")) + expect_same_value(second_entry["rainfall_24h"], second_latest.dig("environment_metrics", "rainfall24h")) + expect_same_value(second_entry["soil_moisture"], second_latest.dig("environment_metrics", "soilMoisture")) + expect_same_value(second_entry["soil_temperature"], second_latest.dig("environment_metrics", "soilTemperature")) end it "excludes telemetry entries older than seven days" do diff --git a/web/spec/database_spec.rb b/web/spec/database_spec.rb new file mode 100644 index 0000000..73d1e2e --- /dev/null +++ b/web/spec/database_spec.rb @@ -0,0 +1,166 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# frozen_string_literal: true + +require "spec_helper" +require "sqlite3" + +RSpec.describe PotatoMesh::App::Database do + let(:harness_class) do + Class.new do + extend PotatoMesh::App::Database + extend PotatoMesh::App::Helpers + + class << self + attr_reader :warnings + + # Capture warning log entries generated during migrations for + # inspection within the unit tests. + # + # @param message [String] warning message text. + # @param context [String] logical source of the log entry. + # @param metadata [Hash] structured metadata supplied by the caller. + # @return [void] + def warn_log(message, context:, **metadata) + @warnings ||= [] + @warnings << { message: message, context: context, metadata: metadata } + end + + # Capture debug log entries generated during migrations for + # completeness of the helper interface. + # + # @param message [String] debug message text. + # @param context [String] logical source of the log entry. + # @param metadata [Hash] structured metadata supplied by the caller. + # @return [void] + def debug_log(message, context:, **metadata) + @debug_entries ||= [] + @debug_entries << { message: message, context: context, metadata: metadata } + end + + # Reset captured log entries between test examples. + # + # @return [void] + def reset_logs! + @warnings = [] + @debug_entries = [] + end + end + end + end + + around do |example| + harness_class.reset_logs! + + Dir.mktmpdir("db-upgrade-spec-") do |dir| + db_path = File.join(dir, "mesh.db") + + RSpec::Mocks.with_temporary_scope do + allow(PotatoMesh::Config).to receive(:db_path).and_return(db_path) + allow(PotatoMesh::Config).to receive(:default_db_path).and_return(db_path) + allow(PotatoMesh::Config).to receive(:legacy_db_path).and_return(db_path) + + example.run + end + end + ensure + harness_class.reset_logs! + end + + # Retrieve column names for the requested table within the temporary + # database used for upgrade tests. + # + # @param table [String] table name whose columns should be returned. + # @return [Array] names of the columns defined on +table+. + def column_names_for(table) + db = SQLite3::Database.new(PotatoMesh::Config.db_path, readonly: true) + db.execute("PRAGMA table_info(#{table})").map { |row| row[1] } + ensure + db&.close + end + + it "adds missing telemetry columns when upgrading an existing schema" do + SQLite3::Database.new(PotatoMesh::Config.db_path) do |db| + db.execute("CREATE TABLE nodes(node_id TEXT)") + db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)") + db.execute <<~SQL + CREATE TABLE telemetry ( + id INTEGER PRIMARY KEY, + node_id TEXT, + node_num INTEGER, + from_id TEXT, + to_id TEXT, + rx_time INTEGER NOT NULL, + rx_iso TEXT NOT NULL, + telemetry_time INTEGER, + channel INTEGER, + portnum TEXT, + hop_limit INTEGER, + snr REAL, + rssi INTEGER, + bitfield INTEGER, + payload_b64 TEXT, + battery_level REAL, + voltage REAL, + channel_utilization REAL, + air_util_tx REAL, + uptime_seconds INTEGER, + temperature REAL, + relative_humidity REAL, + barometric_pressure REAL + ) + SQL + end + + harness_class.ensure_schema_upgrades + + telemetry_columns = column_names_for("telemetry") + expect(telemetry_columns).to include( + "gas_resistance", + "current", + "iaq", + "distance", + "lux", + "white_lux", + "ir_lux", + "uv_lux", + "wind_direction", + "wind_speed", + "weight", + "wind_gust", + "wind_lull", + "radiation", + "rainfall_1h", + "rainfall_24h", + "soil_moisture", + "soil_temperature", + ) + + expect { harness_class.ensure_schema_upgrades }.not_to raise_error + end + + it "initialises the telemetry table when it is missing" do + SQLite3::Database.new(PotatoMesh::Config.db_path) do |db| + db.execute("CREATE TABLE nodes(node_id TEXT)") + db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)") + end + + expect(column_names_for("telemetry")).to be_empty + + harness_class.ensure_schema_upgrades + + telemetry_columns = column_names_for("telemetry") + expect(telemetry_columns).to include("soil_temperature", "lux", "iaq") + expect(telemetry_columns).to include("rx_time", "battery_level") + end +end