adam/potato-mesh
1
0
Fork 0
mirror of https://github.com/l5yth/potato-mesh.git synced 2026-08-08 09:52:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add full support to telemetry schema and API (#385)

Browse Source 
* feat: auto-upgrade telemetry schema

* Ensure numeric metrics fallback to valid values

* Format data processing numeric metric lookup
This commit is contained in:
l5y
2025-10-18 15:19:33 +02:00
committed by GitHub
parent cf3949ef95
commit 8143fbd8f7
10 changed files with 1020 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/mesh_ingestor/handlers.py
+219
View File
@@ -460,6 +460,189 @@ def store_telemetry_packet(packet: Mapping, decoded: Mapping) -> None:
)
)
current = _coerce_float(
_first(
telemetry_section,
"current",
"deviceMetrics.current",
"deviceMetrics.current_ma",
"deviceMetrics.currentMa",
"environmentMetrics.current",
default=None,
)
)
gas_resistance = _coerce_float(
_first(
telemetry_section,
"gasResistance",
"gas_resistance",
"environmentMetrics.gasResistance",
"environmentMetrics.gas_resistance",
default=None,
)
)
iaq = _coerce_int(
_first(
telemetry_section,
"iaq",
"environmentMetrics.iaq",
"environmentMetrics.iaqIndex",
"environmentMetrics.iaq_index",
default=None,
)
)
distance = _coerce_float(
_first(
telemetry_section,
"distance",
"environmentMetrics.distance",
"environmentMetrics.range",
"environmentMetrics.rangeMeters",
default=None,
)
)
lux = _coerce_float(
_first(
telemetry_section,
"lux",
"environmentMetrics.lux",
"environmentMetrics.illuminance",
default=None,
)
)
white_lux = _coerce_float(
_first(
telemetry_section,
"whiteLux",
"white_lux",
"environmentMetrics.whiteLux",
"environmentMetrics.white_lux",
default=None,
)
)
ir_lux = _coerce_float(
_first(
telemetry_section,
"irLux",
"ir_lux",
"environmentMetrics.irLux",
"environmentMetrics.ir_lux",
default=None,
)
)
uv_lux = _coerce_float(
_first(
telemetry_section,
"uvLux",
"uv_lux",
"environmentMetrics.uvLux",
"environmentMetrics.uv_lux",
"environmentMetrics.uvIndex",
default=None,
)
)
wind_direction = _coerce_int(
_first(
telemetry_section,
"windDirection",
"wind_direction",
"environmentMetrics.windDirection",
"environmentMetrics.wind_direction",
default=None,
)
)
wind_speed = _coerce_float(
_first(
telemetry_section,
"windSpeed",
"wind_speed",
"environmentMetrics.windSpeed",
"environmentMetrics.wind_speed",
"environmentMetrics.windSpeedMps",
default=None,
)
)
wind_gust = _coerce_float(
_first(
telemetry_section,
"windGust",
"wind_gust",
"environmentMetrics.windGust",
"environmentMetrics.wind_gust",
default=None,
)
)
wind_lull = _coerce_float(
_first(
telemetry_section,
"windLull",
"wind_lull",
"environmentMetrics.windLull",
"environmentMetrics.wind_lull",
default=None,
)
)
weight = _coerce_float(
_first(
telemetry_section,
"weight",
"environmentMetrics.weight",
"environmentMetrics.mass",
default=None,
)
)
radiation = _coerce_float(
_first(
telemetry_section,
"radiation",
"environmentMetrics.radiation",
"environmentMetrics.radiationLevel",
default=None,
)
)
rainfall_1h = _coerce_float(
_first(
telemetry_section,
"rainfall1h",
"rainfall_1h",
"environmentMetrics.rainfall1h",
"environmentMetrics.rainfall_1h",
"environmentMetrics.rainfallOneHour",
default=None,
)
)
rainfall_24h = _coerce_float(
_first(
telemetry_section,
"rainfall24h",
"rainfall_24h",
"environmentMetrics.rainfall24h",
"environmentMetrics.rainfall_24h",
"environmentMetrics.rainfallTwentyFourHour",
default=None,
)
)
soil_moisture = _coerce_int(
_first(
telemetry_section,
"soilMoisture",
"soil_moisture",
"environmentMetrics.soilMoisture",
"environmentMetrics.soil_moisture",
default=None,
)
)
soil_temperature = _coerce_float(
_first(
telemetry_section,
"soilTemperature",
"soil_temperature",
"environmentMetrics.soilTemperature",
"environmentMetrics.soil_temperature",
default=None,
)
)
telemetry_payload = {
"id": pkt_id,
"node_id": node_id,
@@ -494,6 +677,42 @@ def store_telemetry_packet(packet: Mapping, decoded: Mapping) -> None:
telemetry_payload["relative_humidity"] = relative_humidity
if barometric_pressure is not None:
telemetry_payload["barometric_pressure"] = barometric_pressure
if current is not None:
telemetry_payload["current"] = current
if gas_resistance is not None:
telemetry_payload["gas_resistance"] = gas_resistance
if iaq is not None:
telemetry_payload["iaq"] = iaq
if distance is not None:
telemetry_payload["distance"] = distance
if lux is not None:
telemetry_payload["lux"] = lux
if white_lux is not None:
telemetry_payload["white_lux"] = white_lux
if ir_lux is not None:
telemetry_payload["ir_lux"] = ir_lux
if uv_lux is not None:
telemetry_payload["uv_lux"] = uv_lux
if wind_direction is not None:
telemetry_payload["wind_direction"] = wind_direction
if wind_speed is not None:
telemetry_payload["wind_speed"] = wind_speed
if wind_gust is not None:
telemetry_payload["wind_gust"] = wind_gust
if wind_lull is not None:
telemetry_payload["wind_lull"] = wind_lull
if weight is not None:
telemetry_payload["weight"] = weight
if radiation is not None:
telemetry_payload["radiation"] = radiation
if rainfall_1h is not None:
telemetry_payload["rainfall_1h"] = rainfall_1h
if rainfall_24h is not None:
telemetry_payload["rainfall_24h"] = rainfall_24h
if soil_moisture is not None:
telemetry_payload["soil_moisture"] = soil_moisture
if soil_temperature is not None:
telemetry_payload["soil_temperature"] = soil_temperature
_queue_post_json(
"/api/telemetry",
data/migrations/20250305_extend_environment_telemetry.sql
+35
View File
@@ -0,0 +1,35 @@
-- Copyright (C) 2025 l5yth
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-- Extend the telemetry table with additional environment metrics.
BEGIN;
ALTER TABLE telemetry ADD COLUMN gas_resistance REAL;
ALTER TABLE telemetry ADD COLUMN current REAL;
ALTER TABLE telemetry ADD COLUMN iaq INTEGER;
ALTER TABLE telemetry ADD COLUMN distance REAL;
ALTER TABLE telemetry ADD COLUMN lux REAL;
ALTER TABLE telemetry ADD COLUMN white_lux REAL;
ALTER TABLE telemetry ADD COLUMN ir_lux REAL;
ALTER TABLE telemetry ADD COLUMN uv_lux REAL;
ALTER TABLE telemetry ADD COLUMN wind_direction INTEGER;
ALTER TABLE telemetry ADD COLUMN wind_speed REAL;
ALTER TABLE telemetry ADD COLUMN weight REAL;
ALTER TABLE telemetry ADD COLUMN wind_gust REAL;
ALTER TABLE telemetry ADD COLUMN wind_lull REAL;
ALTER TABLE telemetry ADD COLUMN radiation REAL;
ALTER TABLE telemetry ADD COLUMN rainfall_1h REAL;
ALTER TABLE telemetry ADD COLUMN rainfall_24h REAL;
ALTER TABLE telemetry ADD COLUMN soil_moisture INTEGER;
ALTER TABLE telemetry ADD COLUMN soil_temperature REAL;
COMMIT;
data/telemetry.sql
+19 -1
View File
@@ -35,7 +35,25 @@ CREATE TABLE IF NOT EXISTS telemetry (
uptime_seconds INTEGER,
temperature REAL,
relative_humidity REAL,
barometric_pressure REAL
barometric_pressure REAL,
gas_resistance REAL,
current REAL,
iaq INTEGER,
distance REAL,
lux REAL,
white_lux REAL,
ir_lux REAL,
uv_lux REAL,
wind_direction INTEGER,
wind_speed REAL,
weight REAL,
wind_gust REAL,
wind_lull REAL,
radiation REAL,
rainfall_1h REAL,
rainfall_24h REAL,
soil_moisture INTEGER,
soil_temperature REAL
);
CREATE INDEX IF NOT EXISTS idx_telemetry_rx_time ON telemetry(rx_time);
tests/telemetry.json
+54 -3
View File
@@ -12,12 +12,31 @@
"battery_level": 101,
"bitfield": 1,
"payload_b64": "DTVr0mgSFQhlFQIrh0AdJb8YPyXYFSA9KJTPEg==",
"current": 0.0715,
"gas_resistance": 1456.0,
"iaq": 83,
"distance": 12.5,
"lux": 100.25,
"white_lux": 64.5,
"ir_lux": 12.75,
"uv_lux": 1.6,
"wind_direction": 270,
"wind_speed": 5.9,
"wind_gust": 7.4,
"wind_lull": 4.8,
"weight": 32.7,
"radiation": 0.45,
"rainfall_1h": 0.18,
"rainfall_24h": 1.42,
"soil_moisture": 3100,
"soil_temperature": 18.9,
"device_metrics": {
"batteryLevel": 101,
"voltage": 4.224,
"channelUtilization": 0.59666663,
"airUtilTx": 0.03908333,
"uptimeSeconds": 305044
"uptimeSeconds": 305044,
"current": 0.0715
},
"raw": {
"device_metrics": {
@@ -43,7 +62,24 @@
"environment_metrics": {
"temperature": 21.98,
"relativeHumidity": 39.475586,
"barometricPressure": 1017.8353
"barometricPressure": 1017.8353,
"gasResistance": 1456.0,
"iaq": 83,
"distance": 12.5,
"lux": 100.25,
"whiteLux": 64.5,
"irLux": 12.75,
"uvLux": 1.6,
"windDirection": 270,
"windSpeed": 5.9,
"windGust": 7.4,
"windLull": 4.8,
"weight": 32.7,
"radiation": 0.45,
"rainfall1h": 0.18,
"rainfall24h": 1.42,
"soilMoisture": 3100,
"soilTemperature": 18.9
},
"raw": {
"environment_metrics": {
@@ -70,7 +106,22 @@
"voltage": 3.92,
"channel_utilization": 0.284,
"air_util_tx": 0.051,
"uptime_seconds": 86400
"uptime_seconds": 86400,
"current": 0.033
},
"environment_metrics": {
"temperature": 19.5,
"relative_humidity": 48.2,
"barometric_pressure": 1013.1,
"distance": 7.25,
"lux": 75.5,
"whiteLux": 40.0,
"windDirection": 180,
"windSpeed": 4.3,
"weight": 28.4,
"rainfall24h": 0.75,
"soilMoisture": 2850,
"soilTemperature": 17.1
},
"local_stats": {
"numPacketsTx": 1280,
tests/test_mesh.py
+36
View File
@@ -1575,6 +1575,7 @@ def test_store_packet_dict_handles_telemetry_packet(mesh_module, monkeypatch):
"channelUtilization": 0.59666663,
"airUtilTx": 0.03908333,
"uptimeSeconds": 305044,
"current": 0.0715,
},
"localStats": {
"numPacketsTx": 1280,
@@ -1606,6 +1607,7 @@ def test_store_packet_dict_handles_telemetry_packet(mesh_module, monkeypatch):
assert payload["channel_utilization"] == pytest.approx(0.59666663)
assert payload["air_util_tx"] == pytest.approx(0.03908333)
assert payload["uptime_seconds"] == 305044
assert payload["current"] == pytest.approx(0.0715)
assert payload["lora_freq"] == 868
assert payload["modem_preset"] == "MediumFast"
@@ -1634,6 +1636,23 @@ def test_store_packet_dict_handles_environment_telemetry(mesh_module, monkeypatc
"temperature": 21.98,
"relativeHumidity": 39.475586,
"barometricPressure": 1017.8353,
"gasResistance": 1456.0,
"iaq": 83,
"distance": 12.5,
"lux": 100.25,
"whiteLux": 64.5,
"irLux": 12.75,
"uvLux": 1.6,
"windDirection": 270,
"windSpeed": 5.9,
"windGust": 7.4,
"windLull": 4.8,
"weight": 32.7,
"radiation": 0.45,
"rainfall1h": 0.18,
"rainfall24h": 1.42,
"soilMoisture": 3100,
"soilTemperature": 18.9,
},
},
},
@@ -1651,6 +1670,23 @@ def test_store_packet_dict_handles_environment_telemetry(mesh_module, monkeypatc
assert payload["temperature"] == pytest.approx(21.98)
assert payload["relative_humidity"] == pytest.approx(39.475586)
assert payload["barometric_pressure"] == pytest.approx(1017.8353)
assert payload["gas_resistance"] == pytest.approx(1456.0)
assert payload["iaq"] == 83
assert payload["distance"] == pytest.approx(12.5)
assert payload["lux"] == pytest.approx(100.25)
assert payload["white_lux"] == pytest.approx(64.5)
assert payload["ir_lux"] == pytest.approx(12.75)
assert payload["uv_lux"] == pytest.approx(1.6)
assert payload["wind_direction"] == 270
assert payload["wind_speed"] == pytest.approx(5.9)
assert payload["wind_gust"] == pytest.approx(7.4)
assert payload["wind_lull"] == pytest.approx(4.8)
assert payload["weight"] == pytest.approx(32.7)
assert payload["radiation"] == pytest.approx(0.45)
assert payload["rainfall_1h"] == pytest.approx(0.18)
assert payload["rainfall_24h"] == pytest.approx(1.42)
assert payload["soil_moisture"] == 3100
assert payload["soil_temperature"] == pytest.approx(18.9)
assert payload["lora_freq"] == 868
assert payload["modem_preset"] == "MediumFast"
web/lib/potato_mesh/application/data_processing.rb
+362 -49
View File
@@ -731,6 +731,50 @@ module PotatoMesh
end
end
# Resolve a telemetry metric from the provided data sources.
#
# @param key_map [Hash{Symbol=>Array<String>}] ordered mapping of source names to candidate keys.
# @param sources [Hash{Symbol=>Hash}] data structures to search for metric values.
# @param type [Symbol] coercion strategy, ``:float`` or ``:integer``.
# @return [Numeric, nil] coerced metric value or nil when no candidates exist.
def resolve_numeric_metric(key_map, sources, type)
key_map.each do |source, keys|
next if keys.nil? || keys.empty?
data = sources[source]
next unless data.is_a?(Hash)
keys.each do |name|
next if name.nil?
key = name.to_s
value = if data.key?(key)
data[key]
else
sym_key = key.to_sym
data.key?(sym_key) ? data[sym_key] : nil
end
next if value.nil?
coerced = case type
when :float
coerce_float(value)
when :integer
coerce_integer(value)
else
value
end
return coerced unless coerced.nil?
end
end
nil
end
private :resolve_numeric_metric
def insert_telemetry(db, payload)
return unless payload.is_a?(Hash)
@@ -776,54 +820,285 @@ module PotatoMesh
environment_metrics = normalize_json_object(payload["environment_metrics"] || payload["environmentMetrics"])
environment_metrics ||= normalize_json_object(telemetry_section["environmentMetrics"]) if telemetry_section&.key?("environmentMetrics")
fetch_metric = lambda do |map, *names|
next nil unless map.is_a?(Hash)
names.each do |name|
next unless name
key = name.to_s
return map[key] if map.key?(key)
end
nil
sources = {
payload: payload,
telemetry: telemetry_section,
device: device_metrics,
environment: environment_metrics,
}
metric_definitions = [
[
"battery_level",
:float,
{
payload: %w[battery_level batteryLevel],
telemetry: %w[batteryLevel],
device: %w[battery_level batteryLevel],
environment: %w[battery_level batteryLevel],
},
],
[
"voltage",
:float,
{
payload: %w[voltage],
telemetry: %w[voltage],
device: %w[voltage],
environment: %w[voltage],
},
],
[
"channel_utilization",
:float,
{
payload: %w[channel_utilization channelUtilization],
telemetry: %w[channelUtilization],
device: %w[channel_utilization channelUtilization],
},
],
[
"air_util_tx",
:float,
{
payload: %w[air_util_tx airUtilTx],
telemetry: %w[airUtilTx],
device: %w[air_util_tx airUtilTx],
},
],
[
"uptime_seconds",
:integer,
{
payload: %w[uptime_seconds uptimeSeconds],
telemetry: %w[uptimeSeconds],
device: %w[uptime_seconds uptimeSeconds],
},
],
[
"temperature",
:float,
{
payload: %w[temperature temperatureC tempC],
telemetry: %w[temperature temperatureC tempC],
environment: %w[temperature temperatureC temperature_c tempC],
},
],
[
"relative_humidity",
:float,
{
payload: %w[relative_humidity relativeHumidity humidity],
telemetry: %w[relative_humidity relativeHumidity humidity],
environment: %w[relative_humidity relativeHumidity humidity],
},
],
[
"barometric_pressure",
:float,
{
payload: %w[barometric_pressure barometricPressure pressure],
telemetry: %w[barometric_pressure barometricPressure pressure],
environment: %w[barometric_pressure barometricPressure pressure],
},
],
[
"gas_resistance",
:float,
{
payload: %w[gas_resistance gasResistance],
telemetry: %w[gas_resistance gasResistance],
environment: %w[gas_resistance gasResistance],
},
],
[
"current",
:float,
{
payload: %w[current current_ma currentMa],
telemetry: %w[current current_ma currentMa],
device: %w[current current_ma currentMa],
environment: %w[current],
},
],
[
"iaq",
:integer,
{
payload: %w[iaq iaqIndex iaq_index],
telemetry: %w[iaq iaqIndex iaq_index],
environment: %w[iaq iaqIndex iaq_index],
},
],
[
"distance",
:float,
{
payload: %w[distance range rangeMeters],
telemetry: %w[distance range rangeMeters],
environment: %w[distance range rangeMeters],
},
],
[
"lux",
:float,
{
payload: %w[lux illuminance lightLux],
telemetry: %w[lux illuminance lightLux],
environment: %w[lux illuminance lightLux],
},
],
[
"white_lux",
:float,
{
payload: %w[white_lux whiteLux],
telemetry: %w[white_lux whiteLux],
environment: %w[white_lux whiteLux],
},
],
[
"ir_lux",
:float,
{
payload: %w[ir_lux irLux],
telemetry: %w[ir_lux irLux],
environment: %w[ir_lux irLux],
},
],
[
"uv_lux",
:float,
{
payload: %w[uv_lux uvLux uvIndex],
telemetry: %w[uv_lux uvLux uvIndex],
environment: %w[uv_lux uvLux uvIndex],
},
],
[
"wind_direction",
:integer,
{
payload: %w[wind_direction windDirection],
telemetry: %w[wind_direction windDirection],
environment: %w[wind_direction windDirection],
},
],
[
"wind_speed",
:float,
{
payload: %w[wind_speed windSpeed windSpeedMps],
telemetry: %w[wind_speed windSpeed windSpeedMps],
environment: %w[wind_speed windSpeed windSpeedMps],
},
],
[
"weight",
:float,
{
payload: %w[weight mass],
telemetry: %w[weight mass],
environment: %w[weight mass],
},
],
[
"wind_gust",
:float,
{
payload: %w[wind_gust windGust],
telemetry: %w[wind_gust windGust],
environment: %w[wind_gust windGust],
},
],
[
"wind_lull",
:float,
{
payload: %w[wind_lull windLull],
telemetry: %w[wind_lull windLull],
environment: %w[wind_lull windLull],
},
],
[
"radiation",
:float,
{
payload: %w[radiation radiationLevel],
telemetry: %w[radiation radiationLevel],
environment: %w[radiation radiationLevel],
},
],
[
"rainfall_1h",
:float,
{
payload: %w[rainfall_1h rainfall1h rainfallOneHour],
telemetry: %w[rainfall_1h rainfall1h rainfallOneHour],
environment: %w[rainfall_1h rainfall1h rainfallOneHour],
},
],
[
"rainfall_24h",
:float,
{
payload: %w[rainfall_24h rainfall24h rainfallTwentyFourHour],
telemetry: %w[rainfall_24h rainfall24h rainfallTwentyFourHour],
environment: %w[rainfall_24h rainfall24h rainfallTwentyFourHour],
},
],
[
"soil_moisture",
:integer,
{
payload: %w[soil_moisture soilMoisture],
telemetry: %w[soil_moisture soilMoisture],
environment: %w[soil_moisture soilMoisture],
},
],
[
"soil_temperature",
:float,
{
payload: %w[soil_temperature soilTemperature],
telemetry: %w[soil_temperature soilTemperature],
environment: %w[soil_temperature soilTemperature],
},
],
]
metric_values = {}
metric_definitions.each do |column, type, key_map|
value = resolve_numeric_metric(key_map, sources, type)
metric_values[column] = value unless value.nil?
end
battery_level = payload.key?("battery_level") ? payload["battery_level"] : nil
battery_level = coerce_float(battery_level)
battery_level ||= coerce_float(fetch_metric.call(device_metrics, :battery_level, :batteryLevel))
voltage = payload.key?("voltage") ? payload["voltage"] : nil
voltage = coerce_float(voltage)
voltage ||= coerce_float(fetch_metric.call(device_metrics, :voltage))
channel_utilization = payload.key?("channel_utilization") ? payload["channel_utilization"] : nil
channel_utilization ||= payload["channelUtilization"] if payload.key?("channelUtilization")
channel_utilization = coerce_float(channel_utilization)
channel_utilization ||= coerce_float(fetch_metric.call(device_metrics, :channel_utilization, :channelUtilization))
air_util_tx = payload.key?("air_util_tx") ? payload["air_util_tx"] : nil
air_util_tx ||= payload["airUtilTx"] if payload.key?("airUtilTx")
air_util_tx = coerce_float(air_util_tx)
air_util_tx ||= coerce_float(fetch_metric.call(device_metrics, :air_util_tx, :airUtilTx))
uptime_seconds = payload.key?("uptime_seconds") ? payload["uptime_seconds"] : nil
uptime_seconds ||= payload["uptimeSeconds"] if payload.key?("uptimeSeconds")
uptime_seconds = coerce_integer(uptime_seconds)
uptime_seconds ||= coerce_integer(fetch_metric.call(device_metrics, :uptime_seconds, :uptimeSeconds))
temperature = payload.key?("temperature") ? payload["temperature"] : nil
temperature = coerce_float(temperature)
temperature ||= coerce_float(fetch_metric.call(environment_metrics, :temperature, :temperatureC, :temperature_c, :tempC))
relative_humidity = payload.key?("relative_humidity") ? payload["relative_humidity"] : nil
relative_humidity ||= payload["relativeHumidity"] if payload.key?("relativeHumidity")
relative_humidity ||= payload["humidity"] if payload.key?("humidity")
relative_humidity = coerce_float(relative_humidity)
relative_humidity ||= coerce_float(fetch_metric.call(environment_metrics, :relative_humidity, :relativeHumidity, :humidity))
barometric_pressure = payload.key?("barometric_pressure") ? payload["barometric_pressure"] : nil
barometric_pressure ||= payload["barometricPressure"] if payload.key?("barometricPressure")
barometric_pressure ||= payload["pressure"] if payload.key?("pressure")
barometric_pressure = coerce_float(barometric_pressure)
barometric_pressure ||= coerce_float(fetch_metric.call(environment_metrics, :barometric_pressure, :barometricPressure, :pressure))
battery_level = metric_values["battery_level"]
voltage = metric_values["voltage"]
channel_utilization = metric_values["channel_utilization"]
air_util_tx = metric_values["air_util_tx"]
uptime_seconds = metric_values["uptime_seconds"]
temperature = metric_values["temperature"]
relative_humidity = metric_values["relative_humidity"]
barometric_pressure = metric_values["barometric_pressure"]
gas_resistance = metric_values["gas_resistance"]
current = metric_values["current"]
iaq = metric_values["iaq"]
distance = metric_values["distance"]
lux = metric_values["lux"]
white_lux = metric_values["white_lux"]
ir_lux = metric_values["ir_lux"]
uv_lux = metric_values["uv_lux"]
wind_direction = metric_values["wind_direction"]
wind_speed = metric_values["wind_speed"]
weight = metric_values["weight"]
wind_gust = metric_values["wind_gust"]
wind_lull = metric_values["wind_lull"]
radiation = metric_values["radiation"]
rainfall_1h = metric_values["rainfall_1h"]
rainfall_24h = metric_values["rainfall_24h"]
soil_moisture = metric_values["soil_moisture"]
soil_temperature = metric_values["soil_temperature"]
row = [
telemetry_id,
@@ -849,13 +1124,33 @@ module PotatoMesh
temperature,
relative_humidity,
barometric_pressure,
gas_resistance,
current,
iaq,
distance,
lux,
white_lux,
ir_lux,
uv_lux,
wind_direction,
wind_speed,
weight,
wind_gust,
wind_lull,
radiation,
rainfall_1h,
rainfall_24h,
soil_moisture,
soil_temperature,
]
placeholders = Array.new(row.length, "?").join(",")
with_busy_retry do
db.execute <<~SQL, row
INSERT INTO telemetry(id,node_id,node_num,from_id,to_id,rx_time,rx_iso,telemetry_time,channel,portnum,hop_limit,snr,rssi,bitfield,payload_b64,
battery_level,voltage,channel_utilization,air_util_tx,uptime_seconds,temperature,relative_humidity,barometric_pressure)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
battery_level,voltage,channel_utilization,air_util_tx,uptime_seconds,temperature,relative_humidity,barometric_pressure,gas_resistance,current,iaq,distance,lux,white_lux,ir_lux,uv_lux,wind_direction,wind_speed,weight,wind_gust,wind_lull,radiation,rainfall_1h,rainfall_24h,soil_moisture,soil_temperature)
VALUES (#{placeholders})
ON CONFLICT(id) DO UPDATE SET
node_id=COALESCE(excluded.node_id,telemetry.node_id),
node_num=COALESCE(excluded.node_num,telemetry.node_num),
@@ -878,7 +1173,25 @@ module PotatoMesh
uptime_seconds=COALESCE(excluded.uptime_seconds,telemetry.uptime_seconds),
temperature=COALESCE(excluded.temperature,telemetry.temperature),
relative_humidity=COALESCE(excluded.relative_humidity,telemetry.relative_humidity),
barometric_pressure=COALESCE(excluded.barometric_pressure,telemetry.barometric_pressure)
barometric_pressure=COALESCE(excluded.barometric_pressure,telemetry.barometric_pressure),
gas_resistance=COALESCE(excluded.gas_resistance,telemetry.gas_resistance),
current=COALESCE(excluded.current,telemetry.current),
iaq=COALESCE(excluded.iaq,telemetry.iaq),
distance=COALESCE(excluded.distance,telemetry.distance),
lux=COALESCE(excluded.lux,telemetry.lux),
white_lux=COALESCE(excluded.white_lux,telemetry.white_lux),
ir_lux=COALESCE(excluded.ir_lux,telemetry.ir_lux),
uv_lux=COALESCE(excluded.uv_lux,telemetry.uv_lux),
wind_direction=COALESCE(excluded.wind_direction,telemetry.wind_direction),
wind_speed=COALESCE(excluded.wind_speed,telemetry.wind_speed),
weight=COALESCE(excluded.weight,telemetry.weight),
wind_gust=COALESCE(excluded.wind_gust,telemetry.wind_gust),
wind_lull=COALESCE(excluded.wind_lull,telemetry.wind_lull),
radiation=COALESCE(excluded.radiation,telemetry.radiation),
rainfall_1h=COALESCE(excluded.rainfall_1h,telemetry.rainfall_1h),
rainfall_24h=COALESCE(excluded.rainfall_24h,telemetry.rainfall_24h),
soil_moisture=COALESCE(excluded.soil_moisture,telemetry.soil_moisture),
soil_temperature=COALESCE(excluded.soil_temperature,telemetry.soil_temperature)
SQL
end
web/lib/potato_mesh/application/database.rb
+39
View File
@@ -15,6 +15,30 @@
module PotatoMesh
module App
module Database
# Column definitions required for environment telemetry support. Each
# entry pairs the column name with the SQL type used when backfilling
# legacy databases that pre-date the extended telemetry schema.
TELEMETRY_COLUMN_DEFINITIONS = [
["gas_resistance", "REAL"],
["current", "REAL"],
["iaq", "INTEGER"],
["distance", "REAL"],
["lux", "REAL"],
["white_lux", "REAL"],
["ir_lux", "REAL"],
["uv_lux", "REAL"],
["wind_direction", "INTEGER"],
["wind_speed", "REAL"],
["weight", "REAL"],
["wind_gust", "REAL"],
["wind_lull", "REAL"],
["radiation", "REAL"],
["rainfall_1h", "REAL"],
["rainfall_24h", "REAL"],
["soil_moisture", "INTEGER"],
["soil_temperature", "REAL"],
].freeze
# Open a connection to the application database applying common pragmas.
#
# @param readonly [Boolean] whether to open the database in read-only mode.
@@ -119,6 +143,21 @@ module PotatoMesh
sql_file = File.expand_path("../../../../data/instances.sql", __dir__)
db.execute_batch(File.read(sql_file))
end
telemetry_tables =
db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='telemetry'").flatten
if telemetry_tables.empty?
telemetry_schema = File.expand_path("../../../../data/telemetry.sql", __dir__)
db.execute_batch(File.read(telemetry_schema))
end
telemetry_columns = db.execute("PRAGMA table_info(telemetry)").map { |row| row[1] }
TELEMETRY_COLUMN_DEFINITIONS.each do |name, type|
next if telemetry_columns.include?(name)
db.execute("ALTER TABLE telemetry ADD COLUMN #{name} #{type}")
telemetry_columns << name
end
rescue SQLite3::SQLException, Errno::ENOENT => e
warn_log(
"Failed to apply schema upgrade",
web/lib/potato_mesh/application/queries.rb
+18
View File
@@ -400,6 +400,24 @@ module PotatoMesh
r["temperature"] = coerce_float(r["temperature"])
r["relative_humidity"] = coerce_float(r["relative_humidity"])
r["barometric_pressure"] = coerce_float(r["barometric_pressure"])
r["gas_resistance"] = coerce_float(r["gas_resistance"])
r["current"] = coerce_float(r["current"])
r["iaq"] = coerce_integer(r["iaq"])
r["distance"] = coerce_float(r["distance"])
r["lux"] = coerce_float(r["lux"])
r["white_lux"] = coerce_float(r["white_lux"])
r["ir_lux"] = coerce_float(r["ir_lux"])
r["uv_lux"] = coerce_float(r["uv_lux"])
r["wind_direction"] = coerce_integer(r["wind_direction"])
r["wind_speed"] = coerce_float(r["wind_speed"])
r["weight"] = coerce_float(r["weight"])
r["wind_gust"] = coerce_float(r["wind_gust"])
r["wind_lull"] = coerce_float(r["wind_lull"])
r["radiation"] = coerce_float(r["radiation"])
r["rainfall_1h"] = coerce_float(r["rainfall_1h"])
r["rainfall_24h"] = coerce_float(r["rainfall_24h"])
r["soil_moisture"] = coerce_integer(r["soil_moisture"])
r["soil_temperature"] = coerce_float(r["soil_temperature"])
end
rows
ensure
web/spec/app_spec.rb
+72
View File
@@ -2875,11 +2875,57 @@ RSpec.describe "Potato Mesh Sinatra app" do
expect_same_value(first["channel_utilization"], payload[0].dig("device_metrics", "channelUtilization"))
expect_same_value(first["air_util_tx"], payload[0].dig("device_metrics", "airUtilTx"))
expect(first["uptime_seconds"]).to eq(payload[0].dig("device_metrics", "uptimeSeconds"))
expect_same_value(first["current"], payload[0]["current"])
expect_same_value(first["gas_resistance"], payload[0]["gas_resistance"])
expect_same_value(first["iaq"], payload[0]["iaq"])
expect_same_value(first["distance"], payload[0]["distance"])
expect_same_value(first["lux"], payload[0]["lux"])
expect_same_value(first["white_lux"], payload[0]["white_lux"])
expect_same_value(first["ir_lux"], payload[0]["ir_lux"])
expect_same_value(first["uv_lux"], payload[0]["uv_lux"])
expect_same_value(first["wind_direction"], payload[0]["wind_direction"])
expect_same_value(first["wind_speed"], payload[0]["wind_speed"])
expect_same_value(first["wind_gust"], payload[0]["wind_gust"])
expect_same_value(first["wind_lull"], payload[0]["wind_lull"])
expect_same_value(first["weight"], payload[0]["weight"])
expect_same_value(first["radiation"], payload[0]["radiation"])
expect_same_value(first["rainfall_1h"], payload[0]["rainfall_1h"])
expect_same_value(first["rainfall_24h"], payload[0]["rainfall_24h"])
expect_same_value(first["soil_moisture"], payload[0]["soil_moisture"])
expect_same_value(first["soil_temperature"], payload[0]["soil_temperature"])
environment_row = rows.find { |row| row["id"] == payload[1]["id"] }
expect(environment_row["temperature"]).to be_within(1e-6).of(payload[1].dig("environment_metrics", "temperature"))
expect(environment_row["relative_humidity"]).to be_within(1e-6).of(payload[1].dig("environment_metrics", "relativeHumidity"))
expect(environment_row["barometric_pressure"]).to be_within(1e-6).of(payload[1].dig("environment_metrics", "barometricPressure"))
expect_same_value(environment_row["gas_resistance"], payload[1].dig("environment_metrics", "gasResistance"))
expect_same_value(environment_row["iaq"], payload[1].dig("environment_metrics", "iaq"))
expect_same_value(environment_row["distance"], payload[1].dig("environment_metrics", "distance"))
expect_same_value(environment_row["lux"], payload[1].dig("environment_metrics", "lux"))
expect_same_value(environment_row["white_lux"], payload[1].dig("environment_metrics", "whiteLux"))
expect_same_value(environment_row["ir_lux"], payload[1].dig("environment_metrics", "irLux"))
expect_same_value(environment_row["uv_lux"], payload[1].dig("environment_metrics", "uvLux"))
expect_same_value(environment_row["wind_direction"], payload[1].dig("environment_metrics", "windDirection"))
expect_same_value(environment_row["wind_speed"], payload[1].dig("environment_metrics", "windSpeed"))
expect_same_value(environment_row["wind_gust"], payload[1].dig("environment_metrics", "windGust"))
expect_same_value(environment_row["wind_lull"], payload[1].dig("environment_metrics", "windLull"))
expect_same_value(environment_row["weight"], payload[1].dig("environment_metrics", "weight"))
expect_same_value(environment_row["radiation"], payload[1].dig("environment_metrics", "radiation"))
expect_same_value(environment_row["rainfall_1h"], payload[1].dig("environment_metrics", "rainfall1h"))
expect_same_value(environment_row["rainfall_24h"], payload[1].dig("environment_metrics", "rainfall24h"))
expect_same_value(environment_row["soil_moisture"], payload[1].dig("environment_metrics", "soilMoisture"))
expect_same_value(environment_row["soil_temperature"], payload[1].dig("environment_metrics", "soilTemperature"))
third_row = rows.find { |row| row["id"] == payload[2]["id"] }
expect_same_value(third_row["current"], payload[2].dig("device_metrics", "current"))
expect_same_value(third_row["distance"], payload[2].dig("environment_metrics", "distance"))
expect_same_value(third_row["lux"], payload[2].dig("environment_metrics", "lux"))
expect_same_value(third_row["wind_direction"], payload[2].dig("environment_metrics", "windDirection"))
expect_same_value(third_row["wind_speed"], payload[2].dig("environment_metrics", "windSpeed"))
expect_same_value(third_row["weight"], payload[2].dig("environment_metrics", "weight"))
expect_same_value(third_row["rainfall_24h"], payload[2].dig("environment_metrics", "rainfall24h"))
expect_same_value(third_row["soil_moisture"], payload[2].dig("environment_metrics", "soilMoisture"))
expect_same_value(third_row["soil_temperature"], payload[2].dig("environment_metrics", "soilTemperature"))
end
with_db(readonly: true) do |db|
@@ -3707,6 +3753,15 @@ RSpec.describe "Potato Mesh Sinatra app" do
expect(first_entry["telemetry_time_iso"]).to eq(Time.at(latest["telemetry_time"]).utc.iso8601)
expect(first_entry).not_to have_key("device_metrics")
expect_same_value(first_entry["battery_level"], latest.dig("device_metrics", "battery_level") || latest.dig("device_metrics", "batteryLevel"))
expect_same_value(first_entry["current"], latest.dig("device_metrics", "current"))
expect_same_value(first_entry["distance"], latest.dig("environment_metrics", "distance"))
expect_same_value(first_entry["lux"], latest.dig("environment_metrics", "lux"))
expect_same_value(first_entry["wind_direction"], latest.dig("environment_metrics", "windDirection"))
expect_same_value(first_entry["wind_speed"], latest.dig("environment_metrics", "windSpeed"))
expect_same_value(first_entry["weight"], latest.dig("environment_metrics", "weight"))
expect_same_value(first_entry["rainfall_24h"], latest.dig("environment_metrics", "rainfall24h"))
expect_same_value(first_entry["soil_moisture"], latest.dig("environment_metrics", "soilMoisture"))
expect_same_value(first_entry["soil_temperature"], latest.dig("environment_metrics", "soilTemperature"))
second_entry = data.last
expect(second_entry["id"]).to eq(second_latest["id"])
@@ -3714,6 +3769,23 @@ RSpec.describe "Potato Mesh Sinatra app" do
expect(second_entry["temperature"]).to be_within(1e-6).of(second_latest["environment_metrics"]["temperature"])
expect(second_entry["relative_humidity"]).to be_within(1e-6).of(second_latest["environment_metrics"]["relativeHumidity"])
expect(second_entry["barometric_pressure"]).to be_within(1e-6).of(second_latest["environment_metrics"]["barometricPressure"])
expect_same_value(second_entry["gas_resistance"], second_latest.dig("environment_metrics", "gasResistance"))
expect_same_value(second_entry["iaq"], second_latest.dig("environment_metrics", "iaq"))
expect_same_value(second_entry["distance"], second_latest.dig("environment_metrics", "distance"))
expect_same_value(second_entry["lux"], second_latest.dig("environment_metrics", "lux"))
expect_same_value(second_entry["white_lux"], second_latest.dig("environment_metrics", "whiteLux"))
expect_same_value(second_entry["ir_lux"], second_latest.dig("environment_metrics", "irLux"))
expect_same_value(second_entry["uv_lux"], second_latest.dig("environment_metrics", "uvLux"))
expect_same_value(second_entry["wind_direction"], second_latest.dig("environment_metrics", "windDirection"))
expect_same_value(second_entry["wind_speed"], second_latest.dig("environment_metrics", "windSpeed"))
expect_same_value(second_entry["wind_gust"], second_latest.dig("environment_metrics", "windGust"))
expect_same_value(second_entry["wind_lull"], second_latest.dig("environment_metrics", "windLull"))
expect_same_value(second_entry["weight"], second_latest.dig("environment_metrics", "weight"))
expect_same_value(second_entry["radiation"], second_latest.dig("environment_metrics", "radiation"))
expect_same_value(second_entry["rainfall_1h"], second_latest.dig("environment_metrics", "rainfall1h"))
expect_same_value(second_entry["rainfall_24h"], second_latest.dig("environment_metrics", "rainfall24h"))
expect_same_value(second_entry["soil_moisture"], second_latest.dig("environment_metrics", "soilMoisture"))
expect_same_value(second_entry["soil_temperature"], second_latest.dig("environment_metrics", "soilTemperature"))
end
it "excludes telemetry entries older than seven days" do
web/spec/database_spec.rb
+166
View File
@@ -0,0 +1,166 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# frozen_string_literal: true
require "spec_helper"
require "sqlite3"
RSpec.describe PotatoMesh::App::Database do
let(:harness_class) do
Class.new do
extend PotatoMesh::App::Database
extend PotatoMesh::App::Helpers
class << self
attr_reader :warnings
# Capture warning log entries generated during migrations for
# inspection within the unit tests.
#
# @param message [String] warning message text.
# @param context [String] logical source of the log entry.
# @param metadata [Hash] structured metadata supplied by the caller.
# @return [void]
def warn_log(message, context:, **metadata)
@warnings ||= []
@warnings << { message: message, context: context, metadata: metadata }
end
# Capture debug log entries generated during migrations for
# completeness of the helper interface.
#
# @param message [String] debug message text.
# @param context [String] logical source of the log entry.
# @param metadata [Hash] structured metadata supplied by the caller.
# @return [void]
def debug_log(message, context:, **metadata)
@debug_entries ||= []
@debug_entries << { message: message, context: context, metadata: metadata }
end
# Reset captured log entries between test examples.
#
# @return [void]
def reset_logs!
@warnings = []
@debug_entries = []
end
end
end
end
around do |example|
harness_class.reset_logs!
Dir.mktmpdir("db-upgrade-spec-") do |dir|
db_path = File.join(dir, "mesh.db")
RSpec::Mocks.with_temporary_scope do
allow(PotatoMesh::Config).to receive(:db_path).and_return(db_path)
allow(PotatoMesh::Config).to receive(:default_db_path).and_return(db_path)
allow(PotatoMesh::Config).to receive(:legacy_db_path).and_return(db_path)
example.run
end
end
ensure
harness_class.reset_logs!
end
# Retrieve column names for the requested table within the temporary
# database used for upgrade tests.
#
# @param table [String] table name whose columns should be returned.
# @return [Array<String>] names of the columns defined on +table+.
def column_names_for(table)
db = SQLite3::Database.new(PotatoMesh::Config.db_path, readonly: true)
db.execute("PRAGMA table_info(#{table})").map { |row| row[1] }
ensure
db&.close
end
it "adds missing telemetry columns when upgrading an existing schema" do
SQLite3::Database.new(PotatoMesh::Config.db_path) do |db|
db.execute("CREATE TABLE nodes(node_id TEXT)")
db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)")
db.execute <<~SQL
CREATE TABLE telemetry (
id INTEGER PRIMARY KEY,
node_id TEXT,
node_num INTEGER,
from_id TEXT,
to_id TEXT,
rx_time INTEGER NOT NULL,
rx_iso TEXT NOT NULL,
telemetry_time INTEGER,
channel INTEGER,
portnum TEXT,
hop_limit INTEGER,
snr REAL,
rssi INTEGER,
bitfield INTEGER,
payload_b64 TEXT,
battery_level REAL,
voltage REAL,
channel_utilization REAL,
air_util_tx REAL,
uptime_seconds INTEGER,
temperature REAL,
relative_humidity REAL,
barometric_pressure REAL
)
SQL
end
harness_class.ensure_schema_upgrades
telemetry_columns = column_names_for("telemetry")
expect(telemetry_columns).to include(
"gas_resistance",
"current",
"iaq",
"distance",
"lux",
"white_lux",
"ir_lux",
"uv_lux",
"wind_direction",
"wind_speed",
"weight",
"wind_gust",
"wind_lull",
"radiation",
"rainfall_1h",
"rainfall_24h",
"soil_moisture",
"soil_temperature",
)
expect { harness_class.ensure_schema_upgrades }.not_to raise_error
end
it "initialises the telemetry table when it is missing" do
SQLite3::Database.new(PotatoMesh::Config.db_path) do |db|
db.execute("CREATE TABLE nodes(node_id TEXT)")
db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)")
end
expect(column_names_for("telemetry")).to be_empty
harness_class.ensure_schema_upgrades
telemetry_columns = column_names_for("telemetry")
expect(telemetry_columns).to include("soil_temperature", "lux", "iaq")
expect(telemetry_columns).to include("rx_time", "battery_level")
end
end