From e502ddd436ca91d7032e6e7bc03fee72d2152bd8 Mon Sep 17 00:00:00 2001 From: l5y <220195275+l5yth@users.noreply.github.com> Date: Fri, 14 Nov 2025 21:18:37 +0100 Subject: [PATCH] fix telemetry parsing for charts (#451) --- .../js/app/__tests__/node-details.test.js | 30 +++- .../node-snapshot-normalizer.test.js | 73 ++++++++ web/public/assets/js/app/main.js | 2 + web/public/assets/js/app/node-details.js | 9 +- web/public/assets/js/app/node-page.js | 7 +- .../assets/js/app/node-snapshot-normalizer.js | 168 ++++++++++++++++++ 6 files changed, 280 insertions(+), 9 deletions(-) create mode 100644 web/public/assets/js/app/__tests__/node-snapshot-normalizer.test.js create mode 100644 web/public/assets/js/app/node-snapshot-normalizer.js diff --git a/web/public/assets/js/app/__tests__/node-details.test.js b/web/public/assets/js/app/__tests__/node-details.test.js index 828bcdc..52842e2 100644 --- a/web/public/assets/js/app/__tests__/node-details.test.js +++ b/web/public/assets/js/app/__tests__/node-details.test.js @@ -53,7 +53,7 @@ test('refreshNodeInformation merges telemetry metrics when the base node lacks t modem_preset: 'MediumFast', lora_freq: '868.1', })], - ['/api/telemetry/!test?limit=7', createResponse(200, [{ + ['/api/telemetry/!test?limit=1000', createResponse(200, [{ node_id: '!test', battery_level: 73.5, rx_time: 1_200, @@ -113,13 +113,37 @@ test('refreshNodeInformation merges telemetry metrics when the base node lacks t }); }); +test('refreshNodeInformation normalizes telemetry aliases for downstream consumers', async () => { + const responses = new Map([ + ['/api/nodes/!chan?limit=7', createResponse(404, { error: 'not found' })], + ['/api/telemetry/!chan?limit=1000', createResponse(200, [{ + node_id: '!chan', + channel: '76.5', + air_util_tx: '12.25', + }])], + ['/api/positions/!chan?limit=7', createResponse(404, { error: 'not found' })], + ['/api/neighbors/!chan?limit=1000', createResponse(404, { error: 'not found' })], + ]); + + const fetchImpl = async url => responses.get(url) ?? createResponse(404, { error: 'not found' }); + const node = await refreshNodeInformation('!chan', { fetchImpl }); + + assert.equal(node.nodeId, '!chan'); + assert.equal(node.channel_utilization, 76.5); + assert.equal(node.channelUtilization, 76.5); + assert.equal(node.channel, 76.5); + assert.equal(node.air_util_tx, 12.25); + assert.equal(node.airUtil, 12.25); + assert.equal(node.airUtilTx, 12.25); +}); + test('refreshNodeInformation preserves fallback metrics when telemetry is unavailable', async () => { const responses = new Map([ ['/api/nodes/42?limit=7', createResponse(200, { node_id: '!num', short_name: 'NUM', })], - ['/api/telemetry/42?limit=7', createResponse(404, { error: 'not found' })], + ['/api/telemetry/42?limit=1000', createResponse(404, { error: 'not found' })], ['/api/positions/42?limit=7', createResponse(404, { error: 'not found' })], ['/api/neighbors/42?limit=1000', createResponse(404, { error: 'not found' })], ]); @@ -148,7 +172,7 @@ test('refreshNodeInformation requires a node identifier', async () => { test('refreshNodeInformation handles missing node records by falling back to telemetry data', async () => { const responses = new Map([ ['/api/nodes/!missing?limit=7', createResponse(404, { error: 'not found' })], - ['/api/telemetry/!missing?limit=7', createResponse(200, [{ + ['/api/telemetry/!missing?limit=1000', createResponse(200, [{ node_id: '!missing', node_num: 77, battery_level: 66, diff --git a/web/public/assets/js/app/__tests__/node-snapshot-normalizer.test.js b/web/public/assets/js/app/__tests__/node-snapshot-normalizer.test.js new file mode 100644 index 0000000..840f8b9 --- /dev/null +++ b/web/public/assets/js/app/__tests__/node-snapshot-normalizer.test.js @@ -0,0 +1,73 @@ +/* + * Copyright © 2025-26 l5yth & contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { normalizeNodeSnapshot, normalizeNodeCollection, __testUtils } from '../node-snapshot-normalizer.js'; + +const { normalizeNumber, normalizeString } = __testUtils; + +test('normalizeNodeSnapshot synchronises telemetry aliases', () => { + const node = { + node_id: '!test', + channel: '56.2', + airUtil: 13.5, + battery_level: 45.5, + relativeHumidity: 24.3, + lastHeard: '1234', + }; + + const normalised = normalizeNodeSnapshot(node); + + assert.equal(normalised.nodeId, '!test'); + assert.equal(normalised.channel_utilization, 56.2); + assert.equal(normalised.channelUtilization, 56.2); + assert.equal(normalised.channel, 56.2); + assert.equal(normalised.air_util_tx, 13.5); + assert.equal(normalised.airUtilTx, 13.5); + assert.equal(normalised.airUtil, 13.5); + assert.equal(normalised.battery, 45.5); + assert.equal(normalised.batteryLevel, 45.5); + assert.equal(normalised.relative_humidity, 24.3); + assert.equal(normalised.humidity, 24.3); + assert.equal(normalised.last_heard, 1234); +}); + +test('normalizeNodeCollection applies canonical forms to all nodes', () => { + const nodes = [ + { short_name: ' AAA ', voltage: '3.7' }, + { shortName: 'BBB', uptime_seconds: '3600', airUtilTx: '5.5' }, + ]; + + normalizeNodeCollection(nodes); + + assert.equal(nodes[0].shortName, 'AAA'); + assert.equal(nodes[0].short_name, 'AAA'); + assert.equal(nodes[0].voltage, 3.7); + assert.equal(nodes[1].uptime, 3600); + assert.equal(nodes[1].air_util_tx, 5.5); +}); + +test('normaliser helpers coerce primitive values consistently', () => { + assert.equal(normalizeNumber('42.1'), 42.1); + assert.equal(normalizeNumber('not-a-number'), null); + assert.equal(normalizeNumber(Infinity), null); + + assert.equal(normalizeString(' hello '), 'hello'); + assert.equal(normalizeString(''), null); + assert.equal(normalizeString(null), null); +}); diff --git a/web/public/assets/js/app/main.js b/web/public/assets/js/app/main.js index d9b23e8..c9920d3 100644 --- a/web/public/assets/js/app/main.js +++ b/web/public/assets/js/app/main.js @@ -54,6 +54,7 @@ import { aggregatePositionSnapshots, aggregateTelemetrySnapshots, } from './snapshot-aggregator.js'; +import { normalizeNodeCollection } from './node-snapshot-normalizer.js'; /** * Entry point for the interactive dashboard. Wires up event listeners, @@ -3859,6 +3860,7 @@ let messagesById = new Map(); mergePositionsIntoNodes(aggregatedNodes, aggregatedPositions); computeDistances(aggregatedNodes); mergeTelemetryIntoNodes(aggregatedNodes, aggregatedTelemetry); + normalizeNodeCollection(aggregatedNodes); allNodes = aggregatedNodes; rebuildNodeIndex(allNodes); const [chatMessages, encryptedChatMessages] = await Promise.all([ diff --git a/web/public/assets/js/app/node-details.js b/web/public/assets/js/app/node-details.js index ca61690..cea77f8 100644 --- a/web/public/assets/js/app/node-details.js +++ b/web/public/assets/js/app/node-details.js @@ -15,6 +15,7 @@ */ import { extractModemMetadata } from './node-modem-metadata.js'; +import { normalizeNodeSnapshot } from './node-snapshot-normalizer.js'; import { SNAPSHOT_WINDOW, aggregateNeighborSnapshots, @@ -24,7 +25,7 @@ import { } from './snapshot-aggregator.js'; const DEFAULT_FETCH_OPTIONS = Object.freeze({ cache: 'no-store' }); -const TELEMETRY_LIMIT = SNAPSHOT_WINDOW; +const TELEMETRY_LIMIT = 1000; const POSITION_LIMIT = SNAPSHOT_WINDOW; const NEIGHBOR_LIMIT = 1000; @@ -183,7 +184,7 @@ function mergeNodeFields(target, record) { assignNumber(target, 'battery', extractNumber(record, ['battery', 'battery_level', 'batteryLevel'])); assignNumber(target, 'voltage', extractNumber(record, ['voltage'])); assignNumber(target, 'uptime', extractNumber(record, ['uptime', 'uptime_seconds', 'uptimeSeconds'])); - assignNumber(target, 'channel', extractNumber(record, ['channel_utilization', 'channelUtilization'])); + assignNumber(target, 'channel', extractNumber(record, ['channel_utilization', 'channelUtilization', 'channel'])); assignNumber(target, 'airUtil', extractNumber(record, ['airUtil', 'air_util_tx', 'airUtilTx'])); assignNumber(target, 'temperature', extractNumber(record, ['temperature'])); assignNumber(target, 'humidity', extractNumber(record, ['humidity', 'relative_humidity', 'relativeHumidity'])); @@ -214,7 +215,7 @@ function mergeTelemetry(target, telemetry) { assignNumber(target, 'battery', extractNumber(telemetry, ['battery_level', 'batteryLevel']), { preferExisting: true }); assignNumber(target, 'voltage', extractNumber(telemetry, ['voltage']), { preferExisting: true }); assignNumber(target, 'uptime', extractNumber(telemetry, ['uptime_seconds', 'uptimeSeconds']), { preferExisting: true }); - assignNumber(target, 'channel', extractNumber(telemetry, ['channel_utilization', 'channelUtilization']), { preferExisting: true }); + assignNumber(target, 'channel', extractNumber(telemetry, ['channel_utilization', 'channelUtilization', 'channel']), { preferExisting: true }); assignNumber(target, 'airUtil', extractNumber(telemetry, ['air_util_tx', 'airUtilTx', 'airUtil']), { preferExisting: true }); assignNumber(target, 'temperature', extractNumber(telemetry, ['temperature']), { preferExisting: true }); assignNumber(target, 'humidity', extractNumber(telemetry, ['relative_humidity', 'relativeHumidity', 'humidity']), { preferExisting: true }); @@ -453,6 +454,8 @@ export async function refreshNodeInformation(reference, options = {}) { neighbors: neighborEntries, }; + normalizeNodeSnapshot(node); + return node; } diff --git a/web/public/assets/js/app/node-page.js b/web/public/assets/js/app/node-page.js index 0f91feb..2f5b6e7 100644 --- a/web/public/assets/js/app/node-page.js +++ b/web/public/assets/js/app/node-page.js @@ -1102,12 +1102,13 @@ function renderTelemetryChart(spec, entries, nowMs) { */ function renderTelemetryCharts(node, { nowMs = Date.now() } = {}) { const telemetrySource = node?.rawSources?.telemetry; - const snapshotFallback = Array.isArray(node?.rawSources?.telemetrySnapshots) + const snapshotHistory = Array.isArray(node?.rawSources?.telemetrySnapshots) && node.rawSources.telemetrySnapshots.length > 0 ? node.rawSources.telemetrySnapshots : null; - const rawSnapshots = Array.isArray(telemetrySource?.snapshots) + const aggregatedSnapshots = Array.isArray(telemetrySource?.snapshots) ? telemetrySource.snapshots - : snapshotFallback; + : null; + const rawSnapshots = snapshotHistory ?? aggregatedSnapshots; if (!Array.isArray(rawSnapshots) || rawSnapshots.length === 0) { return ''; } diff --git a/web/public/assets/js/app/node-snapshot-normalizer.js b/web/public/assets/js/app/node-snapshot-normalizer.js new file mode 100644 index 0000000..3e3bed6 --- /dev/null +++ b/web/public/assets/js/app/node-snapshot-normalizer.js @@ -0,0 +1,168 @@ +/* + * Copyright © 2025-26 l5yth & contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Determine whether the supplied value acts like an object instance. + * + * @param {*} value Candidate reference. + * @returns {boolean} True when the value is non-null and of type ``object``. + */ +function isObject(value) { + return value != null && typeof value === 'object'; +} + +/** + * Convert a raw value into a trimmed string when possible. + * + * @param {*} value Candidate value. + * @returns {string|null} Trimmed string or ``null`` when blank. + */ +function normalizeString(value) { + if (value == null) return null; + const str = String(value).trim(); + return str.length === 0 ? null : str; +} + +/** + * Convert a raw value into a finite number when possible. + * + * @param {*} value Candidate numeric value. + * @returns {number|null} Finite number or ``null`` when coercion fails. + */ +function normalizeNumber(value) { + if (typeof value === 'number') { + return Number.isFinite(value) ? value : null; + } + if (value == null || value === '') return null; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +/** + * Field alias metadata describing how canonical keys map to alternate names. + * + * @type {Array<{keys: Array, normalise?: (value: *) => *}>} + */ +const FIELD_ALIASES = Object.freeze([ + { keys: ['node_id', 'nodeId'], normalise: normalizeString }, + { keys: ['node_num', 'nodeNum', 'num'], normalise: normalizeNumber }, + { keys: ['short_name', 'shortName'], normalise: normalizeString }, + { keys: ['long_name', 'longName'], normalise: normalizeString }, + { keys: ['role'], normalise: normalizeString }, + { keys: ['hw_model', 'hwModel'], normalise: normalizeString }, + { keys: ['modem_preset', 'modemPreset'], normalise: normalizeString }, + { keys: ['lora_freq', 'loraFreq'], normalise: normalizeNumber }, + { keys: ['battery_level', 'battery', 'batteryLevel'], normalise: normalizeNumber }, + { keys: ['voltage'], normalise: normalizeNumber }, + { keys: ['uptime_seconds', 'uptime', 'uptimeSeconds'], normalise: normalizeNumber }, + { keys: ['channel_utilization', 'channelUtilization', 'channel'], normalise: normalizeNumber }, + { keys: ['air_util_tx', 'airUtilTx', 'airUtil'], normalise: normalizeNumber }, + { keys: ['temperature'], normalise: normalizeNumber }, + { keys: ['relative_humidity', 'relativeHumidity', 'humidity'], normalise: normalizeNumber }, + { keys: ['barometric_pressure', 'barometricPressure', 'pressure'], normalise: normalizeNumber }, + { keys: ['gas_resistance', 'gasResistance'], normalise: normalizeNumber }, + { keys: ['snr'], normalise: normalizeNumber }, + { keys: ['last_heard', 'lastHeard'], normalise: normalizeNumber }, + { keys: ['last_seen_iso', 'lastSeenIso'], normalise: normalizeString }, + { keys: ['telemetry_time', 'telemetryTime'], normalise: normalizeNumber }, + { keys: ['position_time', 'positionTime'], normalise: normalizeNumber }, + { keys: ['position_time_iso', 'positionTimeIso'], normalise: normalizeString }, + { keys: ['latitude', 'lat'], normalise: normalizeNumber }, + { keys: ['longitude', 'lon'], normalise: normalizeNumber }, + { keys: ['altitude', 'alt'], normalise: normalizeNumber }, + { keys: ['distance_km', 'distanceKm'], normalise: normalizeNumber }, + { keys: ['precision_bits', 'precisionBits'], normalise: normalizeNumber }, +]); + +/** + * Resolve the first usable value amongst the provided alias keys. + * + * @param {Object} node Node snapshot inspected for values. + * @param {{keys: Array, normalise?: Function}} config Alias metadata. + * @returns {*|null} Normalized value or ``null``. + */ +function resolveAliasValue(node, config) { + if (!isObject(node)) return null; + for (const key of config.keys) { + if (!Object.prototype.hasOwnProperty.call(node, key)) continue; + const raw = node[key]; + const value = typeof config.normalise === 'function' + ? config.normalise(raw) + : raw; + if (value != null) { + return value; + } + } + return null; +} + +/** + * Populate alias keys with the supplied value. + * + * @param {Object} node Node snapshot mutated in-place. + * @param {{keys: Array}} config Alias metadata. + * @param {*} value Canonical value assigned to all aliases. + * @returns {void} + */ +function assignAliasValue(node, config, value) { + for (const key of config.keys) { + node[key] = value; + } +} + +/** + * Normalise a node snapshot to ensure canonical telemetry and identity fields + * exist under all supported aliases. + * + * @param {*} node Candidate node snapshot. + * @returns {*} Normalised node snapshot. + */ +export function normalizeNodeSnapshot(node) { + if (!isObject(node)) { + return node; + } + for (const aliasConfig of FIELD_ALIASES) { + const value = resolveAliasValue(node, aliasConfig); + if (value == null) continue; + assignAliasValue(node, aliasConfig, value); + } + return node; +} + +/** + * Apply {@link normalizeNodeSnapshot} to each node in the provided collection. + * + * @param {Array<*>} nodes Node collection. + * @returns {Array<*>} Normalised node collection. + */ +export function normalizeNodeCollection(nodes) { + if (!Array.isArray(nodes)) { + return nodes; + } + nodes.forEach(node => { + normalizeNodeSnapshot(node); + }); + return nodes; +} + +export const __testUtils = { + isObject, + normalizeString, + normalizeNumber, + FIELD_ALIASES, + resolveAliasValue, + assignAliasValue, +};