From caaecfb3c295adb1c3ef30d23f7425229f1c8d59 Mon Sep 17 00:00:00 2001 From: Louis King Date: Tue, 23 Jun 2026 08:40:01 +0100 Subject: [PATCH] feat(spam): retune default scoring config and add v0.15 upgrade notes Adjust the default spam-scoring knobs across Python settings, SpamConfig, docker-compose, .env.example and docs to reduce false positives on chatty legitimate users: SPAM_MIN_PATH_HOPS 5 -> 3 SPAM_PATH_THRESHOLD 5 -> 6 SPAM_NAME_THRESHOLD 5 -> 10 SPAM_WEIGHT_PATH 0.7 -> 0.75 SPAM_WEIGHT_NAME 0.3 -> 0.25 SPAM_SCORE_THRESHOLD 0.6 -> 0.65 Also document the spam-detection feature and the pull_policy change in a new v0.15.0 section of docs/upgrading.md. Co-Authored-By: Claude Opus 4.8 --- .env.example | 12 +++--- docker-compose.yml | 14 +++---- docs/configuration.md | 12 +++--- docs/upgrading.md | 55 +++++++++++++++++++++++++ src/meshcore_hub/api/app.py | 2 +- src/meshcore_hub/api/routes/messages.py | 2 +- src/meshcore_hub/collector/spam.py | 12 +++--- src/meshcore_hub/common/config.py | 12 +++--- 8 files changed, 88 insertions(+), 33 deletions(-) diff --git a/.env.example b/.env.example index 06b2c27..2c21388 100644 --- a/.env.example +++ b/.env.example @@ -298,16 +298,16 @@ DATA_RETENTION_INTERVAL_HOURS=24 # Score at/above which a message is treated as likely spam (hidden by default in # the API, logged at WARNING by the collector). Read by the collector + api. -# SPAM_SCORE_THRESHOLD=0.6 +# SPAM_SCORE_THRESHOLD=0.65 # Scoring tuning (collector only; only consulted when detection is enabled). # SPAM_WINDOW_SECONDS=300 # sliding window for frequency counts # SPAM_PATH_HOPS=3 # leading origin-side hops that form the prefix -# SPAM_MIN_PATH_HOPS=5 # min path_len before the path signal applies -# SPAM_PATH_THRESHOLD=5 # joint path+sender count that saturates the path signal -# SPAM_NAME_THRESHOLD=5 # sender count that saturates the name signal -# SPAM_WEIGHT_PATH=0.7 # weight of the path signal -# SPAM_WEIGHT_NAME=0.3 # weight of the name signal +# SPAM_MIN_PATH_HOPS=3 # min path_len before the path signal applies +# SPAM_PATH_THRESHOLD=6 # joint path+sender count that saturates the path signal +# SPAM_NAME_THRESHOLD=10 # sender count that saturates the name signal +# SPAM_WEIGHT_PATH=0.75 # weight of the path signal +# SPAM_WEIGHT_NAME=0.25 # weight of the name signal # SPAM_RESCORE_INTERVAL_SECONDS=120 # background re-scoring sweep cadence (0 disables) # ------------------- diff --git a/docker-compose.yml b/docker-compose.yml index 4ad4cd7..004391e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -242,14 +242,14 @@ services: # Spam detection (derived from FEATURE_SPAM_DETECTION so one var drives the # backend switch and the web toggle together). Off by default. - SPAM_DETECTION_ENABLED=${FEATURE_SPAM_DETECTION:-false} - - SPAM_SCORE_THRESHOLD=${SPAM_SCORE_THRESHOLD:-0.6} + - SPAM_SCORE_THRESHOLD=${SPAM_SCORE_THRESHOLD:-0.65} - SPAM_WINDOW_SECONDS=${SPAM_WINDOW_SECONDS:-300} - SPAM_PATH_HOPS=${SPAM_PATH_HOPS:-3} - - SPAM_MIN_PATH_HOPS=${SPAM_MIN_PATH_HOPS:-5} - - SPAM_PATH_THRESHOLD=${SPAM_PATH_THRESHOLD:-5} - - SPAM_NAME_THRESHOLD=${SPAM_NAME_THRESHOLD:-5} - - SPAM_WEIGHT_PATH=${SPAM_WEIGHT_PATH:-0.7} - - SPAM_WEIGHT_NAME=${SPAM_WEIGHT_NAME:-0.3} + - SPAM_MIN_PATH_HOPS=${SPAM_MIN_PATH_HOPS:-3} + - SPAM_PATH_THRESHOLD=${SPAM_PATH_THRESHOLD:-6} + - SPAM_NAME_THRESHOLD=${SPAM_NAME_THRESHOLD:-10} + - SPAM_WEIGHT_PATH=${SPAM_WEIGHT_PATH:-0.75} + - SPAM_WEIGHT_NAME=${SPAM_WEIGHT_NAME:-0.25} - SPAM_RESCORE_INTERVAL_SECONDS=${SPAM_RESCORE_INTERVAL_SECONDS:-120} command: ["collector"] healthcheck: @@ -317,7 +317,7 @@ services: # Spam detection: hide-filter switch + threshold (bridged from # FEATURE_SPAM_DETECTION so it tracks the collector and web toggle). - SPAM_DETECTION_ENABLED=${FEATURE_SPAM_DETECTION:-false} - - SPAM_SCORE_THRESHOLD=${SPAM_SCORE_THRESHOLD:-0.6} + - SPAM_SCORE_THRESHOLD=${SPAM_SCORE_THRESHOLD:-0.65} command: ["api"] healthcheck: test: diff --git a/docs/configuration.md b/docs/configuration.md index 6f11e79..f02faab 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -135,14 +135,14 @@ Off by default. `FEATURE_SPAM_DETECTION` is the single switch operators set: in | Variable | Default | Read by | Description | | --- | --- | --- | --- | | `SPAM_DETECTION_ENABLED` | `false` | collector, api | Operational switch for scoring + hiding. In Compose, derived from `FEATURE_SPAM_DETECTION` (`SPAM_DETECTION_ENABLED=${FEATURE_SPAM_DETECTION}`) | -| `SPAM_SCORE_THRESHOLD` | `0.6` | collector, api | Score at/above which a message is treated as likely spam (hidden by default; logged at `WARNING`) | +| `SPAM_SCORE_THRESHOLD` | `0.65` | collector, api | Score at/above which a message is treated as likely spam (hidden by default; logged at `WARNING`) | | `SPAM_WINDOW_SECONDS` | `300` | collector | Sliding window (seconds) for frequency counts | | `SPAM_PATH_HOPS` | `3` | collector | Leading origin-side hops that form the `path_prefix` | -| `SPAM_MIN_PATH_HOPS` | `5` | collector | Minimum `path_len` before the path signal applies (short local-mesh paths share prefixes) | -| `SPAM_PATH_THRESHOLD` | `5` | collector | Joint `(path_prefix, sender)` count that saturates the path signal | -| `SPAM_NAME_THRESHOLD` | `5` | collector | Sender count that saturates the name signal | -| `SPAM_WEIGHT_PATH` | `0.7` | collector | Weight of the path signal in the combined score | -| `SPAM_WEIGHT_NAME` | `0.3` | collector | Weight of the name signal in the combined score | +| `SPAM_MIN_PATH_HOPS` | `3` | collector | Minimum `path_len` before the path signal applies (short local-mesh paths share prefixes) | +| `SPAM_PATH_THRESHOLD` | `6` | collector | Joint `(path_prefix, sender)` count that saturates the path signal | +| `SPAM_NAME_THRESHOLD` | `10` | collector | Sender count that saturates the name signal | +| `SPAM_WEIGHT_PATH` | `0.75` | collector | Weight of the path signal in the combined score | +| `SPAM_WEIGHT_NAME` | `0.25` | collector | Weight of the name signal in the combined score | | `SPAM_RESCORE_INTERVAL_SECONDS` | `120` | collector | Background re-scoring sweep cadence in seconds (`0` disables the sweep) | ## API diff --git a/docs/upgrading.md b/docs/upgrading.md index ee40d31..b280327 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -2,6 +2,61 @@ This guide covers upgrading from a previous MeshCore Hub release to the current version. Check the relevant version section below before upgrading. +## v0.15.0 + +### Spam Detection (score, hide, and toggle likely-spam messages) + +Each message is now scored for spam likelihood **at ingest** and the score is stored on the row. Likely-spam messages are **hidden by default** on the Messages page, with a "show potential spam" toggle to reveal them. Nothing is ever dropped — scoring is purely additive and the display layer filters on the stored score, so the feature is fully reversible. Scoring runs on both SQLite and PostgreSQL. + +The scorer combines two windowed signals over a sliding time window: a **path signal** (joint count of the same origin-side path prefix + normalised sender) and a **name signal** (count of the same normalised sender, after stripping a trailing digit/space suffix so rotating `bob1`/`bob2`/`Bob 3` collapse to `bob`). When the path is too short to be useful — including the **zero-hop case** where an observer sits right next to the sender — the name signal stands on its own at full weight, so local/zero-hop spam can still be flagged. A background sweep re-scores recent rows with hindsight (a symmetric window) so the leading edge of a burst is caught once its peers arrive. + +**Off by default. No action required to upgrade** — leave `FEATURE_SPAM_DETECTION` unset and messages behave exactly as before (the new columns stay null and nothing is hidden). + +**Database migration required:** + +``` +meshcore-hub db upgrade +``` + +This adds three nullable columns to the `messages` table — `path_prefix`, `sender_normalized`, `spam_score` — plus two composite indexes (`ix_messages_path_prefix_received_at`, `ix_messages_sender_normalized_received_at`). The migration is batch-mode and runs on both SQLite and Postgres. On Docker deployments it runs automatically on startup. **No backfill:** only messages ingested *after* enabling are scored; historical rows keep null scores and are never hidden. + +**New optional environment variables (all safe to omit):** + +| Variable | Default | Description | +| ------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `FEATURE_SPAM_DETECTION` | `false` | The single operator switch. Shows the "show potential spam" toggle on the Messages page. In Compose this **derives** the backend `SPAM_DETECTION_ENABLED` for the collector and api. | +| `SPAM_DETECTION_ENABLED` | `false` | Backend operational switch read by the collector (scoring + sweep) and api (hide-filter). In Compose derived from `FEATURE_SPAM_DETECTION`; set directly only when running without Compose. | +| `SPAM_SCORE_THRESHOLD` | `0.65` | Score at/above which a message is treated as likely spam — hidden by default in the API, logged at WARNING by the collector. Read by collector + api. | + +**Scoring tuning (collector only; consulted only when detection is enabled):** + +| Variable | Default | Description | +| ------------------------------- | ------- | ------------------------------------------------------------------ | +| `SPAM_WINDOW_SECONDS` | `300` | Sliding window for the frequency counts | +| `SPAM_PATH_HOPS` | `3` | Leading origin-side hops that form the path prefix | +| `SPAM_MIN_PATH_HOPS` | `3` | Minimum `path_len` before the path signal applies | +| `SPAM_PATH_THRESHOLD` | `6` | Joint path+sender count that saturates the path signal | +| `SPAM_NAME_THRESHOLD` | `10` | Sender count that saturates the name signal | +| `SPAM_WEIGHT_PATH` | `0.75` | Weight of the path signal | +| `SPAM_WEIGHT_NAME` | `0.25` | Weight of the name signal | +| `SPAM_RESCORE_INTERVAL_SECONDS` | `120` | Background re-scoring sweep cadence (`0` disables the sweep) | + +**Feature ↔ backend split:** the UI toggle is served by the `web` app while scoring runs in the `collector` and the hide-filter in the `api` — separate processes with separate settings. Docker Compose links them: setting `FEATURE_SPAM_DETECTION=true` enables scoring + sweep on the collector, the hide-filter on the api (both via `SPAM_DETECTION_ENABLED=${FEATURE_SPAM_DETECTION}`), and the toggle in the web UI. Operators running the processes directly can set `SPAM_DETECTION_ENABLED` independently. + +**API:** message endpoints gain a `spam_score` field and an `include_spam` query parameter. By default the API hides rows scoring at/above `SPAM_SCORE_THRESHOLD`; `include_spam=true` returns them (rows with a null score — i.e. ingested before the feature was on, or while it was off — are always shown). With the feature off the filter is a no-op. + +To enable, set `FEATURE_SPAM_DETECTION=true` in your `.env` and recreate the `collector`, `api`, and `web` services. All variables are passed through `docker-compose.yml` automatically. + +### Docker Compose `pull_policy` removed from base + +A `pull_policy: daily` that had been added to the five hub services (`collector`, `api`, `web`, `migrate`, `seed`) in the base `docker-compose.yml` has been removed. It caused `up` to pull the published image over a freshly built local image in development, clobbering local builds. + +- **Base** (`docker-compose.yml`) now sets **no** `pull_policy`, falling back to Compose's default `missing` (pull only when the image is absent). +- **Dev** (`docker-compose.dev.yml`) sets `pull_policy: build` on all five hub services, so `make build && make up` always (re)builds from local source and never pulls. +- **Prod** (`docker-compose.prod.yml`) is unchanged and inherits the default `missing` policy. + +**No action required** beyond pulling the updated compose files. **Production note:** image refreshes are no longer automatic on `up` — to move prod to a newer published image run `docker compose -f docker-compose.yml -f docker-compose.prod.yml pull` followed by `up -d`. + ## v0.14.0 ### Optional PostgreSQL Backend diff --git a/src/meshcore_hub/api/app.py b/src/meshcore_hub/api/app.py index c9f3eb5..7da423b 100644 --- a/src/meshcore_hub/api/app.py +++ b/src/meshcore_hub/api/app.py @@ -92,7 +92,7 @@ def create_app( redis_cache_ttl: int = 30, redis_cache_ttl_dashboard: int = 30, spam_detection_enabled: bool = False, - spam_score_threshold: float = 0.6, + spam_score_threshold: float = 0.65, ) -> FastAPI: """Create and configure the FastAPI application. diff --git a/src/meshcore_hub/api/routes/messages.py b/src/meshcore_hub/api/routes/messages.py index f3cc090..cd1ad76 100644 --- a/src/meshcore_hub/api/routes/messages.py +++ b/src/meshcore_hub/api/routes/messages.py @@ -105,7 +105,7 @@ def list_messages( # filter — every message is returned regardless of any stored score, so # toggling the feature off instantly un-hides rows scored while it was on. spam_enabled = getattr(request.app.state, "spam_detection_enabled", False) - spam_threshold = getattr(request.app.state, "spam_score_threshold", 0.6) + spam_threshold = getattr(request.app.state, "spam_score_threshold", 0.65) if spam_enabled and not include_spam: query = query.where( or_( diff --git a/src/meshcore_hub/collector/spam.py b/src/meshcore_hub/collector/spam.py index bb4fb5c..fd99f01 100644 --- a/src/meshcore_hub/collector/spam.py +++ b/src/meshcore_hub/collector/spam.py @@ -43,12 +43,12 @@ class SpamConfig: enabled: bool = False window_seconds: int = 300 path_hops: int = 3 - min_path_hops: int = 5 - path_threshold: int = 5 - name_threshold: int = 5 - weight_path: float = 0.7 - weight_name: float = 0.3 - score_threshold: float = 0.6 + min_path_hops: int = 3 + path_threshold: int = 6 + name_threshold: int = 10 + weight_path: float = 0.75 + weight_name: float = 0.25 + score_threshold: float = 0.65 rescore_interval_seconds: int = 120 @classmethod diff --git a/src/meshcore_hub/common/config.py b/src/meshcore_hub/common/config.py index a5d9544..6c08f39 100644 --- a/src/meshcore_hub/common/config.py +++ b/src/meshcore_hub/common/config.py @@ -165,7 +165,7 @@ class CommonSettings(BaseSettings): ), ) spam_score_threshold: float = Field( - default=0.6, + default=0.65, description=( "Score at/above which a message is treated as likely spam (hidden by " "default in the API; logged at the collector)" @@ -275,7 +275,7 @@ class CollectorSettings(CommonSettings): ge=1, ) spam_min_path_hops: int = Field( - default=5, + default=3, description=( "Minimum path_len before the path signal is applied; below this the " "path_prefix is stored null (short local-mesh paths share prefixes)" @@ -283,20 +283,20 @@ class CollectorSettings(CommonSettings): ge=0, ) spam_path_threshold: int = Field( - default=5, + default=6, description="Joint path+sender count that saturates the path signal", ge=1, ) spam_name_threshold: int = Field( - default=5, + default=10, description="Sender count that saturates the name signal", ge=1, ) spam_weight_path: float = Field( - default=0.7, description="Weight of the path signal in the combined score" + default=0.75, description="Weight of the path signal in the combined score" ) spam_weight_name: float = Field( - default=0.3, description="Weight of the name signal in the combined score" + default=0.25, description="Weight of the name signal in the combined score" ) spam_rescore_interval_seconds: int = Field( default=120,