From c52ae53cc696bddf894cf896d83199226130f5c6 Mon Sep 17 00:00:00 2001 From: TJ Downes <273720+tjdownes@users.noreply.github.com> Date: Tue, 21 Apr 2026 19:52:44 -0700 Subject: [PATCH] perf(advert): replace list with deque for _recent_drops; use islice for _known_neighbors cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem 1 — _recent_drops: the list was evicted with pop(0), which is an O(n) memmove every time a drop is recorded. With maxlen=20 this is negligible today, but pop(0) on a list is always O(n) and the pattern is worth eliminating. Problem 2 — _known_neighbors cap: the eviction path did set(list(self._known_neighbors)[500:]) which first materialises the entire set as a list (O(n) allocation) before slicing. itertools.islice works directly on the set iterator and only allocates the 500 kept items, halving peak memory pressure during cleanup. Changes: * Import itertools (already absent from this file) * Import deque from collections alongside OrderedDict * self._recent_drops initialised as deque(maxlen=20); self._max_recent_drops removed (maxlen is the single source of truth) * Drop-recording block: rebuild deque from generator (preserves pubkey dedup filter) then append — automatic eviction replaces the explicit pop(0) guard * Known-neighbors cap: itertools.islice(self._known_neighbors, 500) replaces list(self._known_neighbors)[500:] Co-Authored-By: Claude Sonnet 4.6 --- repeater/handler_helpers/advert.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/repeater/handler_helpers/advert.py b/repeater/handler_helpers/advert.py index c31cbe2..e95f5e0 100644 --- a/repeater/handler_helpers/advert.py +++ b/repeater/handler_helpers/advert.py @@ -8,7 +8,8 @@ Includes adaptive rate limiting based on mesh activity. import asyncio import logging import time -from collections import OrderedDict +import itertools +from collections import OrderedDict, deque from enum import Enum from typing import Dict, Optional, Tuple @@ -123,9 +124,9 @@ class AdvertHelper: self._stats_advert_duplicates = 0 self._stats_tier_changes = 0 - # Recent drops tracking (keep last 20) - self._recent_drops = [] - self._max_recent_drops = 20 + # Recent drops tracking — bounded deque so append is O(1) and the + # oldest entry is evicted automatically (no pop(0) O(n) shift needed). + self._recent_drops: deque = deque(maxlen=20) # Memory management self._last_cleanup = time.time() @@ -194,8 +195,8 @@ class AdvertHelper: # 5. Limit known neighbors set to prevent unbounded growth if len(self._known_neighbors) > 1000: - # Clear the oldest half (simple approach - could be more sophisticated) - self._known_neighbors = set(list(self._known_neighbors)[500:]) + # itertools.islice avoids materialising the full list first (O(n) → O(k)) + self._known_neighbors = set(itertools.islice(self._known_neighbors, 500)) if expired_penalties or inactive_pubkeys: logger.debug( @@ -571,10 +572,13 @@ class AdvertHelper: # Track recent drop (deduplicate by pubkey) pubkey_short = pubkey[:16] - # Remove any existing entry for this pubkey - self._recent_drops = [d for d in self._recent_drops if d["pubkey"] != pubkey_short] - - # Add the new drop entry + # Remove any existing entry for this pubkey, then append the + # updated record. Rebuilding as a deque preserves maxlen so + # the oldest entry is evicted automatically — no pop(0) needed. + self._recent_drops = deque( + (d for d in self._recent_drops if d["pubkey"] != pubkey_short), + maxlen=20, + ) self._recent_drops.append({ "pubkey": pubkey_short, "name": node_name, @@ -582,10 +586,6 @@ class AdvertHelper: "timestamp": now }) - # Keep only last N drops - if len(self._recent_drops) > self._max_recent_drops: - self._recent_drops.pop(0) - return # Skip our own adverts