fix(docker): preserve upgrade defaults and use catalogue plugins

.github/workflows/docker-publish.yml

@@ -11,10 +11,6 @@ on:
         description: "Docker image repository to publish to"
         required: false
         default: ""
-      openhop_console_version:
-        description: "openHop Console release to bundle (latest or a tag such as v0.9.333)"
-        required: false
-        default: "latest"
 
 jobs:
   docker:
@@ -75,28 +71,6 @@ jobs:
           echo "Using image repository: ${image_repository}"
           echo "image_repository=${image_repository}" >> "$GITHUB_OUTPUT"
 
-      - name: Resolve bundled openHop Console release
-        id: console_release
-        shell: bash
-        env:
-          REQUESTED_CONSOLE_VERSION: ${{ inputs.openhop_console_version || 'latest' }}
-        run: |
-          set -euo pipefail
-
-          if [ "${REQUESTED_CONSOLE_VERSION}" = "latest" ]; then
-            release_json="$(curl -fsSL https://api.github.com/repos/Treehouse-00/pymc_console-dist/releases/latest)"
-            tag="$(jq -r '.tag_name' <<<"${release_json}")"
-            published_at="$(jq -r '.published_at' <<<"${release_json}")"
-            cache_bust="${tag}@${published_at}"
-          else
-            tag="${REQUESTED_CONSOLE_VERSION}"
-            cache_bust="${REQUESTED_CONSOLE_VERSION}"
-          fi
-
-          echo "Bundling openHop Console ${tag}"
-          echo "version=${REQUESTED_CONSOLE_VERSION}" >> "$GITHUB_OUTPUT"
-          echo "cache_bust=${cache_bust}" >> "$GITHUB_OUTPUT"
-
       - name: Extract Docker metadata
         id: meta
         uses: docker/metadata-action@v5
@@ -117,8 +91,6 @@ jobs:
           push: true
           build-args: |
             PACKAGE_VERSION=${{ steps.package_version.outputs.version }}
-            OPENHOP_CONSOLE_VERSION=${{ steps.console_release.outputs.version }}
-            OPENHOP_CONSOLE_CACHE_BUST=${{ steps.console_release.outputs.cache_bust }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha

README.md

@@ -255,6 +255,11 @@ The web interface can upgrade an installation or switch branches.
 > Docker installs cannot be upgraded or branch-switched from the web interface.
 > Update the container image instead.
 
+Existing native installations must run the updated `sudo bash ./manage.sh upgrade`
+once as an administrator to replace the privileged OTA helper after the security
+update. Updating only the Python package or using the old web updater does not
+refresh that helper. See [native upgrade prerequisites](docs/plugins.md#native-upgrade-prerequisite).
+
 ![Upgrade](docs/webui-upgrade.png)
 
 ### CLI
@@ -451,22 +456,80 @@ that source as a directory, which breaks startup.
 1. Copy `.env.example` to `.env`.
 2. Review `.env` and update `OPENHOP_REPEATER_IMAGE`, `DIALOUT_GID`,
    `GPIO_GID`, or `SPI_GID` if needed.
-3. Configure `docker-compose.yml` for your hardware and device paths.
-4. Uncomment the USB device mapping only if your host has that device path.
+3. Configure `docker-compose.yml` for your hardware. Remove every device mapping
+   that is absent or unnecessary: the shipped file enables SPI, GPIO, and USB
+   bus examples. Network-radio installations do not need those mappings.
+4. Add a serial-modem mapping only after preparing its host device path.
 5. Pull and start the container.
 
 ```bash
 docker compose up -d
 ```
 
-The Docker image bundles the optional openHop Console frontend at
-`/opt/pymc_console/web/html`. Local builds with `docker-compose.build.yml`
-download the newest openHop Console release by default. Set `OPENHOP_CONSOLE_VERSION`
-in `.env` to pin a release tag such as `v0.9.329`.
+The Docker image includes the default RepeaterUI but no longer downloads or bundles
+openHop Console. Install optional Console functionality through the plugin catalogue.
 
-After the container starts, open Web Settings and choose `openHop Console` as the
-web frontend if you want the repeater to serve that UI. The existing
-`/api/check_pymc_console` endpoint should report that the Console path exists.
+Before upgrading an older image with `/opt/pymc_console/web/html` selected as its
+frontend, switch back to the default RepeaterUI. The old bundled directory is absent
+from new images. Existing plugin data remains in the persistent data volume.
+
+### Image updates and persistent plugins
+
+For a published image, select the intended image tag in `.env`, then run:
+
+```bash
+docker compose pull
+docker compose up -d
+```
+
+For a local source build, after updating the intended branch:
+
+```bash
+docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build
+```
+
+Pulling Git changes or restarting a container alone does not rebuild its image.
+The Dockerfile packages the frontend assets present in the build context; it does
+not fetch or build the default RepeaterUI. Frontend changes belong in
+[RepeaterUI](https://github.com/openhop-dev/openHop_RepeaterUI), using the matching
+UI branch (`feat-plugin-ui` for `feat-plugin-manager`). Stage the complete verified
+UI build for packaging; do not hand-edit generated JavaScript. Optional Console
+installation is managed through the plugin catalogue, independently of image builds.
+
+Back up both config and data volumes before upgrading. Do **not** use
+`docker compose down -v` if you want to retain configuration and plugins.
+The default data volume contains plugin releases, retained wheels, venvs,
+settings, logs, and manager state. A custom `plugins.root` outside that volume
+needs its own persistent writable mount.
+
+Plugin venvs can rebuild after a Python minor-version change. Keep retained wheels;
+plugins with external dependencies may need network access and compatible packages
+to rebuild. Older installations without retained wheels need plugin reinstallation.
+Changing CPU architecture or absolute storage paths is not covered by minor-version
+rebuild detection.
+
+The image runs Repeater and its plugin manager as an unprivileged user under a
+supervisor and `tini`. To run without the manager, set `plugins.enabled: false` in
+configuration, or explicitly add this to the Compose service:
+
+```yaml
+environment:
+  OPENHOP_PLUGIN_MANAGER: "0"
+```
+
+Putting that variable only in `.env` does not pass it through the shipped Compose
+file. Repeater remains available without the manager; plugin operations are then
+unavailable. Plugins are trusted applications, not sandboxed code.
+
+On startup, existing user configuration overrides defaults from the current image's
+bundled example; an older example in the config volume is preserved but is not the
+upgrade merge source. Keep config writable by the image user. The temporary merged
+config fallback is not persistent; use an absolute `storage.storage_dir` inside the
+mounted data directory.
+
+The volume variables accept the default named volumes or absolute bind paths. If
+you choose another named volume, also declare it under top-level `volumes:`; use
+`external: true` when intentionally attaching an existing volume.
 
 ### Example `docker-compose.yml`
 

docker-compose.build.yml

@@ -10,4 +10,3 @@ services:
         DIALOUT_GID: ${DIALOUT_GID:-20}
         GPIO_GID: ${GPIO_GID:-986}
         SPI_GID: ${SPI_GID:-989}
-        OPENHOP_CONSOLE_VERSION: ${OPENHOP_CONSOLE_VERSION:-${PYMC_CONSOLE_VERSION:-latest}}

docker-entrypoint.sh

@@ -57,8 +57,13 @@ use_runtime_merged_config() {
 
 merge_config_from_example() {
     config_path="$1"
+    # Image defaults evolve on upgrade; preserve the volume's example unchanged.
+    merge_example_path="${EXAMPLE_PATH}"
+    if [ -f "${BUNDLED_EXAMPLE_PATH}" ]; then
+        merge_example_path="${BUNDLED_EXAMPLE_PATH}"
+    fi
 
-    if [ ! -f "${config_path}" ] || [ ! -f "${EXAMPLE_PATH}" ]; then
+    if [ ! -f "${config_path}" ] || [ ! -f "${merge_example_path}" ]; then
         return 0
     fi
 
@@ -79,8 +84,8 @@ merge_config_from_example() {
     # Keep only the example's comments to avoid comment duplication across upgrades.
     "${YQ_CMD}" eval '... comments=""' "${config_path}" > "${stripped_user}" 2>/dev/null || cp "${config_path}" "${stripped_user}"
 
-    if ! "${YQ_CMD}" eval-all '. as $item ireduce ({}; . * $item)' "${EXAMPLE_PATH}" "${stripped_user}" > "${merged_config}" 2>/dev/null; then
-        echo "Failed to merge ${config_path} with ${EXAMPLE_PATH}; keeping the existing config." >&2
+    if ! "${YQ_CMD}" eval-all '. as $item ireduce ({}; . * $item)' "${merge_example_path}" "${stripped_user}" > "${merged_config}" 2>/dev/null; then
+        echo "Failed to merge ${config_path} with ${merge_example_path}; keeping the existing config." >&2
         cleanup_merge
         trap - EXIT HUP INT TERM
         return 0

dockerfile

@@ -10,14 +10,10 @@ ARG GPIO_GID=986
 ARG SPI_GID=989
 ARG TARGETARCH
 ARG YQ_VERSION=v4.40.5
-ARG OPENHOP_CONSOLE_REPO=Treehouse-00/pymc_console-dist
-ARG OPENHOP_CONSOLE_VERSION=latest
-ARG OPENHOP_CONSOLE_CACHE_BUST=default
 
 ENV INSTALL_DIR=/opt/openhop_repeater \
     CONFIG_DIR=/etc/openhop_repeater \
     DATA_DIR=/var/lib/openhop_repeater \
-    OPENHOP_CONSOLE_WEB_DIR=/opt/pymc_console/web/html \
     HOME_DIR=/home/${USER} \
     PATH=/home/${USER}/.local/bin:${PATH} \
     PYTHONUNBUFFERED=1 \
@@ -54,22 +50,6 @@ RUN arch="${TARGETARCH:-}" \
     && wget -qO /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${YQ_BINARY}" \
     && chmod +x /usr/local/bin/yq
 
-# Bundle the optional openHop Console frontend so the web UI can select it without
-# requiring a host bind mount. Use OPENHOP_CONSOLE_VERSION=latest to pull the
-# newest release at image build time, or pin a tag such as v0.9.329.
-RUN set -eux; \
-    echo "Bundling openHop Console cache key: ${OPENHOP_CONSOLE_CACHE_BUST}"; \
-    mkdir -p "${OPENHOP_CONSOLE_WEB_DIR}"; \
-    if [ "${OPENHOP_CONSOLE_VERSION}" = "latest" ]; then \
-        console_url="https://github.com/${OPENHOP_CONSOLE_REPO}/releases/latest/download/pymc-ui-latest.tar.gz"; \
-    else \
-        console_url="https://github.com/${OPENHOP_CONSOLE_REPO}/releases/download/${OPENHOP_CONSOLE_VERSION}/pymc-ui-${OPENHOP_CONSOLE_VERSION}.tar.gz"; \
-    fi; \
-    wget -qO /tmp/pymc-console.tar.gz "${console_url}"; \
-    tar -xzf /tmp/pymc-console.tar.gz -C "${OPENHOP_CONSOLE_WEB_DIR}"; \
-    rm /tmp/pymc-console.tar.gz; \
-    test -f "${OPENHOP_CONSOLE_WEB_DIR}/index.html"
-
 # Create the group and user in order to run without root privileges
 RUN groupadd --gid "$PGID" "$GROUP" \
     && (getent group dialout >/dev/null || groupadd --gid "$DIALOUT_GID" dialout) \
@@ -80,7 +60,7 @@ RUN groupadd --gid "$PGID" "$GROUP" \
 
 # Create runtime directories
 RUN mkdir -p ${INSTALL_DIR} ${CONFIG_DIR} ${DATA_DIR} \
-    && chown -R "$USER":"$GROUP" ${INSTALL_DIR} ${CONFIG_DIR} ${DATA_DIR} ${HOME_DIR} /opt/pymc_console
+    && chown -R "$USER":"$GROUP" ${INSTALL_DIR} ${CONFIG_DIR} ${DATA_DIR} ${HOME_DIR}
 
 WORKDIR ${INSTALL_DIR}
 

env.example

@@ -29,9 +29,3 @@ SPI_GID=989
 # image yourself instead of pulling OPENHOP_REPEATER_IMAGE.
 PUID=15888
 PGID=15888
-
-# Local build only. The Docker image bundles openHop Console under
-# /opt/pymc_console/web/html so it is available in Web Settings without a host
-# bind mount. Use "latest" to pull the newest openHop Console release at build
-# time, or pin a tag such as v0.9.329.
-OPENHOP_CONSOLE_VERSION=latest

scripts/docker-plugin-smoke.sh

@@ -14,7 +14,7 @@ cleanup() {
 trap cleanup EXIT HUP INT TERM
 
 start_container() {
-    docker run -d \
+    docker run -d --network none \
         --name "${CONTAINER}" \
         -v "${CONFIG_VOLUME}:/etc/openhop_repeater" \
         -v "${DATA_VOLUME}:/var/lib/openhop_repeater" \
@@ -92,10 +92,14 @@ def stop(signum, frame):
 def main():
     signal.signal(signal.SIGTERM, stop)
     signal.signal(signal.SIGINT, stop)
+    child = os.fork()
     with lifecycle.open("a", encoding="utf-8") as handle:
-        handle.write(f"START {os.getpid()}\\n")
+        kind = "START" if child else "CHILD"
+        handle.write(f"{kind} {os.getpid()}\\n")
     while running:
         time.sleep(0.1)
+    if child:
+        os.waitpid(child, 0)
 '''
 metadata = "Metadata-Version: 2.1\nName: openhop-smoke-plugin\nVersion: 0.1.0\n"
 wheel = "Wheel-Version: 1.0\nGenerator: openhop-container-smoke\nRoot-Is-Purelib: true\nTag: py3-none-any\n"
@@ -228,6 +232,47 @@ assert status["enabled"] is True
 assert status["state"] == "RUNNING"
 '
 
+echo "Checking plugin stop/start, descendants, permissions and venv rebuild"
+docker exec -i "${CONTAINER}" python3 - <<'PY'
+import os
+import re
+import stat
+import sys
+import time
+from pathlib import Path
+from repeater.plugins.ipc import PluginIPCClient
+client = PluginIPCClient("/var/lib/openhop_repeater/plugin-manager.sock")
+root = Path("/var/lib/openhop_repeater/plugins/openhop.smoke")
+log = root / "data/lifecycle.log"
+for _ in range(50):
+    lines = log.read_text().splitlines()
+    children = [int(line.split()[1]) for line in lines if line.startswith("CHILD ")]
+    if children:
+        break
+    time.sleep(0.1)
+assert children
+pid = client.status("openhop.smoke")["pid"]
+child = children[-1]
+assert Path(f"/proc/{child}").exists()
+client.call("stop", plugin_id="openhop.smoke")
+for _ in range(50):
+    if not any(Path(f"/proc/{p}").exists() for p in (pid, child)):
+        break
+    time.sleep(0.1)
+assert not any(Path(f"/proc/{p}").exists() for p in (pid, child))
+assert f"TERM {pid}" in log.read_text().splitlines()
+assert f"TERM {child}" in log.read_text().splitlines()
+assert os.getuid() == 15888
+assert root.stat().st_uid == 15888
+assert stat.S_IMODE(Path(client.socket_path).stat().st_mode) == 0o660
+venv_config = root / "releases/0.1.0/venv/pyvenv.cfg"
+venv_config.write_text(re.sub(r"(?m)^version = .*", "version = 3.11.0", venv_config.read_text()))
+client.call("start", plugin_id="openhop.smoke")
+assert client.status("openhop.smoke")["state"] == "RUNNING"
+assert f"version = {sys.version_info.major}.{sys.version_info.minor}." in venv_config.read_text()
+print("stop/start descendants, non-root/socket permissions, real offline venv rebuild passed")
+PY
+
 echo "Checking graceful plugin shutdown"
 EXPECTED_PLUGIN_PID="$(docker exec "${CONTAINER}" python3 -c '
 from repeater.plugins.ipc import PluginIPCClient
@@ -235,7 +280,7 @@ client = PluginIPCClient("/var/lib/openhop_repeater/plugin-manager.sock")
 print(client.status("openhop.smoke")["pid"])
 ')"
 docker stop -t 10 "${CONTAINER}" >/dev/null
-docker run --rm --entrypoint python3 -e EXPECTED_PLUGIN_PID="${EXPECTED_PLUGIN_PID}" \
+docker run --rm --network none --entrypoint python3 -e EXPECTED_PLUGIN_PID="${EXPECTED_PLUGIN_PID}" \
     -v "${DATA_VOLUME}:/var/lib/openhop_repeater" \
     "${IMAGE}" -c '
 import os

tests/test_docker_console_packaging.py

@@ -0,0 +1,28 @@
+"""Console is installed through plugins, not fetched during Docker builds."""
+
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+@pytest.mark.parametrize(
+    "relative",
+    [
+        "dockerfile",
+        "docker-compose.build.yml",
+        ".github/workflows/docker-publish.yml",
+        "env.example",
+    ],
+)
+def test_docker_build_has_no_console_bundle_dependency(relative):
+    content = (ROOT / relative).read_text().lower()
+    for obsolete in (
+        "openhop_console",
+        "pymc_console",
+        "console_release",
+        "pymc-ui-",
+        "/opt/pymc_console",
+    ):
+        assert obsolete not in content, (relative, obsolete)

tests/test_docker_entrypoint_defaults.py

@@ -0,0 +1,80 @@
+"""Exercise entrypoint upgrade defaults without starting the repeater."""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+import pytest
+import yaml
+
+
+ENTRYPOINT = Path(__file__).resolve().parents[1] / "docker-entrypoint.sh"
+
+
+@pytest.mark.parametrize("bundled", [True, False], ids=["image-upgrade", "stored-fallback"])
+def test_upgrade_defaults_preserve_user_config_and_stored_example(tmp_path, bundled):
+    install = tmp_path / "install"
+    config = tmp_path / "config"
+    bin_dir = tmp_path / "bin"
+    for directory in (install, config, bin_dir):
+        directory.mkdir()
+    stored = {"http": {"port": 8000, "old_default": True}}
+    current = {"http": {"port": 9000, "new_default": True}}
+    user = {"http": {"port": 1234}, "user_only": "keep"}
+    example = config / "config.yaml.example"
+    example.write_text(json.dumps(stored))
+    original_example = example.read_bytes()
+    config_path = config / "config.yaml"
+    config_path.write_text(json.dumps(user))
+    if bundled:
+        (install / "config.yaml.example").write_text(json.dumps(current))
+
+    # Only the daemon launcher is stubbed; the real shell entrypoint runs.
+    launcher = bin_dir / "python3"
+    launcher.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$LAUNCH_ARGS"\n')
+    launcher.chmod(0o755)
+    # A JSON-only yq double keeps normal unit tests independent of Docker.
+    # Set ENTRYPOINT_TEST_YQ to exercise the identical regression with real yq.
+    yq = bin_dir / "yq"
+    yq.write_text(
+        f"#!{sys.executable}\n"
+        "import json, sys\n"
+        "if sys.argv[1] == '--version':\n"
+        "    print('mikefarah/yq test double'); sys.exit(0)\n"
+        "def merge(a, b):\n"
+        "    if isinstance(a, dict) and isinstance(b, dict):\n"
+        "        return {k: merge(a.get(k), b[k]) if k in b else a[k] "
+        "for k in a.keys() | b.keys()}\n"
+        "    return b\n"
+        "with open(sys.argv[3]) as f: result = json.load(f)\n"
+        "if sys.argv[1] == 'eval-all':\n"
+        "    with open(sys.argv[4]) as f: result = merge(result, json.load(f))\n"
+        "print(json.dumps(result))\n"
+    )
+    yq.chmod(0o755)
+    args_path = tmp_path / "launch-args"
+    env = {
+        **os.environ,
+        "INSTALL_DIR": str(install),
+        "CONFIG_DIR": str(config),
+        "OPENHOP_REPEATER_CONFIG": str(config_path),
+        "YQ_CMD": os.environ.get("ENTRYPOINT_TEST_YQ", str(yq)),
+        "PATH": f"{bin_dir}:{os.environ['PATH']}",
+        "LAUNCH_ARGS": str(args_path),
+    }
+    result = subprocess.run(
+        ["sh", str(ENTRYPOINT)], env=env, capture_output=True, text=True, timeout=20
+    )
+    assert result.returncode == 0, result.stderr
+    merged = yaml.safe_load(config_path.read_text())
+    defaults = current if bundled else stored
+    assert merged == {"http": {**defaults["http"], "port": 1234}, "user_only": "keep"}
+    assert example.read_bytes() == original_example
+    assert args_path.read_text().splitlines() == [
+        "-m",
+        "repeater.plugins.container_supervisor",
+        "--config",
+        str(config_path),
+    ]