diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9116e76..03de5f6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,10 +11,6 @@ on: description: "Docker image repository to publish to" required: false default: "" - openhop_console_version: - description: "openHop Console release to bundle (latest or a tag such as v0.9.333)" - required: false - default: "latest" jobs: docker: @@ -75,28 +71,6 @@ jobs: echo "Using image repository: ${image_repository}" echo "image_repository=${image_repository}" >> "$GITHUB_OUTPUT" - - name: Resolve bundled openHop Console release - id: console_release - shell: bash - env: - REQUESTED_CONSOLE_VERSION: ${{ inputs.openhop_console_version || 'latest' }} - run: | - set -euo pipefail - - if [ "${REQUESTED_CONSOLE_VERSION}" = "latest" ]; then - release_json="$(curl -fsSL https://api.github.com/repos/Treehouse-00/pymc_console-dist/releases/latest)" - tag="$(jq -r '.tag_name' <<<"${release_json}")" - published_at="$(jq -r '.published_at' <<<"${release_json}")" - cache_bust="${tag}@${published_at}" - else - tag="${REQUESTED_CONSOLE_VERSION}" - cache_bust="${REQUESTED_CONSOLE_VERSION}" - fi - - echo "Bundling openHop Console ${tag}" - echo "version=${REQUESTED_CONSOLE_VERSION}" >> "$GITHUB_OUTPUT" - echo "cache_bust=${cache_bust}" >> "$GITHUB_OUTPUT" - - name: Extract Docker metadata id: meta uses: docker/metadata-action@v5 @@ -117,8 +91,6 @@ jobs: push: true build-args: | PACKAGE_VERSION=${{ steps.package_version.outputs.version }} - OPENHOP_CONSOLE_VERSION=${{ steps.console_release.outputs.version }} - OPENHOP_CONSOLE_CACHE_BUST=${{ steps.console_release.outputs.cache_bust }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha diff --git a/README.md b/README.md index efed791..32b971e 100644 --- a/README.md +++ b/README.md @@ -255,6 +255,11 @@ The web interface can upgrade an installation or switch branches. > Docker installs cannot be upgraded or branch-switched from the web interface. > Update the container image instead. +Existing native installations must run the updated `sudo bash ./manage.sh upgrade` +once as an administrator to replace the privileged OTA helper after the security +update. Updating only the Python package or using the old web updater does not +refresh that helper. See [native upgrade prerequisites](docs/plugins.md#native-upgrade-prerequisite). + ![Upgrade](docs/webui-upgrade.png) ### CLI @@ -451,22 +456,80 @@ that source as a directory, which breaks startup. 1. Copy `.env.example` to `.env`. 2. Review `.env` and update `OPENHOP_REPEATER_IMAGE`, `DIALOUT_GID`, `GPIO_GID`, or `SPI_GID` if needed. -3. Configure `docker-compose.yml` for your hardware and device paths. -4. Uncomment the USB device mapping only if your host has that device path. +3. Configure `docker-compose.yml` for your hardware. Remove every device mapping + that is absent or unnecessary: the shipped file enables SPI, GPIO, and USB + bus examples. Network-radio installations do not need those mappings. +4. Add a serial-modem mapping only after preparing its host device path. 5. Pull and start the container. ```bash docker compose up -d ``` -The Docker image bundles the optional openHop Console frontend at -`/opt/pymc_console/web/html`. Local builds with `docker-compose.build.yml` -download the newest openHop Console release by default. Set `OPENHOP_CONSOLE_VERSION` -in `.env` to pin a release tag such as `v0.9.329`. +The Docker image includes the default RepeaterUI but no longer downloads or bundles +openHop Console. Install optional Console functionality through the plugin catalogue. -After the container starts, open Web Settings and choose `openHop Console` as the -web frontend if you want the repeater to serve that UI. The existing -`/api/check_pymc_console` endpoint should report that the Console path exists. +Before upgrading an older image with `/opt/pymc_console/web/html` selected as its +frontend, switch back to the default RepeaterUI. The old bundled directory is absent +from new images. Existing plugin data remains in the persistent data volume. + +### Image updates and persistent plugins + +For a published image, select the intended image tag in `.env`, then run: + +```bash +docker compose pull +docker compose up -d +``` + +For a local source build, after updating the intended branch: + +```bash +docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build +``` + +Pulling Git changes or restarting a container alone does not rebuild its image. +The Dockerfile packages the frontend assets present in the build context; it does +not fetch or build the default RepeaterUI. Frontend changes belong in +[RepeaterUI](https://github.com/openhop-dev/openHop_RepeaterUI), using the matching +UI branch (`feat-plugin-ui` for `feat-plugin-manager`). Stage the complete verified +UI build for packaging; do not hand-edit generated JavaScript. Optional Console +installation is managed through the plugin catalogue, independently of image builds. + +Back up both config and data volumes before upgrading. Do **not** use +`docker compose down -v` if you want to retain configuration and plugins. +The default data volume contains plugin releases, retained wheels, venvs, +settings, logs, and manager state. A custom `plugins.root` outside that volume +needs its own persistent writable mount. + +Plugin venvs can rebuild after a Python minor-version change. Keep retained wheels; +plugins with external dependencies may need network access and compatible packages +to rebuild. Older installations without retained wheels need plugin reinstallation. +Changing CPU architecture or absolute storage paths is not covered by minor-version +rebuild detection. + +The image runs Repeater and its plugin manager as an unprivileged user under a +supervisor and `tini`. To run without the manager, set `plugins.enabled: false` in +configuration, or explicitly add this to the Compose service: + +```yaml +environment: + OPENHOP_PLUGIN_MANAGER: "0" +``` + +Putting that variable only in `.env` does not pass it through the shipped Compose +file. Repeater remains available without the manager; plugin operations are then +unavailable. Plugins are trusted applications, not sandboxed code. + +On startup, existing user configuration overrides defaults from the current image's +bundled example; an older example in the config volume is preserved but is not the +upgrade merge source. Keep config writable by the image user. The temporary merged +config fallback is not persistent; use an absolute `storage.storage_dir` inside the +mounted data directory. + +The volume variables accept the default named volumes or absolute bind paths. If +you choose another named volume, also declare it under top-level `volumes:`; use +`external: true` when intentionally attaching an existing volume. ### Example `docker-compose.yml` diff --git a/docker-compose.build.yml b/docker-compose.build.yml index 76e4337..f8b1b49 100644 --- a/docker-compose.build.yml +++ b/docker-compose.build.yml @@ -10,4 +10,3 @@ services: DIALOUT_GID: ${DIALOUT_GID:-20} GPIO_GID: ${GPIO_GID:-986} SPI_GID: ${SPI_GID:-989} - OPENHOP_CONSOLE_VERSION: ${OPENHOP_CONSOLE_VERSION:-${PYMC_CONSOLE_VERSION:-latest}} diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 32a3362..565bed5 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -57,8 +57,13 @@ use_runtime_merged_config() { merge_config_from_example() { config_path="$1" + # Image defaults evolve on upgrade; preserve the volume's example unchanged. + merge_example_path="${EXAMPLE_PATH}" + if [ -f "${BUNDLED_EXAMPLE_PATH}" ]; then + merge_example_path="${BUNDLED_EXAMPLE_PATH}" + fi - if [ ! -f "${config_path}" ] || [ ! -f "${EXAMPLE_PATH}" ]; then + if [ ! -f "${config_path}" ] || [ ! -f "${merge_example_path}" ]; then return 0 fi @@ -79,8 +84,8 @@ merge_config_from_example() { # Keep only the example's comments to avoid comment duplication across upgrades. "${YQ_CMD}" eval '... comments=""' "${config_path}" > "${stripped_user}" 2>/dev/null || cp "${config_path}" "${stripped_user}" - if ! "${YQ_CMD}" eval-all '. as $item ireduce ({}; . * $item)' "${EXAMPLE_PATH}" "${stripped_user}" > "${merged_config}" 2>/dev/null; then - echo "Failed to merge ${config_path} with ${EXAMPLE_PATH}; keeping the existing config." >&2 + if ! "${YQ_CMD}" eval-all '. as $item ireduce ({}; . * $item)' "${merge_example_path}" "${stripped_user}" > "${merged_config}" 2>/dev/null; then + echo "Failed to merge ${config_path} with ${merge_example_path}; keeping the existing config." >&2 cleanup_merge trap - EXIT HUP INT TERM return 0 diff --git a/dockerfile b/dockerfile index f6818cb..921d2a5 100644 --- a/dockerfile +++ b/dockerfile @@ -10,14 +10,10 @@ ARG GPIO_GID=986 ARG SPI_GID=989 ARG TARGETARCH ARG YQ_VERSION=v4.40.5 -ARG OPENHOP_CONSOLE_REPO=Treehouse-00/pymc_console-dist -ARG OPENHOP_CONSOLE_VERSION=latest -ARG OPENHOP_CONSOLE_CACHE_BUST=default ENV INSTALL_DIR=/opt/openhop_repeater \ CONFIG_DIR=/etc/openhop_repeater \ DATA_DIR=/var/lib/openhop_repeater \ - OPENHOP_CONSOLE_WEB_DIR=/opt/pymc_console/web/html \ HOME_DIR=/home/${USER} \ PATH=/home/${USER}/.local/bin:${PATH} \ PYTHONUNBUFFERED=1 \ @@ -54,22 +50,6 @@ RUN arch="${TARGETARCH:-}" \ && wget -qO /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${YQ_BINARY}" \ && chmod +x /usr/local/bin/yq -# Bundle the optional openHop Console frontend so the web UI can select it without -# requiring a host bind mount. Use OPENHOP_CONSOLE_VERSION=latest to pull the -# newest release at image build time, or pin a tag such as v0.9.329. -RUN set -eux; \ - echo "Bundling openHop Console cache key: ${OPENHOP_CONSOLE_CACHE_BUST}"; \ - mkdir -p "${OPENHOP_CONSOLE_WEB_DIR}"; \ - if [ "${OPENHOP_CONSOLE_VERSION}" = "latest" ]; then \ - console_url="https://github.com/${OPENHOP_CONSOLE_REPO}/releases/latest/download/pymc-ui-latest.tar.gz"; \ - else \ - console_url="https://github.com/${OPENHOP_CONSOLE_REPO}/releases/download/${OPENHOP_CONSOLE_VERSION}/pymc-ui-${OPENHOP_CONSOLE_VERSION}.tar.gz"; \ - fi; \ - wget -qO /tmp/pymc-console.tar.gz "${console_url}"; \ - tar -xzf /tmp/pymc-console.tar.gz -C "${OPENHOP_CONSOLE_WEB_DIR}"; \ - rm /tmp/pymc-console.tar.gz; \ - test -f "${OPENHOP_CONSOLE_WEB_DIR}/index.html" - # Create the group and user in order to run without root privileges RUN groupadd --gid "$PGID" "$GROUP" \ && (getent group dialout >/dev/null || groupadd --gid "$DIALOUT_GID" dialout) \ @@ -80,7 +60,7 @@ RUN groupadd --gid "$PGID" "$GROUP" \ # Create runtime directories RUN mkdir -p ${INSTALL_DIR} ${CONFIG_DIR} ${DATA_DIR} \ - && chown -R "$USER":"$GROUP" ${INSTALL_DIR} ${CONFIG_DIR} ${DATA_DIR} ${HOME_DIR} /opt/pymc_console + && chown -R "$USER":"$GROUP" ${INSTALL_DIR} ${CONFIG_DIR} ${DATA_DIR} ${HOME_DIR} WORKDIR ${INSTALL_DIR} diff --git a/env.example b/env.example index c89b449..c0a3d88 100644 --- a/env.example +++ b/env.example @@ -29,9 +29,3 @@ SPI_GID=989 # image yourself instead of pulling OPENHOP_REPEATER_IMAGE. PUID=15888 PGID=15888 - -# Local build only. The Docker image bundles openHop Console under -# /opt/pymc_console/web/html so it is available in Web Settings without a host -# bind mount. Use "latest" to pull the newest openHop Console release at build -# time, or pin a tag such as v0.9.329. -OPENHOP_CONSOLE_VERSION=latest diff --git a/scripts/docker-plugin-smoke.sh b/scripts/docker-plugin-smoke.sh index 57f74c9..0aaa89e 100755 --- a/scripts/docker-plugin-smoke.sh +++ b/scripts/docker-plugin-smoke.sh @@ -14,7 +14,7 @@ cleanup() { trap cleanup EXIT HUP INT TERM start_container() { - docker run -d \ + docker run -d --network none \ --name "${CONTAINER}" \ -v "${CONFIG_VOLUME}:/etc/openhop_repeater" \ -v "${DATA_VOLUME}:/var/lib/openhop_repeater" \ @@ -92,10 +92,14 @@ def stop(signum, frame): def main(): signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) + child = os.fork() with lifecycle.open("a", encoding="utf-8") as handle: - handle.write(f"START {os.getpid()}\\n") + kind = "START" if child else "CHILD" + handle.write(f"{kind} {os.getpid()}\\n") while running: time.sleep(0.1) + if child: + os.waitpid(child, 0) ''' metadata = "Metadata-Version: 2.1\nName: openhop-smoke-plugin\nVersion: 0.1.0\n" wheel = "Wheel-Version: 1.0\nGenerator: openhop-container-smoke\nRoot-Is-Purelib: true\nTag: py3-none-any\n" @@ -228,6 +232,47 @@ assert status["enabled"] is True assert status["state"] == "RUNNING" ' +echo "Checking plugin stop/start, descendants, permissions and venv rebuild" +docker exec -i "${CONTAINER}" python3 - <<'PY' +import os +import re +import stat +import sys +import time +from pathlib import Path +from repeater.plugins.ipc import PluginIPCClient +client = PluginIPCClient("/var/lib/openhop_repeater/plugin-manager.sock") +root = Path("/var/lib/openhop_repeater/plugins/openhop.smoke") +log = root / "data/lifecycle.log" +for _ in range(50): + lines = log.read_text().splitlines() + children = [int(line.split()[1]) for line in lines if line.startswith("CHILD ")] + if children: + break + time.sleep(0.1) +assert children +pid = client.status("openhop.smoke")["pid"] +child = children[-1] +assert Path(f"/proc/{child}").exists() +client.call("stop", plugin_id="openhop.smoke") +for _ in range(50): + if not any(Path(f"/proc/{p}").exists() for p in (pid, child)): + break + time.sleep(0.1) +assert not any(Path(f"/proc/{p}").exists() for p in (pid, child)) +assert f"TERM {pid}" in log.read_text().splitlines() +assert f"TERM {child}" in log.read_text().splitlines() +assert os.getuid() == 15888 +assert root.stat().st_uid == 15888 +assert stat.S_IMODE(Path(client.socket_path).stat().st_mode) == 0o660 +venv_config = root / "releases/0.1.0/venv/pyvenv.cfg" +venv_config.write_text(re.sub(r"(?m)^version = .*", "version = 3.11.0", venv_config.read_text())) +client.call("start", plugin_id="openhop.smoke") +assert client.status("openhop.smoke")["state"] == "RUNNING" +assert f"version = {sys.version_info.major}.{sys.version_info.minor}." in venv_config.read_text() +print("stop/start descendants, non-root/socket permissions, real offline venv rebuild passed") +PY + echo "Checking graceful plugin shutdown" EXPECTED_PLUGIN_PID="$(docker exec "${CONTAINER}" python3 -c ' from repeater.plugins.ipc import PluginIPCClient @@ -235,7 +280,7 @@ client = PluginIPCClient("/var/lib/openhop_repeater/plugin-manager.sock") print(client.status("openhop.smoke")["pid"]) ')" docker stop -t 10 "${CONTAINER}" >/dev/null -docker run --rm --entrypoint python3 -e EXPECTED_PLUGIN_PID="${EXPECTED_PLUGIN_PID}" \ +docker run --rm --network none --entrypoint python3 -e EXPECTED_PLUGIN_PID="${EXPECTED_PLUGIN_PID}" \ -v "${DATA_VOLUME}:/var/lib/openhop_repeater" \ "${IMAGE}" -c ' import os diff --git a/tests/test_docker_console_packaging.py b/tests/test_docker_console_packaging.py new file mode 100644 index 0000000..eb75d3c --- /dev/null +++ b/tests/test_docker_console_packaging.py @@ -0,0 +1,28 @@ +"""Console is installed through plugins, not fetched during Docker builds.""" + +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize( + "relative", + [ + "dockerfile", + "docker-compose.build.yml", + ".github/workflows/docker-publish.yml", + "env.example", + ], +) +def test_docker_build_has_no_console_bundle_dependency(relative): + content = (ROOT / relative).read_text().lower() + for obsolete in ( + "openhop_console", + "pymc_console", + "console_release", + "pymc-ui-", + "/opt/pymc_console", + ): + assert obsolete not in content, (relative, obsolete) diff --git a/tests/test_docker_entrypoint_defaults.py b/tests/test_docker_entrypoint_defaults.py new file mode 100644 index 0000000..138ad74 --- /dev/null +++ b/tests/test_docker_entrypoint_defaults.py @@ -0,0 +1,80 @@ +"""Exercise entrypoint upgrade defaults without starting the repeater.""" + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest +import yaml + + +ENTRYPOINT = Path(__file__).resolve().parents[1] / "docker-entrypoint.sh" + + +@pytest.mark.parametrize("bundled", [True, False], ids=["image-upgrade", "stored-fallback"]) +def test_upgrade_defaults_preserve_user_config_and_stored_example(tmp_path, bundled): + install = tmp_path / "install" + config = tmp_path / "config" + bin_dir = tmp_path / "bin" + for directory in (install, config, bin_dir): + directory.mkdir() + stored = {"http": {"port": 8000, "old_default": True}} + current = {"http": {"port": 9000, "new_default": True}} + user = {"http": {"port": 1234}, "user_only": "keep"} + example = config / "config.yaml.example" + example.write_text(json.dumps(stored)) + original_example = example.read_bytes() + config_path = config / "config.yaml" + config_path.write_text(json.dumps(user)) + if bundled: + (install / "config.yaml.example").write_text(json.dumps(current)) + + # Only the daemon launcher is stubbed; the real shell entrypoint runs. + launcher = bin_dir / "python3" + launcher.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$LAUNCH_ARGS"\n') + launcher.chmod(0o755) + # A JSON-only yq double keeps normal unit tests independent of Docker. + # Set ENTRYPOINT_TEST_YQ to exercise the identical regression with real yq. + yq = bin_dir / "yq" + yq.write_text( + f"#!{sys.executable}\n" + "import json, sys\n" + "if sys.argv[1] == '--version':\n" + " print('mikefarah/yq test double'); sys.exit(0)\n" + "def merge(a, b):\n" + " if isinstance(a, dict) and isinstance(b, dict):\n" + " return {k: merge(a.get(k), b[k]) if k in b else a[k] " + "for k in a.keys() | b.keys()}\n" + " return b\n" + "with open(sys.argv[3]) as f: result = json.load(f)\n" + "if sys.argv[1] == 'eval-all':\n" + " with open(sys.argv[4]) as f: result = merge(result, json.load(f))\n" + "print(json.dumps(result))\n" + ) + yq.chmod(0o755) + args_path = tmp_path / "launch-args" + env = { + **os.environ, + "INSTALL_DIR": str(install), + "CONFIG_DIR": str(config), + "OPENHOP_REPEATER_CONFIG": str(config_path), + "YQ_CMD": os.environ.get("ENTRYPOINT_TEST_YQ", str(yq)), + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "LAUNCH_ARGS": str(args_path), + } + result = subprocess.run( + ["sh", str(ENTRYPOINT)], env=env, capture_output=True, text=True, timeout=20 + ) + assert result.returncode == 0, result.stderr + merged = yaml.safe_load(config_path.read_text()) + defaults = current if bundled else stored + assert merged == {"http": {**defaults["http"], "port": 1234}, "user_only": "keep"} + assert example.read_bytes() == original_example + assert args_path.read_text().splitlines() == [ + "-m", + "repeater.plugins.container_supervisor", + "--config", + str(config_path), + ]