adam/pyMC_Repeater
1
0
Fork 0
mirror of https://github.com/pyMC-dev/pyMC_Repeater.git synced 2026-08-06 08:53:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #178 from rightup/feat-remove-sys-packages

Browse Source 
feat: migrate to virtual environment and clean up system-level packages
This commit is contained in:
Lloyd
2026-04-15 10:40:58 +01:00
committed by GitHub
parent 4d49eb701b f2a72eb203
commit 4d9c560b11
3 changed files with 188 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manage.sh
+137 -49
View File
@@ -4,12 +4,69 @@
set -e
INSTALL_DIR="/opt/pymc_repeater"
VENV_DIR="$INSTALL_DIR/venv"
VENV_PIP="$VENV_DIR/bin/pip"
VENV_PYTHON="$VENV_DIR/bin/python"
CONFIG_DIR="/etc/pymc_repeater"
LOG_DIR="/var/log/pymc_repeater"
SERVICE_USER="repeater"
SERVICE_NAME="pymc-repeater"
SILENT_MODE="${PYMC_SILENT:-${SILENT:-}}"
# ---------------------------------------------------------------------------
# Virtual-environment helpers
# ---------------------------------------------------------------------------
# Create (or re-create) the dedicated venv for pymc_repeater
ensure_venv() {
if [ ! -x "$VENV_PYTHON" ]; then
echo ">>> Creating virtual environment at $VENV_DIR ..."
python3 -m venv --system-site-packages "$VENV_DIR"
# Upgrade pip inside the venv
"$VENV_PIP" install --upgrade pip setuptools wheel >/dev/null 2>&1 || true
fi
}
# Migrate an existing system-pip install into the venv.
# Idempotent: safe to call on every upgrade.
migrate_to_venv() {
echo ">>> Checking for legacy system-pip installation..."
# 1. Ensure the venv exists
ensure_venv
# 2. Remove legacy PYTHONPATH from the service unit
local svc_unit="/etc/systemd/system/pymc-repeater.service"
if [ -f "$svc_unit" ]; then
if grep -q 'PYTHONPATH' "$svc_unit" 2>/dev/null; then
sed -i '/^Environment=.*PYTHONPATH/d' "$svc_unit"
echo " ✓ Removed legacy PYTHONPATH from service unit"
fi
# 3. Fix WorkingDirectory if still pointing at old source
if grep -q 'WorkingDirectory=/opt/pymc_repeater' "$svc_unit" 2>/dev/null; then
sed -i 's|WorkingDirectory=/opt/pymc_repeater|WorkingDirectory=/var/lib/pymc_repeater|' "$svc_unit"
echo " ✓ Fixed WorkingDirectory in service unit"
fi
# 4. Ensure ExecStart uses the venv python
if grep -q 'ExecStart=/usr/bin/python3' "$svc_unit" 2>/dev/null; then
sed -i "s|ExecStart=/usr/bin/python3|ExecStart=$VENV_PYTHON|" "$svc_unit"
echo " ✓ Updated ExecStart to use venv python"
fi
systemctl daemon-reload
fi
# 5. Remove the package from system python (best-effort)
python3 -m pip uninstall -y pymc_repeater 2>/dev/null || true
python3 -m pip uninstall -y pymc_core 2>/dev/null || true
echo " ✓ Cleaned up system-level packages (if any)"
# 6. Remove stale source trees that could shadow the venv package
if [ -d "$INSTALL_DIR/repeater" ]; then
rm -rf "$INSTALL_DIR/repeater"
echo " ✓ Removed stale source tree from $INSTALL_DIR/repeater"
fi
}
is_silent_flag() {
case "${1:-}" in
--silent|-y|silent) return 0 ;;
@@ -96,9 +153,15 @@ is_enabled() {
# Function to get current version
get_version() {
# Read version from the pip-installed package in dist-packages
python3 -c "from importlib.metadata import version; print(version('pymc_repeater'))" 2>/dev/null \
|| echo "not installed"
# Read version from the pip-installed package in the venv
if [ -x "$VENV_PYTHON" ]; then
"$VENV_PYTHON" -c "from importlib.metadata import version; print(version('pymc_repeater'))" 2>/dev/null \
|| echo "not installed"
else
# Fallback: try system python for pre-migration installs
python3 -c "from importlib.metadata import version; print(version('pymc_repeater'))" 2>/dev/null \
|| echo "not installed"
fi
}
# Function to get service status for display
@@ -272,12 +335,16 @@ install_repeater() {
echo "25"; echo "# Installing system dependencies..."
apt-get update -qq
DEBIAN_FRONTEND=noninteractive apt-get install -y libffi-dev libusb-1.0-0 sudo jq pip python3-rrdtool wget swig build-essential python3-dev
DEBIAN_FRONTEND=noninteractive apt-get install -y libffi-dev libusb-1.0-0 sudo jq pip python3-venv python3-rrdtool wget swig build-essential python3-dev
# Install polkit (package name varies by distro version)
DEBIAN_FRONTEND=noninteractive apt-get install -y policykit-1 2>/dev/null \
|| DEBIAN_FRONTEND=noninteractive apt-get install -y polkitd pkexec 2>/dev/null \
|| echo " Warning: Could not install polkit (sudo fallback will be used)"
pip install --break-system-packages setuptools_scm >/dev/null 2>&1 || true
# setuptools_scm needed for git version detection during build
pip install --break-system-packages setuptools_scm >/dev/null 2>&1 || python3 -m pip install --break-system-packages setuptools_scm >/dev/null 2>&1 || true
echo "28"; echo "# Creating virtual environment..."
ensure_venv
# Install mikefarah yq v4 if not already installed
if ! command -v yq &> /dev/null || [[ "$(yq --version 2>&1)" != *"mikefarah/yq"* ]]; then
@@ -316,8 +383,12 @@ install_repeater() {
fi
echo "65"; echo "# Setting permissions..."
chown -R "$SERVICE_USER:$SERVICE_USER" "$INSTALL_DIR" "$CONFIG_DIR" "$LOG_DIR" /var/lib/pymc_repeater
# Venv stays root-owned (pip runs as root); service user only needs read+execute
chown -R "$SERVICE_USER:$SERVICE_USER" "$CONFIG_DIR" "$LOG_DIR" /var/lib/pymc_repeater
chmod 750 "$CONFIG_DIR" "$LOG_DIR" /var/lib/pymc_repeater
# Ensure manage.sh and support files in INSTALL_DIR are accessible
chown root:root "$INSTALL_DIR"
chmod 755 "$INSTALL_DIR"
# Ensure the service user can create subdirectories in their home directory
chmod 755 /var/lib/pymc_repeater
# Pre-create the .config directory that the service will need
@@ -355,35 +426,46 @@ EOF
set -e
CHANNEL="${1:-main}"
PRETEND_VERSION="${2:-}"
VENV_DIR="/opt/pymc_repeater/venv"
VENV_PIP="$VENV_DIR/bin/pip"
VENV_PYTHON="$VENV_DIR/bin/python"
# Validate: only allow safe git ref characters
if ! [[ "$CHANNEL" =~ ^[a-zA-Z0-9._/-]{1,80}$ ]]; then
echo "Invalid channel name: $CHANNEL" >&2
exit 1
fi
export PIP_ROOT_USER_ACTION=ignore
# If caller supplied a version string, tell setuptools_scm to use it (sudo
# strips env vars so it is passed as a positional argument instead).
[ -n "$PRETEND_VERSION" ] && export SETUPTOOLS_SCM_PRETEND_VERSION="$PRETEND_VERSION"
# Migration: remove legacy PYTHONPATH from service unit if present.
# Old installs set PYTHONPATH=/opt/pymc_repeater which caused the service to
# load from a stale source copy instead of the pip-installed dist-packages.
# ---- Migration: ensure venv exists (handles upgrades from system-pip era) ----
if [ ! -x "$VENV_PYTHON" ]; then
echo "[pymc-do-upgrade] Creating venv at $VENV_DIR ..."
python3 -m venv --system-site-packages "$VENV_DIR"
"$VENV_PIP" install --upgrade pip setuptools wheel >/dev/null 2>&1 || true
fi
# ---- Migration: clean up legacy service unit issues ----
SVC_UNIT=/etc/systemd/system/pymc-repeater.service
if grep -q 'PYTHONPATH' "$SVC_UNIT" 2>/dev/null; then
sed -i '/^Environment=.*PYTHONPATH/d' "$SVC_UNIT"
systemctl daemon-reload
fi
# Migration: fix WorkingDirectory if it still points at the old source checkout.
# /opt/pymc_repeater contains a repeater/ subdirectory which shadows the
# pip-installed package, causing updates to have no effect on the running process.
if grep -q 'WorkingDirectory=/opt/pymc_repeater' "$SVC_UNIT" 2>/dev/null; then
sed -i 's|WorkingDirectory=/opt/pymc_repeater|WorkingDirectory=/var/lib/pymc_repeater|' "$SVC_UNIT"
systemctl daemon-reload
fi
exec python3 -m pip install \
--break-system-packages \
if grep -q 'ExecStart=/usr/bin/python3' "$SVC_UNIT" 2>/dev/null; then
sed -i "s|ExecStart=/usr/bin/python3|ExecStart=$VENV_PYTHON|" "$SVC_UNIT"
systemctl daemon-reload
fi
# ---- Remove stale source trees that shadow the venv package ----
[ -d /opt/pymc_repeater/repeater ] && rm -rf /opt/pymc_repeater/repeater
# ---- Remove old system-level packages to avoid confusion ----
python3 -m pip uninstall -y pymc_repeater 2>/dev/null || true
python3 -m pip uninstall -y pymc_core 2>/dev/null || true
# ---- Install into the venv ----
exec "$VENV_PIP" install \
--upgrade \
--no-cache-dir \
--force-reinstall \
--ignore-installed \
"pymc_repeater[hardware] @ git+https://github.com/rightup/pyMC_Repeater.git@${CHANNEL}"
UPGRADEEOF
chmod 0755 /usr/local/bin/pymc-do-upgrade
@@ -405,9 +487,6 @@ UPGRADEEOF
SCRIPT_DIR="$(dirname "$0")"
cd "$SCRIPT_DIR"
# Suppress pip root user warnings
export PIP_ROOT_USER_ACTION=ignore
# Calculate version from git for setuptools_scm
if [ -d .git ]; then
git fetch --tags 2>/dev/null || true
@@ -423,13 +502,12 @@ UPGRADEEOF
echo "Note: Using optimized binary wheels for faster installation"
echo ""
# Remove old pymc_core first so no stale .py/.pyc files linger
python3 -m pip uninstall -y pymc_core 2>/dev/null || true
# Ensure venv exists
ensure_venv
# Install with --force-reinstall to ensure fresh pymc_core from GitHub
# --ignore-installed avoids failures on system-managed packages (e.g. PyYAML)
echo "Installing pymc_repeater with fresh dependencies from pyproject.toml..."
if python3 -m pip install --break-system-packages --no-cache-dir --force-reinstall --ignore-installed .[hardware]; then
# Install into the venv (clean, no system-packages flags needed)
echo "Installing pymc_repeater into venv ($VENV_DIR)..."
if "$VENV_PIP" install --upgrade --no-cache-dir .[hardware]; then
echo ""
echo "✓ Python package installation completed successfully!"
@@ -601,12 +679,12 @@ upgrade_repeater() {
echo "[3/9] Updating system dependencies..."
apt-get update -qq
apt-get install -y libffi-dev libusb-1.0-0 sudo jq pip python3-rrdtool wget swig build-essential python3-dev
apt-get install -y libffi-dev libusb-1.0-0 sudo jq pip python3-venv python3-rrdtool wget swig build-essential python3-dev
# Install polkit (package name varies by distro version)
apt-get install -y policykit-1 2>/dev/null \
|| apt-get install -y polkitd pkexec 2>/dev/null \
|| echo " Warning: Could not install polkit (sudo fallback will be used)"
pip install --break-system-packages setuptools_scm >/dev/null 2>&1 || true
pip install --break-system-packages setuptools_scm >/dev/null 2>&1 || python3 -m pip install --break-system-packages setuptools_scm >/dev/null 2>&1 || true
# Install mikefarah yq v4 if not already installed
if ! command -v yq &> /dev/null || [[ "$(yq --version 2>&1)" != *"mikefarah/yq"* ]]; then
@@ -654,7 +732,10 @@ upgrade_repeater() {
echo " ✓ User groups updated"
echo "[6/9] Fixing permissions..."
chown -R "$SERVICE_USER:$SERVICE_USER" "$INSTALL_DIR" "$CONFIG_DIR" "$LOG_DIR" /var/lib/pymc_repeater 2>/dev/null || true
# Venv stays root-owned (pip runs as root); service user only needs read+execute
chown -R "$SERVICE_USER:$SERVICE_USER" "$CONFIG_DIR" "$LOG_DIR" /var/lib/pymc_repeater 2>/dev/null || true
chown root:root "$INSTALL_DIR" 2>/dev/null || true
chmod 755 "$INSTALL_DIR" 2>/dev/null || true
chmod 750 "$CONFIG_DIR" "$LOG_DIR" 2>/dev/null || true
chmod 755 /var/lib/pymc_repeater 2>/dev/null || true
# Pre-create the .config directory that the service will need
@@ -687,35 +768,46 @@ EOF
set -e
CHANNEL="${1:-main}"
PRETEND_VERSION="${2:-}"
VENV_DIR="/opt/pymc_repeater/venv"
VENV_PIP="$VENV_DIR/bin/pip"
VENV_PYTHON="$VENV_DIR/bin/python"
# Validate: only allow safe git ref characters
if ! [[ "$CHANNEL" =~ ^[a-zA-Z0-9._/-]{1,80}$ ]]; then
echo "Invalid channel name: $CHANNEL" >&2
exit 1
fi
export PIP_ROOT_USER_ACTION=ignore
# If caller supplied a version string, tell setuptools_scm to use it (sudo
# strips env vars so it is passed as a positional argument instead).
[ -n "$PRETEND_VERSION" ] && export SETUPTOOLS_SCM_PRETEND_VERSION="$PRETEND_VERSION"
# Migration: remove legacy PYTHONPATH from service unit if present.
# Old installs set PYTHONPATH=/opt/pymc_repeater which caused the service to
# load from a stale source copy instead of the pip-installed dist-packages.
# ---- Migration: ensure venv exists (handles upgrades from system-pip era) ----
if [ ! -x "$VENV_PYTHON" ]; then
echo "[pymc-do-upgrade] Creating venv at $VENV_DIR ..."
python3 -m venv --system-site-packages "$VENV_DIR"
"$VENV_PIP" install --upgrade pip setuptools wheel >/dev/null 2>&1 || true
fi
# ---- Migration: clean up legacy service unit issues ----
SVC_UNIT=/etc/systemd/system/pymc-repeater.service
if grep -q 'PYTHONPATH' "$SVC_UNIT" 2>/dev/null; then
sed -i '/^Environment=.*PYTHONPATH/d' "$SVC_UNIT"
systemctl daemon-reload
fi
# Migration: fix WorkingDirectory if it still points at the old source checkout.
# /opt/pymc_repeater contains a repeater/ subdirectory which shadows the
# pip-installed package, causing updates to have no effect on the running process.
if grep -q 'WorkingDirectory=/opt/pymc_repeater' "$SVC_UNIT" 2>/dev/null; then
sed -i 's|WorkingDirectory=/opt/pymc_repeater|WorkingDirectory=/var/lib/pymc_repeater|' "$SVC_UNIT"
systemctl daemon-reload
fi
exec python3 -m pip install \
--break-system-packages \
if grep -q 'ExecStart=/usr/bin/python3' "$SVC_UNIT" 2>/dev/null; then
sed -i "s|ExecStart=/usr/bin/python3|ExecStart=$VENV_PYTHON|" "$SVC_UNIT"
systemctl daemon-reload
fi
# ---- Remove stale source trees that shadow the venv package ----
[ -d /opt/pymc_repeater/repeater ] && rm -rf /opt/pymc_repeater/repeater
# ---- Remove old system-level packages to avoid confusion ----
python3 -m pip uninstall -y pymc_repeater 2>/dev/null || true
python3 -m pip uninstall -y pymc_core 2>/dev/null || true
# ---- Install into the venv ----
exec "$VENV_PIP" install \
--upgrade \
--no-cache-dir \
--force-reinstall \
--ignore-installed \
"pymc_repeater[hardware] @ git+https://github.com/rightup/pyMC_Repeater.git@${CHANNEL}"
UPGRADEEOF
chmod 0755 /usr/local/bin/pymc-do-upgrade
@@ -735,9 +827,6 @@ UPGRADEEOF
SCRIPT_DIR="$(dirname "$0")"
cd "$SCRIPT_DIR"
# Suppress pip root user warnings
export PIP_ROOT_USER_ACTION=ignore
# Calculate version from git for setuptools_scm
if [ -d .git ]; then
git fetch --tags 2>/dev/null || true
@@ -753,13 +842,12 @@ UPGRADEEOF
echo "Note: Using optimized binary wheels for faster installation"
echo ""
# Remove old pymc_core first so no stale .py/.pyc files linger
python3 -m pip uninstall -y pymc_core 2>/dev/null || true
# Migrate from system pip to venv (idempotent)
migrate_to_venv
# Install with --force-reinstall to ensure fresh pymc_core from GitHub
# --ignore-installed avoids failures on system-managed packages (e.g. PyYAML)
echo "Upgrading pymc_repeater with fresh dependencies from pyproject.toml..."
if python3 -m pip install --break-system-packages --no-cache-dir --force-reinstall --ignore-installed .[hardware]; then
# Install into the venv (clean, no system-packages flags needed)
echo "Upgrading pymc_repeater into venv ($VENV_DIR)..."
if "$VENV_PIP" install --upgrade --no-cache-dir .[hardware]; then
echo ""
echo "✓ Package and dependencies upgraded successfully!"
else
pymc-repeater.service
+2 -2
View File
@@ -12,8 +12,8 @@ User=repeater
Group=repeater
WorkingDirectory=/var/lib/pymc_repeater
# Start command - use python module directly with proper path
ExecStart=/usr/bin/python3 -m repeater.main --config /etc/pymc_repeater/config.yaml
# Start command - use venv python to avoid system package conflicts
ExecStart=/opt/pymc_repeater/venv/bin/python -m repeater.main --config /etc/pymc_repeater/config.yaml
# Restart on failure
Restart=on-failure
repeater/web/update_endpoints.py
+49 -8
View File
@@ -84,6 +84,13 @@ def _get_installed_version() -> str:
for p in sys.path:
if p and ("site-packages" in p or "dist-packages" in p) and p not in dirs:
dirs.append(p)
# Explicitly include the dedicated venv's site-packages
_venv_site = "/opt/pymc_repeater/venv/lib"
if os.path.isdir(_venv_site):
for child in os.listdir(_venv_site):
sp = os.path.join(_venv_site, child, "site-packages")
if os.path.isdir(sp) and sp not in dirs:
dirs.append(sp)
# -- 2. Scan for dist-info METADATA files ------------------------------ #
pkg_glob = PACKAGE_NAME.replace("-", "_") + "-*.dist-info"
@@ -470,6 +477,13 @@ def _cleanup_stale_dist_info() -> None:
dirs.append(_site.getusersitepackages())
except AttributeError:
pass
# Also scan the dedicated venv's site-packages
_venv_site = "/opt/pymc_repeater/venv/lib"
if os.path.isdir(_venv_site):
for child in os.listdir(_venv_site):
sp = os.path.join(_venv_site, child, "site-packages")
if os.path.isdir(sp) and sp not in dirs:
dirs.append(sp)
pkg_glob = PACKAGE_NAME.replace("-", "_") + "-*.dist-info"
@@ -655,10 +669,12 @@ def _do_check() -> None:
def _migrate_service_unit() -> None:
"""Strip legacy PYTHONPATH and fix WorkingDirectory in the systemd service unit.
"""Strip legacy PYTHONPATH, fix WorkingDirectory, and ensure ExecStart
uses the venv python in the systemd service unit.
"""
import subprocess as _sp
_SVC_UNIT = "/etc/systemd/system/pymc-repeater.service"
_VENV_PYTHON = "/opt/pymc_repeater/venv/bin/python"
try:
_sp.run(["sed", "-i", "/^Environment=.*PYTHONPATH/d", _SVC_UNIT], check=False)
_sp.run(
@@ -667,6 +683,12 @@ def _migrate_service_unit() -> None:
_SVC_UNIT],
check=False,
)
_sp.run(
["sed", "-i",
f"s|ExecStart=/usr/bin/python3|ExecStart={_VENV_PYTHON}|",
_SVC_UNIT],
check=False,
)
_sp.run(["systemctl", "daemon-reload"], check=False)
logger.info("[Update] Service unit migration applied (root path).")
except Exception as exc:
@@ -702,9 +724,12 @@ def _do_install() -> None:
import os as _os
env = _os.environ.copy()
env["PIP_ROOT_USER_ACTION"] = "ignore"
env["SETUPTOOLS_SCM_PRETEND_VERSION"] = _state.latest_version or "1.0.0"
_VENV_DIR = "/opt/pymc_repeater/venv"
_VENV_PIP = os.path.join(_VENV_DIR, "bin", "pip")
_VENV_PYTHON = os.path.join(_VENV_DIR, "bin", "python")
_state.append_line(f"[pyMC updater] Installing from channel '{channel}'…")
_UPGRADE_WRAPPER = "/usr/local/bin/pymc-do-upgrade"
@@ -712,22 +737,38 @@ def _do_install() -> None:
if is_root:
_migrate_service_unit()
# Ensure venv exists (migration from system-pip era)
if not os.path.isfile(_VENV_PYTHON):
_state.append_line("[pyMC updater] Creating venv (first-time migration)…")
_run(["python3", "-m", "venv", "--system-site-packages", _VENV_DIR], env=env)
_run([_VENV_PIP, "install", "--upgrade", "pip", "setuptools", "wheel"], env=env)
# Clean up system-level packages to avoid shadowing
_run(["python3", "-m", "pip", "uninstall", "-y", "pymc_repeater"], env=env)
_run(["python3", "-m", "pip", "uninstall", "-y", "pymc_core"], env=env)
# Remove stale source tree that could shadow the venv package
stale_src = "/opt/pymc_repeater/repeater"
if os.path.isdir(stale_src):
_state.append_line("[pyMC updater] Removing stale source tree…")
import shutil
shutil.rmtree(stale_src, ignore_errors=True)
install_spec = (
f"pymc_repeater[hardware] @ git+https://github.com/{GITHUB_OWNER}/{GITHUB_REPO}.git@{channel}"
)
_state.append_line(f"[pyMC updater] Running as root – direct pip install")
_state.append_line(f"[pyMC updater] Running as root – venv pip install")
_state.append_line(f"[pyMC updater] Target: {install_spec}")
cmd = [
"python3", "-m", "pip", "install",
"--break-system-packages",
_VENV_PIP, "install",
"--upgrade",
"--no-cache-dir",
"--force-reinstall",
install_spec,
]
elif _os.path.isfile(_UPGRADE_WRAPPER):
_state.append_line(f"[pyMC updater] Using sudo wrapper: {_UPGRADE_WRAPPER}")
# Pass the target version as $2 so the wrapper can set
# SETUPTOOLS_SCM_PRETEND_VERSION (sudo strips our env).
# The wrapper handles venv creation/migration internally
cmd = ["sudo", _UPGRADE_WRAPPER, channel, _state.latest_version or ""]
else:
msg = (