diff --git a/.gitignore b/.gitignore index 90d48c2..6d98e9b 100644 --- a/.gitignore +++ b/.gitignore @@ -224,4 +224,5 @@ meshcore.db # Frontend build artifacts node_modules/ src/meshcore_hub/web/static/vendor/ +src/meshcore_hub/web/static/dist/ src/meshcore_hub/web/static/css/tailwind.css diff --git a/Dockerfile b/Dockerfile index 1e36849..b978956 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,7 @@ COPY alembic.ini ./ # Overlay built frontend assets onto source tree COPY --from=frontend /app/src/meshcore_hub/web/static/vendor ./src/meshcore_hub/web/static/vendor COPY --from=frontend /app/src/meshcore_hub/web/static/css/tailwind.css ./src/meshcore_hub/web/static/css/tailwind.css +COPY --from=frontend /app/src/meshcore_hub/web/static/dist ./src/meshcore_hub/web/static/dist # Build argument for version (set via CI or manually) ARG BUILD_VERSION=dev diff --git a/build.js b/build.js index fbfa273..0b72bab 100644 --- a/build.js +++ b/build.js @@ -1,9 +1,18 @@ import { execSync } from "node:child_process"; -import { cpSync, mkdirSync, existsSync } from "node:fs"; +import { + cpSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from "node:fs"; +import { createHash } from "node:crypto"; import { join } from "node:path"; const STATIC = join("src", "meshcore_hub", "web", "static"); const VENDOR = join(STATIC, "vendor"); +const DIST = join(STATIC, "dist"); function vendor(pkg, files, dest) { const out = join(VENDOR, dest); @@ -26,25 +35,6 @@ execSync( console.log("Copying vendor files..."); -vendor("lit-html", ["lit-html.js", "lit-html.js.map"], "lit-html"); -vendor( - "lit-html", - [ - "directive.js", - "directive.js.map", - "directive-helpers.js", - "directive-helpers.js.map", - "async-directive.js", - "async-directive.js.map", - ], - "lit-html", -); -vendor( - "lit-html", - ["directives/unsafe-html.js", "directives/unsafe-html.js.map"], - "lit-html/directives", -); - vendor("leaflet", ["dist/leaflet.css", "dist/leaflet.js", "dist/leaflet.js.map"], "leaflet"); mkdirSync(join(VENDOR, "leaflet", "images"), { recursive: true }); cpSync( @@ -56,4 +46,72 @@ cpSync( vendor("chart.js", ["dist/chart.umd.min.js"], "chart.js"); vendor("qrcodejs", ["qrcode.min.js"], "qrcodejs"); +console.log("Bundling SPA with esbuild..."); +mkdirSync(DIST, { recursive: true }); + +const metafilePath = join(DIST, "meta.json"); +execSync( + `npx esbuild ${join(STATIC, "js", "spa", "app.js")}` + + ` --bundle --format=esm --splitting --minify` + + ` --outdir=${DIST}` + + ` --entry-names=[name].[hash]` + + ` --chunk-names=chunks/[name].[hash]` + + ` --metafile=${metafilePath}`, + { stdio: "inherit" }, +); + +console.log("Generating assets manifest..."); + +const meta = JSON.parse(readFileSync(metafilePath, "utf-8")); +const assets = {}; + +for (const [outputPath, info] of Object.entries(meta.outputs)) { + if (!info.entryPoint) continue; + const entryName = info.entryPoint.split("/").pop().replace(/\.js$/, ".js"); + const fileName = outputPath.split("/").pop(); + assets[entryName] = fileName; +} + +const vendorFiles = { + "leaflet.css": join(VENDOR, "leaflet", "leaflet.css"), + "leaflet.js": join(VENDOR, "leaflet", "leaflet.js"), + "chart.umd.min.js": join(VENDOR, "chart.js", "chart.umd.min.js"), + "qrcode.min.js": join(VENDOR, "qrcodejs", "qrcode.min.js"), +}; + +const vendorHashes = {}; +for (const [name, path] of Object.entries(vendorFiles)) { + if (existsSync(path)) { + const hash = createHash("sha256") + .update(readFileSync(path)) + .digest("hex") + .slice(0, 8); + vendorHashes[name] = hash; + } +} + +const localesDir = join(STATIC, "locales"); +let localeContent = ""; +if (existsSync(localesDir)) { + const localeFiles = readdirSync(localesDir) + .filter((f) => f.endsWith(".json")) + .sort(); + for (const f of localeFiles) { + localeContent += readFileSync(join(localesDir, f), "utf-8"); + } +} +const localeVersion = + localeContent.length > 0 + ? createHash("sha256").update(localeContent).digest("hex").slice(0, 8) + : ""; + +const manifest = { + ...assets, + vendor: vendorHashes, + locale_version: localeVersion, +}; + +writeFileSync(join(DIST, "assets.json"), JSON.stringify(manifest, null, 2)); +console.log(" Manifest:", JSON.stringify(manifest, null, 2)); + console.log("Done."); diff --git a/docs/plans/20260505-1735-caching-bundling/plan.md b/docs/plans/20260505-1735-caching-bundling/plan.md new file mode 100644 index 0000000..173c4de --- /dev/null +++ b/docs/plans/20260505-1735-caching-bundling/plan.md @@ -0,0 +1,288 @@ +# Plan: esbuild Bundling + Cache Busting for Static Assets + +**Date:** 2025-05-05 +**Status:** Draft + +## Problem + +Only 4 of ~30+ static assets have cache-busting query parameters. ES module sub-imports (`components.js`, `router.js`, `i18n.js`, etc.) and all vendor libraries are cached at most 1 hour with no invalidation mechanism. The `?v={{ version }}` on `app.js` is the only entry into the ES module graph, but browsers strip query parameters when resolving relative imports -- so the entire module tree loads unversioned. + +### Current state + +| Asset | Cache busting | Cache-Control | +|-------|--------------|---------------| +| `tailwind.css` | `?v={{ version }}` | `immutable` (1 year) | +| `app.css` | `?v={{ version }}` | `immutable` (1 year) | +| `charts.js` | `?v={{ version }}` | `immutable` (1 year) | +| `app.js` (SPA entry) | `?v={{ version }}` | `immutable` (1 year) | +| All other SPA modules (~14 files) | None | 1 hour | +| All page modules (11 files) | None | 1 hour | +| lit-html (via import map) | None | 1 hour | +| Leaflet CSS/JS | None | 1 hour | +| Chart.js, QRCode.js | None | 1 hour | +| Locale JSON files | None | 1 hour | +| Static images (logo, meshcore) | None | 1 hour | + +## Solution + +Use **esbuild** to bundle and minify the SPA JavaScript, generating content-hashed filenames for automatic cache invalidation. Add `?v=` cache busting to vendor libs that remain as global ` + + + + ``` + +3. **Add** `?v=` cache busting to vendor ` + + + + + + + ``` + +4. `charts.js` remains unchanged (it already has `?v={{ version }}`, which is fine since it changes with releases). + +5. **Graceful fallback**: If `asset_app_js` is empty (no `dist/` directory), fall back to the original `app.js?v={{ version }}` path. This ensures source installs without `npm run build` still work. + +### Step 4: Update i18n.js locale fetching + +**File:** `src/meshcore_hub/web/static/js/spa/i18n.js` + +The locale JSON fetch at line 23 needs a cache-busting parameter. Since this file is now bundled by esbuild, the version string must come from the embedded `window.__APP_CONFIG__`: + +```javascript +// Before +const res = await fetch(`/static/locales/${locale}.json`); + +// After +const config = window.__APP_CONFIG__ || {}; +const v = config.locale_version || ''; +const res = await fetch(`/static/locales/${locale}.json${v ? '?v=' + v : ''}`); +``` + +The `locale_version` value is set by `build.js` (content hash of all locale files combined) and passed through the Python config JSON. + +### Step 5: Update static image references + +**File:** `src/meshcore_hub/web/static/js/spa/pages/home.js` + +Two hardcoded image references without cache busting: +- Line ~168: `'/static/img/logo.svg'` +- Line ~221: `"/static/img/meshcore.svg"` + +These are embedded in lit-html templates and rarely change. Options: +- **(A)** Include image hashes in the manifest and reference them via `window.__APP_CONFIG__` -- adds complexity for minimal benefit. +- **(B)** Leave as-is with the existing 1-hour cache -- images rarely change and are small. + +**Recommendation:** Option B. Images are small SVGs that rarely change. The 1-hour cache is acceptable. + +### Step 6: Update Dockerfile + +**File:** `Dockerfile` + +Add a `COPY --from=frontend` line to overlay the `dist/` directory: + +```dockerfile +# Overlay built frontend assets onto source tree +COPY --from=frontend /app/src/meshcore_hub/web/static/vendor ./src/meshcore_hub/web/static/vendor +COPY --from=frontend /app/src/meshcore_hub/web/static/css/tailwind.css ./src/meshcore_hub/web/static/css/tailwind.css +COPY --from=frontend /app/src/meshcore_hub/web/static/dist ./src/meshcore_hub/web/static/dist +``` + +The `dist/` directory is generated by `npm run build` in the frontend stage and includes: +- `app.[hash].js` (SPA entry point) +- `chunks/` directory (shared chunks, page modules) +- `assets.json` (manifest for Python to read) +- `meta.json` (esbuild metafile, optional -- could be excluded from Docker image) + +### Step 7: Remove vendored lit-html + +**Files to remove:** `src/meshcore_hub/web/static/vendor/lit-html/` (entire directory) + +This directory is already gitignored (line 226 of `.gitignore`). The lit-html vendor copy step in `build.js` (lines 29-46) is removed in Step 1. + +## Middleware impact + +**File:** `src/meshcore_hub/web/middleware.py` + +The `/static/dist/` directory serves content-hashed files (the filename itself is the cache-buster — when content changes, the filename changes). These need `immutable` caching: + +```python +# Static dist/ files use content-hashed filenames — immutable +elif path.startswith("/static/dist/"): + response.headers["cache-control"] = "public, max-age=31536000, immutable" +``` + +Add this rule **before** the generic `/static/` rule (line 50-51) so it takes priority. Placement: + +1. `/health` → `no-cache` *(unchanged)* +2. `/static/` + `v=` → `immutable` *(unchanged)* +3. **`/static/dist/` → `immutable`** *(new)* +4. `/static/` → `1-hour` *(unchanged)* + +The `?v=` params on vendor files (e.g., `/static/vendor/leaflet/leaflet.css?v=hash`) already match rule 2 and get `immutable`. No changes needed for the vendor path pattern. + +## Development workflow + +- `npm run build` is required after any JS change to see it reflected +- Source files in `js/spa/` remain the source of truth +- `dist/` is a build artifact (gitignored) +- For CSS-only changes, only the Tailwind build runs (esbuild step is fast enough that running it is harmless) + +## Files changed + +| File | Change | +|------|--------| +| `package.json` | Add `esbuild` to devDependencies | +| `build.js` | Add esbuild step, generate manifest, remove lit-html vendor copy | +| `.gitignore` | Add `src/meshcore_hub/web/static/dist/` | +| `Dockerfile` | Add `COPY --from=frontend` for `dist/` directory | +| `src/meshcore_hub/web/app.py` | Add manifest loader, pass asset paths to template | +| `src/meshcore_hub/web/templates/spa.html` | Remove import map, use hashed bundle path, add vendor `?v=` | +| `src/meshcore_hub/web/middleware.py` | Add `/static/dist/` immutable cache rule | +| `src/meshcore_hub/web/static/js/spa/i18n.js` | Add `?v=` to locale fetch URL | + +## Risks and mitigations + +| Risk | Mitigation | +|------|------------| +| Source install without `npm run build` breaks | Fallback in `spa.html`: if `asset_app_js` is empty, use original `app.js?v={{ version }}` path | +| esbuild `--splitting` requires HTTP/2 for optimal loading | Already the case with current native ES modules (multiple parallel requests) | +| Dynamic import paths change after bundling | esbuild handles this automatically -- `import('./pages/home.js')` becomes `import('./chunks/home.abc123.js')` in the output | +| lit-html import map removal breaks third-party extensions | Import map was only used internally. No external consumers. | +| Build step required for every JS change during development | Accepted trade-off. esbuild is <50ms. `npm run build` is already required for Tailwind. | + +## Future considerations + +- The `charts.js` file could be converted to an ES module and imported by the dashboard page module, eliminating the need for a separate ` + ``` + +2. **Update** vendor tags to add `?v=` cache busting: + - Line 44: `` + - Line 177: `` + - Line 180: `` + - Line 183: `` + +3. **Update** SPA entry point (line 210): + ```html + {% if asset_app_js %} + + {% else %} + + {% endif %} + ``` + +4. `charts.js` (line 186) stays unchanged — already has `?v={{ version }}`. + +### Task 3.2 — Update i18n.js locale fetching + +**File:** `src/meshcore_hub/web/static/js/spa/i18n.js` + +Update line 23: +```javascript +// Before +const res = await fetch(`/static/locales/${locale}.json`); + +// After +const config = window.__APP_CONFIG__ || {}; +const v = config.locale_version || ''; +const res = await fetch(`/static/locales/${locale}.json${v ? '?v=' + v : ''}`); +``` + +Note: `window.__APP_CONFIG__` is already set by the inline ` @@ -174,13 +164,13 @@ - + - + - + @@ -207,6 +197,10 @@ + {% if asset_app_js %} + + {% else %} + {% endif %} diff --git a/tests/test_web/test_advertisements.py b/tests/test_web/test_advertisements.py index 34458e3..b8ac12f 100644 --- a/tests/test_web/test_advertisements.py +++ b/tests/test_web/test_advertisements.py @@ -31,7 +31,9 @@ class TestAdvertisementsPage: def test_advertisements_contains_spa_script(self, client: TestClient) -> None: """Test that advertisements page includes SPA application script.""" response = client.get("/advertisements") - assert "/static/js/spa/app.js" in response.text + has_bundled = "/static/dist/" in response.text + has_fallback = "/static/js/spa/app.js" in response.text + assert has_bundled or has_fallback class TestAdvertisementsPageFilters: diff --git a/tests/test_web/test_caching.py b/tests/test_web/test_caching.py index a5d0429..11ac945 100644 --- a/tests/test_web/test_caching.py +++ b/tests/test_web/test_caching.py @@ -151,17 +151,24 @@ class TestVersionParameterInHTML: assert f"?v={__version__}" in charts_script["src"] def test_app_js_has_version(self, client): - """SPA app.js script should include version parameter.""" + """SPA app.js script should include version or content hash.""" response = client.get("/") assert response.status_code == 200 soup = BeautifulSoup(response.text, "html.parser") - app_script = soup.find( + bundled_script = soup.find( + "script", + {"src": lambda x: x and "/static/dist/" in x and x.endswith(".js")}, + ) + fallback_script = soup.find( "script", {"src": lambda x: x and "/static/js/spa/app.js" in x} ) - assert app_script is not None - assert f"?v={__version__}" in app_script["src"] + if bundled_script: + assert "/static/dist/" in bundled_script["src"] + else: + assert fallback_script is not None + assert f"?v={__version__}" in fallback_script["src"] def test_cdn_resources_unchanged(self, client): """CDN resources should not have version parameters.""" diff --git a/tests/test_web/test_home.py b/tests/test_web/test_home.py index 3d2363c..676c319 100644 --- a/tests/test_web/test_home.py +++ b/tests/test_web/test_home.py @@ -86,4 +86,6 @@ class TestHomePage: def test_home_contains_spa_app_script(self, client: TestClient) -> None: """Test that home page includes the SPA application script.""" response = client.get("/") - assert "/static/js/spa/app.js" in response.text + has_bundled = "/static/dist/" in response.text + has_fallback = "/static/js/spa/app.js" in response.text + assert has_bundled or has_fallback diff --git a/tests/test_web/test_messages.py b/tests/test_web/test_messages.py index 8914436..b6b7091 100644 --- a/tests/test_web/test_messages.py +++ b/tests/test_web/test_messages.py @@ -31,7 +31,9 @@ class TestMessagesPage: def test_messages_contains_spa_script(self, client: TestClient) -> None: """Test that messages page includes SPA application script.""" response = client.get("/messages") - assert "/static/js/spa/app.js" in response.text + has_bundled = "/static/dist/" in response.text + has_fallback = "/static/js/spa/app.js" in response.text + assert has_bundled or has_fallback class TestMessagesPageFilters: diff --git a/tests/test_web/test_nodes.py b/tests/test_web/test_nodes.py index 9c79491..1577d23 100644 --- a/tests/test_web/test_nodes.py +++ b/tests/test_web/test_nodes.py @@ -31,7 +31,9 @@ class TestNodesListPage: def test_nodes_contains_spa_script(self, client: TestClient) -> None: """Test that nodes page includes SPA application script.""" response = client.get("/nodes") - assert "/static/js/spa/app.js" in response.text + has_bundled = "/static/dist/" in response.text + has_fallback = "/static/js/spa/app.js" in response.text + assert has_bundled or has_fallback def test_nodes_with_search_param(self, client: TestClient) -> None: """Test nodes page with search parameter returns SPA shell."""