diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index c99841a..c6a7826 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -46,12 +46,12 @@ jobs: - name: Enable pnpm run: corepack enable pnpm - # actions/cache@v4 is intentionally omitted — probe (D-PROBE-04) showed it - # times out on this runner (socket hang-up between runner container and job - # container cache server). pnpm install without cache takes ~30s; acceptable. + # actions/cache@v4 intentionally omitted (D-PROBE-04). Installs now target the + # host-mounted pnpm store at /pnpm-store (see act_runner config.yaml container.options). + # Without the host mount the flag still works — pnpm creates an ephemeral store there. - name: Install dependencies - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile --store-dir /pnpm-store --prefer-offline - name: Lint run: pnpm lint @@ -104,10 +104,11 @@ jobs: - name: Enable pnpm run: corepack enable pnpm - # actions/cache@v4 intentionally omitted — same reasoning as fast-checks job (D-PROBE-04). + # actions/cache@v4 intentionally omitted (D-PROBE-04). Installs now target the + # host-mounted pnpm store at /pnpm-store (see act_runner config.yaml container.options). - name: Install dependencies - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile --store-dir /pnpm-store --prefer-offline # Pitfall 11: service container healthy != MariaDB accepting connections. # No mysql CLI in the runner image (D-PROBE-03); poll via the already-installed @@ -181,6 +182,9 @@ jobs: DB_USER: familysync DB_PASSWORD: testpass DB_NAME: familysync + # Persist Playwright browser binaries across runs via host-mounted /ms-playwright. + # Without the host mount CI still works — binaries are downloaded to the ephemeral dir. + PLAYWRIGHT_BROWSERS_PATH: /ms-playwright steps: - uses: actions/checkout@v4 @@ -191,10 +195,11 @@ jobs: - name: Enable pnpm run: corepack enable pnpm - # actions/cache@v4 intentionally omitted — same reasoning as fast-checks job (D-PROBE-04). + # actions/cache@v4 intentionally omitted (D-PROBE-04). Installs now target the + # host-mounted pnpm store at /pnpm-store (see act_runner config.yaml container.options). - name: Install dependencies - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile --store-dir /pnpm-store --prefer-offline # Pitfall 11: service container healthy != MariaDB accepting connections. # No mysql CLI in the runner image (D-PROBE-03); poll via the mysql2 driver @@ -270,7 +275,9 @@ jobs: # Install Playwright browsers with system deps BEFORE starting the API, so the long # browser download does not run during the API's lifetime. # Must run from apps/pwa/ where @playwright/test is installed (D-PROBE-05 confirmed exit 0). - # Do NOT cache browser binaries — Playwright explicitly recommends against it in CI. + # PLAYWRIGHT_BROWSERS_PATH=/ms-playwright (job-level env) persists binaries across runs via + # the host-mounted dir. The --with-deps apt step cannot be cached; baking a runner image + # with browsers preinstalled would also drop the --with-deps apt step (future optimization). - name: Install Playwright browsers run: npx playwright install --with-deps webkit chromium working-directory: apps/pwa @@ -462,7 +469,8 @@ jobs: --exit-code 1 # ── pnpm audit + outdated (code-change PRs only, D-12) ─────────────────── - # actions/cache@v4 intentionally omitted — same reasoning as fast-checks job (D-PROBE-04). + # actions/cache@v4 intentionally omitted (D-PROBE-04). Installs now target the + # host-mounted pnpm store at /pnpm-store (see act_runner config.yaml container.options). - uses: actions/setup-node@v4 if: needs.changes.outputs.code == 'true' @@ -475,7 +483,7 @@ jobs: - name: Install dependencies if: needs.changes.outputs.code == 'true' - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile --store-dir /pnpm-store --prefer-offline - name: Dependency audit (blocking on High+Critical) if: needs.changes.outputs.code == 'true' diff --git a/.gitea/workflows/publish.yml b/.gitea/workflows/publish.yml index fcc2bad..c27c999 100644 --- a/.gitea/workflows/publish.yml +++ b/.gitea/workflows/publish.yml @@ -88,6 +88,11 @@ jobs: # Build from REPO ROOT (T-08-10): the Dockerfile copies the pnpm workspace manifest + # lockfile from the root context; building from apps/api/ would fail to find them. - name: Build production image + # DOCKER_BUILDKIT=1 is required: the Dockerfile uses `RUN --mount=type=cache` + # (BuildKit) to persist the pnpm store across builds. The legacy builder would + # fail on that syntax. BuildKit is default on Docker 23+, set explicitly for safety. + env: + DOCKER_BUILDKIT: '1' run: | set -euo pipefail docker build --target production \ diff --git a/.planning/STATE.md b/.planning/STATE.md index b40c334..a14ab1a 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -257,6 +257,7 @@ Recent decisions affecting current work: | 260613-fp9 | `.gitea`/`.planning`-only pushes to main no longer trigger the Docker publish — added `paths-ignore: ['.gitea/**', '.planning/**']` under `on.push` in `.gitea/workflows/publish.yml` (skips only when EVERY changed file matches; mixed code+docs pushes still publish). `.dockerignore` already excludes `.planning` so the image is byte-identical. Done in isolated worktree (phase-10 agent held main tree). | 2026-06-13 | cd5a88c | | [260613-fp9-gitea-and-planning-pushes-should-not-tri](./quick/260613-fp9-gitea-and-planning-pushes-should-not-tri/) | | 260613-ndv | Isolate local apps/api integration tests to a dedicated `familysync_test` DB so test runs stop polluting the dev `familysync` DB. New CI-gated vitest globalSetup root-provisions (CREATE DATABASE + GRANT) + migrates + truncate-resets `familysync_test` each run; `vitest.config.ts` forces `DB_NAME=familysync_test` for local workers (no-op under CI, so CI's `familysync` service DB + db:migrate are untouched). Verified: dev `familysync` users stays 3 across a run, `familysync_test` resets (186→93, not doubled), 244/244 tests pass (flaky list_shares timeout gone), typecheck 0. Branch off main. | 2026-06-13 | 07d5161 | Verified | [260613-ndv-wire-apps-api-integration-tests-to-a-ded](./quick/260613-ndv-wire-apps-api-integration-tests-to-a-ded/) | | 260618-smr | Remove unused Redis service and all references — Redis confirmed unused at runtime (no ioredis/redis client import, no `REDIS_*` env, not a dependency in any package.json). Dropped the `redis` service from both compose files and cleaned all references in CLAUDE.md, README.md, and docs/* + e2e config. Kept the in-memory-vs-Redis design-rationale comments (D-12/D-18) in listEmitter/reminderScheduler/linkNonceStore/localAuth. `docker compose config` parses clean (0 redis); `format:check` green. Branch off main. | 2026-06-18 | 0b42666 | Verified | [260618-smr-remove-unused-redis-service-and-referenc](./quick/260618-smr-remove-unused-redis-service-and-referenc/) | +| 260618-tg2 | Persistent CI dependency caches — point all 4 CI `pnpm install` steps at a host-mounted `/pnpm-store` (`--store-dir /pnpm-store --prefer-offline`) and persist Playwright browsers via `PLAYWRIGHT_BROWSERS_PATH=/ms-playwright` on the harness job; added BuildKit `--mount=type=cache` to all 3 Dockerfile install stages + `DOCKER_BUILDKIT=1` on the publish build. Avoids `actions/cache` (D-PROBE-04 timeout). In-repo only — requires act_runner `config.yaml` `container.options` host mounts (manual host change). Verdaccio deferred. Branch off main. | 2026-06-18 | 6e93e24 | Verified | [260618-tg2-persistent-ci-dependency-caches-pnpm-sto](./quick/260618-tg2-persistent-ci-dependency-caches-pnpm-sto/) | ## Deferred Items diff --git a/.planning/quick/260618-tg2-persistent-ci-dependency-caches-pnpm-sto/260618-tg2-PLAN.md b/.planning/quick/260618-tg2-persistent-ci-dependency-caches-pnpm-sto/260618-tg2-PLAN.md new file mode 100644 index 0000000..e470402 --- /dev/null +++ b/.planning/quick/260618-tg2-persistent-ci-dependency-caches-pnpm-sto/260618-tg2-PLAN.md @@ -0,0 +1,99 @@ +--- +quick_id: 260618-tg2 +slug: persistent-ci-dependency-caches-pnpm-sto +description: Persistent CI dependency caches (pnpm store + Playwright browsers) +type: quick +created: 2026-06-19 +files_modified: + - .gitea/workflows/ci.yml + - docs/DEVELOPMENT.md + - apps/api/Dockerfile + - .gitea/workflows/publish.yml +--- + +# Quick Task 260618-tg2: Persistent CI dependency caches (pnpm store + Playwright) + +## Why + +The Gitea runner re-downloads all deps every run: 4 jobs each run `pnpm install --frozen-lockfile` +cold (lines 54/110/197/478), and the harness job re-downloads Playwright browser binaries every +run (line 275). The runner is long-lived Docker-on-Unraid, so persisting these via host bind-mounts +(`/pnpm-store`, `/ms-playwright`, wired in the act_runner `config.yaml` `container.options` — a +separate manual host change) eliminates the repeat downloads. This avoids `actions/cache@v4`, which +the Phase-8 runner probe found times out on this runner (D-PROBE-04). + +**Scope this task: the two CI caches only.** Verdaccio (registry mirror) and the Dockerfile +BuildKit cache mount are explicitly OUT of scope for now. + +## Tasks + +### Task 1 — pnpm store: point all CI installs at the persistent store +In `.gitea/workflows/ci.yml`, change each of the four install steps: +``` +run: pnpm install --frozen-lockfile +``` +→ +``` +run: pnpm install --frozen-lockfile --store-dir /pnpm-store --prefer-offline +``` +Lines 54 (fast-checks), 110 (api), 197 (harness), 478 (security). `--store-dir` and +`--prefer-offline` are valid pnpm 11.5.1 install flags (verify with `pnpm install --help`). +Do NOT add `store-dir` to a repo `.npmrc` — local dev has no `/pnpm-store`. +Update the stale comment near line 51 (the "no cache backend / ~30s acceptable" note) to reflect +that installs now use the persistent host-mounted store. +- verify: `grep -c -- '--store-dir /pnpm-store --prefer-offline' .gitea/workflows/ci.yml` → 4 +- done: all four installs use the persistent store; YAML still valid. + +### Task 2 — Playwright: persist browser binaries on the harness job +Add a job-level `env:` to the `harness:` job so every step (browser install + test run) resolves +the same persistent path: +```yaml + harness: + runs-on: ubuntu-latest + env: + PLAYWRIGHT_BROWSERS_PATH: /ms-playwright +``` +(If the harness job already has a job-level `env:` map, add the key to it rather than duplicating.) +Leave `npx playwright install --with-deps webkit chromium` (line 275) as-is — the binary download +is now cached by version; the `--with-deps` apt step can't persist (add a one-line comment noting +"baking a runner image with browsers preinstalled would also drop the --with-deps apt step" as a +future optimization). +- verify: `PLAYWRIGHT_BROWSERS_PATH: /ms-playwright` present under the harness job; the test-run + step (PLAYWRIGHT_BASE_URL ~line 340) inherits it. +- done: Playwright browsers persist across runs. + +### Task 3 — Document the host-mount dependency +Add a short subsection to `docs/DEVELOPMENT.md` (CI/runner area) noting: +- CI now uses persistent caches at container paths `/pnpm-store` and `/ms-playwright`. +- These require the act_runner `config.yaml` `container.options` to bind-mount host dirs to those + paths (host change, not in this repo). +- Without the mounts CI still works — it just falls back to uncached (writes to an ephemeral dir). +- done: the host-side requirement is discoverable from the repo. + +## Constraints +- Must pass local gates before each commit: `format:check` (prettier), eslint, YAML validity + (yq or actionlint if available), typecheck (no TS touched, but run if cheap). +- Job names and the required-check contexts (CI / fast-checks, CI / api, CI / harness, + CI / security, CI / gate) MUST stay identical so branch protection still matches. Do not rename + jobs or restructure the job graph. +- Atomic commits (Task 1, Task 2, Task 3 may be one or separate commits — keep changes coherent). + +### Task 4 — Dockerfile BuildKit pnpm-store cache (added mid-task per user request) +`apps/api/Dockerfile`: add `# syntax=docker/dockerfile:1` (line 1) and a BuildKit cache mount to +all three pnpm install stages (builder, pwa-builder, production): +`RUN --mount=type=cache,target=/pnpm-store,id=pnpm-store,sharing=locked pnpm install ... --store-dir /pnpm-store`. +`sharing=locked` because builder + pwa-builder run in parallel and would otherwise race the store. +`.gitea/workflows/publish.yml`: set `DOCKER_BUILDKIT: '1'` on the "Build production image" step so +the legacy builder can't break on the `--mount` syntax (BuildKit is default on Docker 23+; explicit +for safety). +- done: image build reuses a persistent BuildKit pnpm-store cache across builds. + +## OUT OF SCOPE (do not touch) +- Verdaccio / any `.npmrc` registry change (deferred — user will set up later). + +## must_haves +- truth: "All four ci.yml pnpm installs use --store-dir /pnpm-store --prefer-offline" +- truth: "The harness job sets PLAYWRIGHT_BROWSERS_PATH=/ms-playwright" +- truth: "ci.yml remains valid YAML with unchanged job names / required-check contexts" +- truth: "docs note the act_runner config.yaml host-mount requirement" +- artifacts: [.gitea/workflows/ci.yml, docs/DEVELOPMENT.md] diff --git a/.planning/quick/260618-tg2-persistent-ci-dependency-caches-pnpm-sto/260618-tg2-SUMMARY.md b/.planning/quick/260618-tg2-persistent-ci-dependency-caches-pnpm-sto/260618-tg2-SUMMARY.md new file mode 100644 index 0000000..2318747 --- /dev/null +++ b/.planning/quick/260618-tg2-persistent-ci-dependency-caches-pnpm-sto/260618-tg2-SUMMARY.md @@ -0,0 +1,121 @@ +--- +quick_id: 260618-tg2 +slug: persistent-ci-dependency-caches-pnpm-sto +phase: "20" +plan: tg2 +status: complete +completed: 2026-06-18 +tags: [ci, caching, pnpm, playwright] +key-files: + modified: + - .gitea/workflows/ci.yml + - docs/DEVELOPMENT.md + - apps/api/Dockerfile + - .gitea/workflows/publish.yml +decisions: + - All four CI pnpm installs now target /pnpm-store via --store-dir --prefer-offline flags + - PLAYWRIGHT_BROWSERS_PATH added at harness job level (not per-step) so both install and run steps share the same path + - D-PROBE-04 comments updated to reflect the new store strategy rather than "no cache" +--- + +# Quick Task 260618-tg2: Persistent CI dependency caches (pnpm store + Playwright) Summary + +**One-liner:** Point all four CI pnpm installs at `/pnpm-store` and harness Playwright at `/ms-playwright` via host-mounted directories on the act_runner. + +## What Was Done + +### Task 1 — pnpm store (ci.yml, 4 install lines) + +Changed all four `pnpm install --frozen-lockfile` lines to +`pnpm install --frozen-lockfile --store-dir /pnpm-store --prefer-offline`: + +- Line ~54: `fast-checks` job +- Line ~110: `api` job +- Line ~197: `harness` job +- Line ~485: `security` job (conditional) + +Both flags confirmed valid against `pnpm 11.5.1 install --help` before use. + +Updated the stale D-PROBE-04 comment in each job from "no cache backend / ~30s acceptable" +to reflect that installs now target the host-mounted store. + +### Task 2 — Playwright browsers (harness job env) + +Added `PLAYWRIGHT_BROWSERS_PATH: /ms-playwright` to the existing job-level `env:` block on +the `harness:` job (alongside the DB_* creds). This means both the `Install Playwright browsers` +step and the `Run harness` step inherit the same path, so cached binaries are found at install +time and used at test time. + +Added a comment on the Playwright install step noting the future optimization: baking a runner +image with browsers preinstalled would also eliminate the `--with-deps` apt step. + +### Task 3 — Host-mount documentation (docs/DEVELOPMENT.md) + +Added a "CI dependency caches" subsection under the CI Pipeline Overview. Documents: +- The two container paths (`/pnpm-store`, `/ms-playwright`) with a reference table +- That the act\_runner `config.yaml` `container.options` bind-mount is a **host-side** change +- That CI still works without the mounts (ephemeral fallback — just no caching) + +## Verification + +``` +grep -c -- '--store-dir /pnpm-store --prefer-offline' .gitea/workflows/ci.yml +→ 4 + +grep -n 'PLAYWRIGHT_BROWSERS_PATH' .gitea/workflows/ci.yml +→ 187: PLAYWRIGHT_BROWSERS_PATH: /ms-playwright (job-level env) +→ 278: # PLAYWRIGHT_BROWSERS_PATH=/ms-playwright... (comment) + +python3 -c 'import yaml,sys; yaml.safe_load(open(".gitea/workflows/ci.yml")); print("YAML valid")' +→ YAML valid + +pnpm format:check → All matched files use Prettier code style! +pnpm md:lint → Summary: 0 error(s) +``` + +### Task 4 — Dockerfile BuildKit pnpm-store cache (added mid-task by user request) + +Added by the orchestrator after the initial 3 tasks, when the user asked to include the Dockerfile: + +- `apps/api/Dockerfile`: added `# syntax=docker/dockerfile:1` (line 1) and a BuildKit cache mount + (`RUN --mount=type=cache,target=/pnpm-store,id=pnpm-store,sharing=locked ... --store-dir /pnpm-store`) + to all three pnpm install stages (builder, pwa-builder, production). `sharing=locked` because + builder + pwa-builder run in parallel and would otherwise race the shared store. +- `.gitea/workflows/publish.yml`: set `DOCKER_BUILDKIT: '1'` on the "Build production image" step — + the publish path uses plain `docker build` (not buildx), and the legacy builder would fail on the + `--mount` syntax. BuildKit is default on Docker 23+; set explicitly for safety. +- Verified: `format:check` clean (Dockerfile is outside prettier's scope), publish.yml valid YAML. + +## Commits + +| Hash | Message | +| --- | --- | +| `80b2038` | chore(20): persistent CI caches — pnpm store + Playwright browsers | +| `f83d423` | docs(20): document CI persistent cache host-mount dependency | +| `6e93e24` | chore(260618-tg2): BuildKit pnpm-store cache mount in Dockerfile build | + +## Deviations from Plan + +Dockerfile cache (Task 4) was added mid-task at the user's request after the initial 3-task plan +(it had been explicitly deferred/out-of-scope). The publish workflow's `DOCKER_BUILDKIT=1` was a +required companion change so the `--mount` syntax doesn't break the legacy builder. + +- `--store-dir` flag form matches plan exactly (plan said verify against pnpm; verified: valid) +- harness job already had a job-level `env:` map; `PLAYWRIGHT_BROWSERS_PATH` was added to it as instructed +- stale comment text updated as instructed + +## Known Stubs + +None. + +## Threat Flags + +None — YAML-only and doc-only changes; no new network endpoints, auth paths, or trust boundaries introduced. + +## Self-Check: PASSED + +- `.gitea/workflows/ci.yml` — modified and committed at 80b2038 +- `docs/DEVELOPMENT.md` — modified and committed at f83d423 +- YAML validity confirmed by python3 yaml.safe_load +- 4 install lines confirmed by grep -c +- PLAYWRIGHT_BROWSERS_PATH confirmed at job-level env line 187 diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 09418ff..117ca98 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # Built from the REPO ROOT context (see docker-compose.yml: build.context: .) # so the pnpm workspace manifest + lockfile are available for a deterministic, # workspace-aware install. apps/api is one package in the pnpm workspace. @@ -12,7 +13,8 @@ FROM base AS builder COPY pnpm-workspace.yaml pnpm-lock.yaml package.json ./ COPY apps/api/package.json ./apps/api/ COPY apps/pwa/package.json ./apps/pwa/ -RUN pnpm install --frozen-lockfile --filter @familysync/api... +RUN --mount=type=cache,target=/pnpm-store,id=pnpm-store,sharing=locked \ + pnpm install --frozen-lockfile --filter @familysync/api... --store-dir /pnpm-store COPY apps/api ./apps/api RUN pnpm --filter @familysync/api build @@ -28,7 +30,8 @@ FROM base AS pwa-builder COPY pnpm-workspace.yaml pnpm-lock.yaml package.json ./ COPY apps/api/package.json ./apps/api/ COPY apps/pwa/package.json ./apps/pwa/ -RUN pnpm install --frozen-lockfile --filter @familysync/pwa... +RUN --mount=type=cache,target=/pnpm-store,id=pnpm-store,sharing=locked \ + pnpm install --frozen-lockfile --filter @familysync/pwa... --store-dir /pnpm-store COPY apps/pwa ./apps/pwa RUN pnpm --filter @familysync/pwa build @@ -36,7 +39,8 @@ FROM base AS production COPY pnpm-workspace.yaml pnpm-lock.yaml package.json ./ COPY apps/api/package.json ./apps/api/ COPY apps/pwa/package.json ./apps/pwa/ -RUN pnpm install --frozen-lockfile --prod --filter @familysync/api... +RUN --mount=type=cache,target=/pnpm-store,id=pnpm-store,sharing=locked \ + pnpm install --frozen-lockfile --prod --filter @familysync/api... --store-dir /pnpm-store COPY --from=builder /app/apps/api/dist ./apps/api/dist WORKDIR /app/apps/api # Enforce production identity — engages the NODE_ENV=production hard guard diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 9198982..0fc712e 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -205,6 +205,21 @@ Every PR to `main` runs through `.gitea/workflows/ci.yml`. A `changes` path-filt All five jobs must pass (or be legitimately skipped) before a PR can merge. See [docs/TESTING.md](TESTING.md) for test suite details. +### CI dependency caches + +CI uses two persistent cache paths inside job containers: + +| Path | Content | +| ---------------- | ----------------------------------------------------------------------- | +| `/pnpm-store` | pnpm content-addressable store (`--store-dir /pnpm-store`) | +| `/ms-playwright` | Playwright browser binaries (`PLAYWRIGHT_BROWSERS_PATH=/ms-playwright`) | + +These paths must be bind-mounted from host directories in the act_runner `config.yaml` +`container.options` field — that is a **host-side change, not tracked in this repo**. Without the +mounts, CI still works correctly — pnpm creates an ephemeral store at `/pnpm-store` inside the +container and Playwright downloads browsers fresh each run. The mounts only eliminate repeat +downloads across runs. + ## Drizzle Migration Workflow Schema changes follow a strict two-step process. **`drizzle-kit push` is not available** — it has been removed from the scripts because it emits a false destructive diff (table truncation) on populated MariaDB databases.