One container, one yaml file

.dockerignore

@@ -0,0 +1,15 @@
+# Your machine's business, and it is mounted at runtime rather than baked in.
+repos.yml
+.env
+
+_book/
+docs/
+.git/
+.github/
+.collector-logs/
+**/.venv/
+**/.ruff_cache/
+**/.pytest_cache/
+**/__pycache__/
+**/_tests/
+**/.coverage

.env.example

@@ -1,9 +1,11 @@
-# Copy to .env, or let scripts/up.sh generate it from `gh auth token`.
+# Only needed if you start the board with `docker compose up` rather than
+# `docker run`; with docker run these are plain -e flags.
 #
 # Which repos are monitored is NOT set here - that lives in repos.yml, one
-# entry per checkout. This file holds credentials and cadences only.
+# entry per repo. This file holds credentials and cadences only.
 
-# Needs `repo` (private repo metadata) and `read:org`. A gh OAuth token works.
+# Needs `repo` (private repo metadata) and `read:org`. A gh OAuth token works:
+#   GITHUB_TOKEN=$(gh auth token)
 # It has to be able to read every repo listed in repos.yml.
 GITHUB_TOKEN=
 

.github/workflows/ci.yml

@@ -81,43 +81,81 @@ jobs:
       - name: Panel ids, link targets and grid overlaps
         run: python3 scripts/check-dashboard.py
 
-  compose:
-    name: compose and scripts
+  image:
+    name: image builds and comes up
     runs-on: ubuntu-latest
-    # GITHUB_TOKEN is a required variable in the compose file; any value
-    # satisfies interpolation.
-    env:
-      GITHUB_TOKEN: dummy-value-for-interpolation-only
     steps:
       - uses: actions/checkout@v4
 
-      - name: Compose files parse
-        run: |
-          docker compose -f docker-compose.yml config --quiet
-          docker compose -f docker-compose.yml -f docker-compose.admin.yml config --quiet
+      - name: Build
+        run: docker build -t jq-monitoring:ci .
 
-      # The collector runs on the host now, so these are the entry points a
-      # user actually touches. A syntax error in one of them used to be caught
-      # by nothing at all.
+      # The entry points a user actually touches. A syntax error in either used
+      # to be caught by nothing at all - and now they are the container's PID 1
+      # and its only maintenance command, so a broken one is a board that does
+      # not start.
       - name: Shell scripts parse
-        run: for f in scripts/*.sh; do bash -n "$f"; done
-
-      # repos.yml is gitignored, so CI builds one the way a new user would and
-      # proves gen-repos.py emits the two lines the collector reads - including
-      # for a checkout that does NOT sit at <root>/<owner>/<name>, which is the
-      # case the paths line exists for. Nothing else in CI reads repos.yml, so
-      # a malformed one would otherwise only surface on somebody's laptop.
-      - name: The fleet resolves to an environment
+        run: for f in image/*.sh; do bash -n "$f"; done
+
+      - name: Compose file parses
+        env:
+          GITHUB_TOKEN: dummy-value-for-interpolation-only
+        run: docker compose -f docker-compose.yml config --quiet
+
+      # repos.yml is gitignored, so CI writes one the way a new user would.
+      # This is the whole install path in one step: a checkout that does NOT sit
+      # at <root>/<owner>/<name> - the case the paths in repos.yml exist for -
+      # plus a repo with no checkout at all, and the board has to come up
+      # serving metrics for both. Nothing else in CI reads repos.yml, so a
+      # regression here would otherwise only surface on somebody's laptop.
+      - name: It comes up and serves the fleet
+        run: |
+          mkdir -p home/nested/somewhere
+          git init -q -b main home/nested/somewhere/rhiza
+          git -C home/nested/somewhere/rhiza remote add origin \
+            https://github.com/Jebel-Quant/rhiza.git
+          cat > repos.yml <<'YML'
+          repos:
+            - path: ~/nested/somewhere/rhiza
+            - repo: Jebel-Quant/actions
+          YML
+
+          # No GITHUB_TOKEN on purpose: this asserts the container comes up and
+          # resolves the fleet off repos.yml, which needs no network. The
+          # GitHub half will fail (several endpoints 401 unauthenticated), so
+          # nothing below may depend on it having answered.
+          docker run -d --name fleet \
+            -p 127.0.0.1:9109:9109 \
+            -v "$PWD/repos.yml:/config/repos.yml:ro" \
+            -v "$PWD/home:/host:ro" \
+            jq-monitoring:ci
+
+          for _ in $(seq 1 60); do
+            curl -fsS http://localhost:9109/metrics -o metrics.txt && break
+            docker ps -q -f name=fleet | grep -q . || { docker logs fleet; exit 1; }
+            sleep 5
+          done
+
+          docker logs fleet > logs.txt 2>&1
+          cat logs.txt
+
+          # The awkward path was found: `cloned` is 1 only if the collector
+          # reached the checkout through the /host mount, which is the whole
+          # reason repos.yml carries paths rather than deriving them.
+          grep -q 'jq_repo_cloned{repo="Jebel-Quant/rhiza"} 1.0' metrics.txt
+
+          # Both entries parsed, one of them with no checkout. Asserted on the
+          # startup line rather than on a metric: a repo with no checkout
+          # reaches the board only through the GitHub half, which has no token
+          # here - so `repo="Jebel-Quant/actions"` is legitimately absent from
+          # /metrics and asserting on it would be testing GitHub, not this.
+          grep -q 'fleet: 2 repos, 1 with a checkout' logs.txt
+
+      # A refusal at startup, not a board that is quietly one repo short. This
+      # is the failure mode the whole config path is shaped to avoid.
+      - name: A broken repos.yml stops the container
         run: |
-          pip install --quiet pyyaml
-          printf 'repos:\n' > repos.yml
-          git init -q -b main nested/somewhere/rhiza
-          git -C nested/somewhere/rhiza remote add origin https://github.com/Jebel-Quant/rhiza.git
-          printf '  - path: nested/somewhere/rhiza\n' >> repos.yml
-          # A repo with no checkout: GitHub panels only, no path.
-          printf '  - repo: Jebel-Quant/actions\n' >> repos.yml
-
-          python3 scripts/gen-repos.py | tee env.out
-          grep -qx 'JQ_REPOS=Jebel-Quant/rhiza,Jebel-Quant/actions' env.out
-          grep -q 'JQ_REPO_PATHS=Jebel-Quant/rhiza=.*/nested/somewhere/rhiza$' env.out
-          test "$(grep -c 'Jebel-Quant/actions=' env.out)" = 0
+          printf 'repos:\n  - repo: no-slash\n' > broken.yml
+          docker run --rm -v "$PWD/broken.yml:/config/repos.yml:ro" \
+            jq-monitoring:ci 2>&1 | tee out.txt || true
+          grep -q 'no-slash' out.txt

.github/workflows/image.yml

@@ -0,0 +1,62 @@
+# Publishes the image the README tells people to run. Without this, the
+# `docker run ghcr.io/jebel-quant/monitoring` in every doc points at nothing.
+name: Image
+
+on:
+  push:
+    branches: [main]
+    # Prose cannot change the image. CI already builds it on every pull
+    # request, so this workflow only has to publish what main now holds.
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+      - 'mkdocs.yml'
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  # A second push while the first is still building would race to the same
+  # `latest` tag, and the loser would win.
+  group: image-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # Apple silicon and CI runners are not the same architecture, and the
+      # README is written for a laptop. Both or the recipe does not work.
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=semver,pattern={{version}}
+            type=sha,format=short
+
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.gitignore

@@ -2,13 +2,10 @@
 
 # Your fleet: it describes the folder layout of one machine, which has no
 # business in a public repo. Start from repos.example.yml. Nothing is generated
-# from it any more - scripts/collector.sh reads it at every launch.
+# from it - the collector reads it at startup, inside the container.
 repos.yml
 
-# Where the launchd agent's output goes.
-.collector-logs/
-
-# Left behind by running the collector, which is how it runs now.
+# Left behind by running the collector's tests locally.
 .venv/
 .ruff_cache/
 .pytest_cache/

Dockerfile

@@ -0,0 +1,76 @@
+# One image, one `docker run`. Prometheus, Grafana and the collector in a single
+# container, with the dashboard, the datasource, the alert rules and the scrape
+# config baked in - so nothing has to be cloned to run the board.
+#
+# The three used to be two containers plus a launchd agent on the host, because
+# the collector reads your working copies and a bind mount could only name a
+# checkout at /repos/<owner>/<name>. That constraint is gone: the home
+# directory is mounted once, whole, at /host, and repos.yml names paths inside
+# it. One mount expresses any layout.
+#
+#   docker build -t jq-monitoring .
+
+FROM prom/prometheus:v2.55.1 AS prometheus
+# The Ubuntu variant, not the default Alpine one: the runtime below is Debian,
+# and a musl-linked grafana will not start there.
+FROM grafana/grafana:11.3.1-ubuntu AS grafana
+
+FROM python:3.12-slim-bookworm
+
+# git for the working-copy half, curl for the healthcheck and purge-repo,
+# ca-certificates for api.github.com.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends git curl ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY --from=prometheus /bin/prometheus /bin/promtool /usr/local/bin/
+COPY --from=grafana /usr/share/grafana /usr/share/grafana
+COPY --from=grafana /etc/grafana /etc/grafana
+
+# The mounted checkouts belong to your host user, not to root inside here, and
+# git refuses to read a repository owned by someone else. Nothing here ever
+# writes - every call passes --no-optional-locks and the mount is read-only -
+# so the ownership check is protecting against nothing we do.
+RUN git config --system safe.directory '*'
+
+# Baked in, so there is no second copy on your disk to edit by mistake: the
+# only file you own is repos.yml.
+COPY prometheus/prometheus.yml /etc/prometheus/prometheus.yml
+COPY grafana/provisioning /etc/grafana/provisioning
+COPY grafana/dashboards /etc/grafana/dashboards
+
+COPY collector /src/collector
+RUN pip install --no-cache-dir /src/collector && rm -rf /src
+
+COPY image/entrypoint.sh /usr/local/bin/entrypoint
+COPY image/purge-repo.sh /usr/local/bin/purge-repo
+RUN chmod +x /usr/local/bin/entrypoint /usr/local/bin/purge-repo
+
+ENV GF_PATHS_HOME=/usr/share/grafana \
+    GF_PATHS_CONFIG=/etc/grafana/grafana.ini \
+    GF_PATHS_PROVISIONING=/etc/grafana/provisioning \
+    GF_PATHS_DATA=/data/grafana \
+    GF_PATHS_LOGS=/data/grafana/log \
+    GF_PATHS_PLUGINS=/data/grafana/plugins \
+    GF_AUTH_ANONYMOUS_ENABLED=true \
+    GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer \
+    GF_USERS_DEFAULT_THEME=dark \
+    GF_ANALYTICS_REPORTING_ENABLED=false \
+    GF_ANALYTICS_CHECK_FOR_UPDATES=false \
+    GF_PLUGINS_PREINSTALL_DISABLED=true
+# admin/admin is Grafana's own default and is deliberately not restated here:
+# the board opens without signing in and the password is only for settings.
+# Override with -e GF_SECURITY_ADMIN_PASSWORD=... if you expose port 3000.
+ENV JQ_REPOS_FILE=/config/repos.yml \
+    JQ_HOST_ROOT=/host \
+    PROM_RETENTION=180d
+
+# /config is yours (repos.yml, read-only); /host is your home directory,
+# read-only; /data is the history and Grafana's own database.
+VOLUME /data
+EXPOSE 3000 9090 9109
+
+HEALTHCHECK --interval=30s --start-period=60s --retries=3 \
+  CMD curl -fsS http://localhost:3000/api/health >/dev/null || exit 1
+
+ENTRYPOINT ["/usr/local/bin/entrypoint"]