diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c9e835..0277d24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,35 +65,42 @@ jobs: run: python3 scripts/check-dashboard.py compose: - name: compose files parse + name: compose and scripts runs-on: ubuntu-latest - # The overlays only patch services, so they are not valid on their own - - # they are checked the way they are actually used. GITHUB_TOKEN is a - # required variable; any value satisfies interpolation. + # GITHUB_TOKEN is a required variable in the compose file; any value + # satisfies interpolation. env: GITHUB_TOKEN: dummy-value-for-interpolation-only steps: - uses: actions/checkout@v4 + - name: Compose files parse + run: | + docker compose -f docker-compose.yml config --quiet + docker compose -f docker-compose.yml -f docker-compose.admin.yml config --quiet + + # The collector runs on the host now, so these are the entry points a + # user actually touches. A syntax error in one of them used to be caught + # by nothing at all. + - name: Shell scripts parse + run: for f in scripts/*.sh; do bash -n "$f"; done + # repos.yml is gitignored, so CI builds one the way a new user would and - # proves gen-repos.py emits an override that actually merges. Nothing - # else in CI reads it, so a malformed one would otherwise only surface - # on somebody's laptop. - - name: Generate the fleet override + # proves gen-repos.py emits the two lines the collector reads - including + # for a checkout that does NOT sit at //, which is the + # case the paths line exists for. Nothing else in CI reads repos.yml, so + # a malformed one would otherwise only surface on somebody's laptop. + - name: The fleet resolves to an environment run: | pip install --quiet pyyaml printf 'repos:\n' > repos.yml - for full in Jebel-Quant/rhiza cvxgrp/cvxsimulator; do - git init -q -b main "ci-checkouts/$full" - git -C "ci-checkouts/$full" remote add origin "https://github.com/$full.git" - printf ' - path: ci-checkouts/%s\n' "$full" >> repos.yml - done - # A repo with no checkout: GitHub panels only, no bind mount. + git init -q -b main nested/somewhere/rhiza + git -C nested/somewhere/rhiza remote add origin https://github.com/Jebel-Quant/rhiza.git + printf ' - path: nested/somewhere/rhiza\n' >> repos.yml + # A repo with no checkout: GitHub panels only, no path. printf ' - repo: Jebel-Quant/actions\n' >> repos.yml - python3 scripts/gen-repos.py - cat docker-compose.repos.yml - - name: Base + fleet override - run: docker compose -f docker-compose.yml -f docker-compose.repos.yml config --quiet - - name: Base + admin overlay - run: docker compose -f docker-compose.yml -f docker-compose.repos.yml -f docker-compose.admin.yml config --quiet + python3 scripts/gen-repos.py | tee env.out + grep -qx 'JQ_REPOS=Jebel-Quant/rhiza,Jebel-Quant/actions' env.out + grep -q 'JQ_REPO_PATHS=Jebel-Quant/rhiza=.*/nested/somewhere/rhiza$' env.out + test "$(grep -c 'Jebel-Quant/actions=' env.out)" = 0 diff --git a/.gitignore b/.gitignore index d6ea74d..eba6d5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,14 @@ .env -# Your fleet and the mounts generated from it: both describe the folder layout -# of one machine, which has no business in a public repo. Start from -# repos.example.yml. +# Your fleet: it describes the folder layout of one machine, which has no +# business in a public repo. Start from repos.example.yml. Nothing is generated +# from it any more - scripts/collector.sh reads it at every launch. repos.yml -docker-compose.repos.yml -# Left behind by running the collector natively instead of in its container. +# Where the launchd agent's output goes. +.collector-logs/ + +# Left behind by running the collector, which is how it runs now. .venv/ .ruff_cache/ .pytest_cache/ diff --git a/README.md b/README.md index ee468df..ff3373e 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,10 @@ the board because you listed it in `repos.yml`, and for no other reason. ## What you need -Docker, and the [`gh` CLI](https://cli.github.com) signed in (`up.sh` mints the -token from it — otherwise put a `GITHUB_TOKEN` in `.env` yourself). +Docker (for Prometheus and Grafana), [`uv`](https://docs.astral.sh/uv/) (for the +collector, which runs on your machine rather than in a container), and the +[`gh` CLI](https://cli.github.com) signed in — `up.sh` mints the token from it, +otherwise put a `GITHUB_TOKEN` in `.env` yourself. ## Recipe @@ -34,7 +36,9 @@ repos: - repo: Jebel-Quant/actions # monitored, but not cloned here ``` -`owner/name` comes from each checkout's `origin`, so the path is all you write. +`owner/name` comes from each checkout's `origin`, so the path is all you write, +and the path is used as written — a checkout does not have to live at +`//`. Then: ```bash @@ -43,8 +47,9 @@ open http://localhost:3000/d/jq-fleet ``` The board fills in within a minute — the local panels first, the GitHub panels -after the first API refresh. Both `repos.yml` and the generated -`docker-compose.repos.yml` are gitignored: they describe one machine's folders. +after the first API refresh. `repos.yml` is gitignored: it describes one machine's folders. Nothing is +generated from it — the collector reads it at every launch, so there is no +second file to fall out of step. ## Then what @@ -53,6 +58,7 @@ after the first API refresh. Both `repos.yml` and the generated | Add or drop a repo | edit `repos.yml`, `./scripts/up.sh` again — [details](docs/configuration.md) | | Erase a dropped repo's history | [`./scripts/purge-repo.sh owner/name`](docs/configuration.md#dropping-a-repo) (irreversible) | | Stop | `./scripts/down.sh` (add `--volumes` to discard the history too) | +| See the collector's log | `tail -f .collector-logs/collector.log` — it runs on your machine, not in Docker ([why](docs/operations.md#the-collector-runs-on-your-machine)) | | Edit the board | change `grafana/dashboards/fleet.json`; it reloads in 30s — [read the traps first](docs/dashboard.md#traps-worth-not-re-introducing) | | Get notified | add a contact point under *Alerting → Contact points* — [why it is not provisioned](docs/dashboard.md#alerting) | diff --git a/collector/Dockerfile b/collector/Dockerfile deleted file mode 100644 index 2cebba0..0000000 --- a/collector/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM python:3.12-slim - -# git is the collector's other data source, alongside the GitHub API. -RUN apt-get update \ - && apt-get install -y --no-install-recommends git ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# The clones are bind-mounted from the host and are owned by a uid this -# container does not have, which git refuses to touch by default. The mount is -# read-only, so widening the trust here cannot lead to a write. -RUN git config --system --add safe.directory '*' - -WORKDIR /app -COPY pyproject.toml README.md ./ -COPY jq_collector ./jq_collector -RUN pip install --no-cache-dir . - -EXPOSE 9109 -CMD ["python", "-m", "jq_collector"] diff --git a/collector/tests/test_fleet.py b/collector/tests/test_fleet.py index 1475123..0bb64fc 100644 --- a/collector/tests/test_fleet.py +++ b/collector/tests/test_fleet.py @@ -436,44 +436,40 @@ def test_a_directory_that_is_not_a_checkout_fails(gen_repos, tmp_path): gen_repos.resolve({"path": str(tmp_path / "empty")}, 1) -def test_env_mode_prints_the_fleet_and_the_paths_and_writes_nothing( +def test_it_prints_the_fleet_and_the_paths_and_writes_nothing( gen_repos, tmp_path, monkeypatch, capsys ): """Both lines must come from repos.yml, not from anyone's memory. Retyping either is how they drift: a repo added here never reaches the - board and nothing reports the difference. The paths line is what lets a - collector running outside the container find a checkout that does not sit - at // - and a repo with no checkout must not appear in - it at all. --env must also leave the compose override alone, so it is safe - to run on a machine that has no stack. + board and nothing reports the difference. The paths line is what lets the + collector find a checkout that does not sit at //, and a + repo with no checkout must not appear in it at all. Nothing is written to + disk - scripts/collector.sh runs this at every launch and exports the + result, so there is no generated file in between to go stale. """ path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") source = tmp_path / "repos.yml" source.write_text(f"repos:\n - path: {path}\n - repo: cvxgrp/cvxsimulator\n") - target = tmp_path / "docker-compose.repos.yml" monkeypatch.setattr(gen_repos, "SOURCE", source) - monkeypatch.setattr(gen_repos, "TARGET", target) - monkeypatch.setattr(sys, "argv", ["gen-repos.py", "--env"]) + monkeypatch.setattr(sys, "argv", ["gen-repos.py"]) gen_repos.main() - lines = capsys.readouterr().out.strip().splitlines() - assert lines == [ + assert capsys.readouterr().out.strip().splitlines() == [ "JQ_REPOS=Jebel-Quant/rhiza,cvxgrp/cvxsimulator", f"JQ_REPO_PATHS=Jebel-Quant/rhiza={path}", ] - assert not target.exists() + assert list(tmp_path.glob("docker-compose*")) == [] -def test_env_mode_refuses_a_path_it_cannot_express(gen_repos, tmp_path, monkeypatch, capsys): +def test_it_refuses_a_path_it_cannot_express(gen_repos, tmp_path, monkeypatch, capsys): """A comma is the separator, so a path holding one would read as two repos.""" path = make_checkout(tmp_path, "Jebel-Quant", "rhi,za") source = tmp_path / "repos.yml" source.write_text(f"repos:\n - path: {path}\n") monkeypatch.setattr(gen_repos, "SOURCE", source) - monkeypatch.setattr(gen_repos, "TARGET", tmp_path / "out.yml") - monkeypatch.setattr(sys, "argv", ["gen-repos.py", "--env"]) + monkeypatch.setattr(sys, "argv", ["gen-repos.py"]) with pytest.raises(SystemExit): gen_repos.main() diff --git a/docker-compose.yml b/docker-compose.yml index 71b232f..ae1e8e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,37 +1,15 @@ -# The laptop stack. Not usable on its own - the fleet lives in repos.yml, -# so bring it up with scripts/up.sh, or with: +# Prometheus and Grafana. The collector is NOT here: it runs on the host, via +# scripts/collector.sh, because it reads your working copies and a container can +# only see them through bind mounts - which cannot name a checkout that sits +# somewhere other than //, and which are slow enough on +# macOS to have needed a caching layer to work around. # -# python3 scripts/gen-repos.py -# docker compose -f docker-compose.yml -f docker-compose.repos.yml up -d +# ./scripts/up.sh starts both halves +# docker compose up -d starts only this half name: jq-monitoring services: - collector: - build: ./collector - container_name: jq-collector - restart: unless-stopped - environment: - GITHUB_TOKEN: ${GITHUB_TOKEN:?set GITHUB_TOKEN in monitoring/.env - scripts/up.sh does it for you} - # JQ_REPOS and the per-repo bind mounts both come from - # docker-compose.repos.yml, generated from repos.yml. This file has no - # opinion about which repos are yours. - JQ_REPO_ROOT: /repos - JQ_TEMPLATE_REPO: ${JQ_TEMPLATE_REPO:-Jebel-Quant/rhiza} - JQ_GITHUB_INTERVAL: ${JQ_GITHUB_INTERVAL:-600} - JQ_LOCAL_INTERVAL: ${JQ_LOCAL_INTERVAL:-60} - JQ_MEASURE_MAX_AGE: ${JQ_MEASURE_MAX_AGE:-86400} - JQ_IGNORE: ${JQ_IGNORE:-} - JQ_PUBLIC_ONLY: ${JQ_PUBLIC_ONLY:-false} - JQ_LOG_LEVEL: ${JQ_LOG_LEVEL:-INFO} - # No volumes here on purpose. Every checkout is mounted individually by - # docker-compose.repos.yml, read-only: the collector reports on your working - # copies and must never be able to touch them, and mounting only what is - # listed means an unlisted repo is not merely filtered out - it is not even - # visible to the container. Every git call is --no-optional-locks too. - ports: - - "127.0.0.1:9109:9109" - prometheus: image: prom/prometheus:v2.55.1 container_name: jq-prometheus @@ -46,8 +24,10 @@ services: - prometheus-data:/prometheus ports: - "127.0.0.1:9090:9090" - depends_on: - - collector + # Docker Desktop resolves host.docker.internal on its own; plain Docker on + # Linux does not, and without this the collector target is simply down. + extra_hosts: + - "host.docker.internal:host-gateway" grafana: image: grafana/grafana:11.3.1 diff --git a/docs/configuration.md b/docs/configuration.md index 8526031..d0e1ef4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -87,27 +87,23 @@ better a refusal than a board that is quietly one repo short. | `JQ_IGNORE` | — | Repos to drop without editing `repos.yml`, as bare names or `owner/name`. Applies to both halves. | | `JQ_INCLUDE_ARCHIVED` | `false` | Archived repos are dropped from both halves. | | `JQ_PUBLIC_ONLY` | `false` | Drop private repos entirely — not just their details, their existence. | -| `JQ_REPO_PATHS` | — | Where each checkout really is, as `owner/name=path` pairs. Only needed outside the container — see [Checkout paths](#checkout-paths). | +| `JQ_REPO_PATHS` | — | Where each checkout really is, as `owner/name=path` pairs. `scripts/collector.sh` sets it from `repos.yml` — see [Checkout paths](#checkout-paths). | | `JQ_GITHUB_INTERVAL` | `300` | Seconds between GitHub refreshes. | | `JQ_MEASURE_MAX_AGE` | `86400` | Seconds an unchanged line/commit count may stand before it is retaken. See [Size and cadence](dashboard.md#size-and-cadence). | | `PROM_RETENTION` | `180d` | How much history to keep. | -`scripts/gen-repos.py` turns `repos.yml` into the `JQ_REPOS` list the -collector actually reads, so both halves see the same fleet. Setting `JQ_REPOS` -by hand works too, and skips `repos.yml` entirely — but then nothing mounts the -checkouts, so only the GitHub panels have anything to say. +`scripts/gen-repos.py` turns `repos.yml` into the `JQ_REPOS` list the collector +actually reads, so both halves see the same fleet. Setting `JQ_REPOS` by hand +works too and skips `repos.yml` entirely — but then no checkout paths are set +either, so only the GitHub panels have anything to say. ## Checkout paths The collector reads a repo at the path `JQ_REPO_PATHS` gives for it, and failing that at `//`. -**In the container only the fallback is used, and it is always right**, because -the generated bind mounts put every checkout at exactly that path. `repos.yml` -can say anything and the mount normalises it. - -**Outside the container it is often wrong**, because paths are whatever they are -on disk. An entry like +The collector runs on your machine, so paths are whatever they are on disk and +the fallback is often wrong. An entry like ```yaml - path: ~/repos/tschm/rhiza_projects/cs # this is tschm/cs @@ -118,11 +114,13 @@ working-copy panels while staying on the GitHub ones — present on the board, a quietly missing half its columns. Four repos in the fleet this was built against are laid out that way. -So when you run the collector on the host, take both lines from `repos.yml` -rather than writing them out: +You do not normally set this yourself: `scripts/collector.sh` runs +`scripts/gen-repos.py` at every launch and exports both lines, so `repos.yml` +stays the only place the fleet and the layout are written down. To see what it +resolves to: ```bash -uv run --with pyyaml python scripts/gen-repos.py --env >> .env +uv run --with pyyaml python scripts/gen-repos.py ``` A path containing a comma cannot be expressed — comma is the separator — and diff --git a/docs/dashboard.md b/docs/dashboard.md index 68df28c..2b190a7 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -107,6 +107,14 @@ name to `owner/name` when the cvxgrp repos were added, so history from before that point lives under the old names. Windows spanning the change show both generations; they age out. This is inherent to Prometheus, not a misconfiguration. +**The `instance` label is pinned on purpose.** There is one collector, so +`instance` says only where the process happened to be running — and left alone +it is the scrape address. When the collector moved out of its container the +address changed, every series forked in two, and for one stale window each repo +appeared on the board twice. `prometheus.yml` relabels it to a constant so that +cannot recur; series from before the move keep the old value and age out with +retention. + **Never `group_left` onto an `_info` metric without a `topk` guard.** The `_info` metrics carry a label that changes value — `ref`, `branch`, `conclusion`, `checks`. When one changes, Prometheus returns the old *and* the diff --git a/docs/operations.md b/docs/operations.md index d033001..02e4334 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -6,6 +6,42 @@ keywords: grafana login, no data, prometheus staleness, macos sleep, caffeinate # Running it day to day +## The collector runs on your machine + +Two halves, not three containers. Prometheus and Grafana are in Docker; +**the collector is an ordinary process on your Mac**, started by `up.sh` as a +launchd agent and stopped by `down.sh`. + +It is not containerised because it reads your working copies, and a container +can only reach those through bind mounts. Those mounts have to place every +checkout at `//`, which silently loses any repo that lives +somewhere else, and reading thousands of small files back through them on macOS +is slow enough that the line counts needed a cache to stay affordable. On the +host both problems disappear. + +| | | +|---|---| +| Its log | `.collector-logs/collector.log` | +| Restart it | `launchctl kickstart -k gui/$UID/com.jebel-quant.jq-collector` | +| Run it in the foreground instead | `./scripts/collector.sh` (Ctrl-C stops it) | +| Is Prometheus reaching it? | — the `jq-collector` job | + +Prometheus scrapes it at `host.docker.internal:9109`, which is how a container +reaches the machine it runs on. + +**The trade.** Inside the container the collector could only see the checkouts +mounted into it, so an unlisted repo was not merely filtered out — it was +invisible. It now runs as you and could read anything you can. It still never +writes: every git call is read-only and passes `--no-optional-locks`. But that +is now a property of the code rather than something the sandbox enforces. + +**If it will not start**, the usual cause is `PATH`. A launchd agent inherits +`/usr/bin:/bin:/usr/sbin:/sbin` and nothing else, so a `uv` under +`/opt/homebrew` or `~/.local` is invisible to it. `up.sh` pins `uv`'s directory +into the plist when it installs the agent, so re-running `./scripts/up.sh` +after moving or reinstalling `uv` is the fix. + + ## The "Sign in" button Grafana's own local login, against a SQLite file in the `grafana-data` volume on diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 24ac892..9e47616 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -9,8 +9,22 @@ global: scrape_configs: - job_name: jq-collector + # The collector runs on the host, not in this stack - it reads your working + # copies at their real paths, which a bind mount cannot express and which + # is far faster than one anyway. host.docker.internal is how a container + # reaches the machine it runs on; docker-compose.yml maps it on Linux, + # where it is not built in. static_configs: - - targets: ["collector:9109"] + - targets: ["host.docker.internal:9109"] + relabel_configs: + # There is exactly one collector, so `instance` carries no information - + # only where the process happened to be running. Left alone it is the + # scrape address, and moving the collector (container -> host, or back) + # silently forks every series in two: the board then joins a repo against + # itself and shows each one twice until the old series go stale. Pin it, + # and the history stays continuous across the move. + - target_label: instance + replacement: jq-collector - job_name: prometheus static_configs: diff --git a/scripts/collector.sh b/scripts/collector.sh new file mode 100755 index 0000000..cd3b91a --- /dev/null +++ b/scripts/collector.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Run the collector on this machine, in the foreground. +# +# Not in a container, on purpose. It reads your working copies, and a container +# can only reach those through bind mounts - which cannot name a checkout that +# sits somewhere other than //, and which are slow enough on +# macOS that the line counts needed a caching layer to stay affordable. +# +# Prometheus, which is still in Docker, scrapes this at host.docker.internal:9109. +# +# Run it under launchd (./scripts/up.sh sets that up on macOS), in a terminal, +# or under tmux - it is an ordinary foreground process either way, and Ctrl-C +# stops it. +# +# The trade this makes: inside the container the collector could only see the +# checkouts that were mounted into it, so an unlisted repo was not merely +# filtered out, it was invisible. Here it runs as you and could read anything +# you can. It still never writes - every git call is read-only and passes +# --no-optional-locks - but that is now a property of the code rather than +# something the sandbox enforces. +set -euo pipefail +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +die() { printf '\033[31merror\033[0m %s\n' "$1" >&2; exit 1; } + +[[ -f .env ]] || die ".env not found - run ./scripts/up.sh first" +[[ -f repos.yml ]] || die "repos.yml not found - run ./scripts/up.sh first" +# Says "on PATH", not "installed", because under launchd it is almost always +# the former: an agent inherits /usr/bin:/bin:/usr/sbin:/sbin and nothing else, +# so a perfectly good uv under /opt/homebrew or ~/.local is invisible. up.sh +# pins uv's directory into the plist for exactly this reason. +command -v uv >/dev/null 2>&1 || die "uv is not on PATH ($PATH) - see https://docs.astral.sh/uv/" + +set -a; . ./.env; set +a + +# repos.yml is the fleet and the layout, and it stays the only place either is +# written down: both lines are computed at every launch rather than generated +# into a file that can fall out of step with it. +# +# `export "$line"` and not `export $line` - a path may contain spaces, and +# unquoted word splitting would turn one repo into two broken ones. +while IFS= read -r line; do + [[ -n "$line" ]] && export "$line" +done < <(uv run --quiet --with pyyaml python scripts/gen-repos.py) + +# No container means no bind mounts and no /repos to fall back on. Every path +# is real and absolute, and JQ_REPO_PATHS carries all of them. +export JQ_REPO_ROOT="" + +cd collector +exec uv run --quiet python -m jq_collector diff --git a/scripts/down.sh b/scripts/down.sh index 7042271..cba596f 100755 --- a/scripts/down.sh +++ b/scripts/down.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash -# Stop the stack. Add --volumes to also throw away the Prometheus history. +# Stop both halves. Add --volumes to also throw away the Prometheus history. set -euo pipefail cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# The generated override may be missing on a fresh clone; the project name in -# docker-compose.yml is enough to find the running containers either way. -files=(-f docker-compose.yml) -[[ -f docker-compose.repos.yml ]] && files+=(-f docker-compose.repos.yml) -docker compose "${files[@]}" down "$@" + +label="com.jebel-quant.jq-collector" +if [[ "$(uname -s)" == "Darwin" ]]; then + # KeepAlive means the agent restarts itself if merely killed, so unload it. + # Fails when it was never loaded, which is fine. + launchctl bootout "gui/$UID/$label" 2>/dev/null && echo "collector: launchd agent unloaded" || true +else + pkill -f "python -m jq_collector" 2>/dev/null && echo "collector: stopped" || true +fi + +docker compose down "$@" diff --git a/scripts/gen-repos.py b/scripts/gen-repos.py index dea4e0c..0b1965e 100755 --- a/scripts/gen-repos.py +++ b/scripts/gen-repos.py @@ -1,25 +1,25 @@ #!/usr/bin/env python3 -"""Turn repos.yml into the compose override that mounts the fleet. +"""Turn repos.yml into the two environment lines the collector reads. The collector never searches for checkouts. This script resolves every listed -path, asks its origin remote who it is, and writes one read-only bind mount per -repo at /repos// plus the matching JQ_REPOS list. Both halves of -the collector then read the same list, and the board's contents are decided by -repos.yml alone. +path, asks its origin remote who it is, and prints: -Run it after editing repos.yml; scripts/up.sh does that for you. + JQ_REPOS the fleet, as owner/name - read by both halves + JQ_REPO_PATHS where each checkout actually is - read by the local half -With ``--env`` it writes nothing and prints the environment instead - JQ_REPOS, -and JQ_REPO_PATHS naming where each checkout really is: + python3 scripts/gen-repos.py - python3 scripts/gen-repos.py --env +Nothing is written. scripts/collector.sh runs this at every launch and exports +the result, so repos.yml stays the only place the fleet and the layout are +written down - there is no generated file in between to go stale. -That second line is what makes a collector running outside the container agree -with repos.yml. In the container the bind mounts normalise every checkout onto -/repos//, so the path is implied; on the host it is not, because a -repos.yml entry like ~/repos/tschm/rhiza_projects/cs for tschm/cs cannot be -recovered by joining owner to name. Without the paths those repos silently -vanish from the working-copy panels while staying on the GitHub ones. +The paths line exists because a repos.yml entry like + + - path: ~/repos/tschm/rhiza_projects/cs # a checkout of tschm/cs + +cannot be recovered by joining owner to name. This used to be papered over by +bind-mounting every checkout onto /repos//, back when the collector +ran in a container; it does not, so the real path has to be carried. """ from __future__ import annotations @@ -31,7 +31,6 @@ ROOT = Path(__file__).resolve().parent.parent SOURCE = ROOT / "repos.yml" -TARGET = ROOT / "docker-compose.repos.yml" def fail(message: str) -> None: @@ -117,10 +116,9 @@ def resolve(entry: object, index: int) -> tuple[str, Path | None]: def main() -> None: - env_only = "--env" in sys.argv[1:] - unknown = [a for a in sys.argv[1:] if a != "--env"] + unknown = sys.argv[1:] if unknown: - fail(f"unknown argument {unknown[0]} - the only flag is --env") + fail(f"unknown argument {unknown[0]} - this script takes none") resolved: dict[str, Path | None] = {} for index, entry in enumerate(load_entries(), start=1): @@ -129,43 +127,20 @@ def main() -> None: fail(f"{full_name} is listed twice") resolved[full_name] = path - # Nothing but the lines themselves on stdout, so they can be piped or - # appended to a .env. - if env_only: - print(f'JQ_REPOS={",".join(resolved)}') - paths = {name: path for name, path in resolved.items() if path is not None} - # A comma is the separator, so a path containing one cannot be - # expressed. Refuse rather than emit a line the collector would reject - # or, worse, silently misread as two repos. - for name, path in paths.items(): - if "," in str(path): - fail(f"{name}: path contains a comma, which JQ_REPO_PATHS cannot express: {path}") - if paths: - joined = ",".join(f"{name}={path}" for name, path in paths.items()) - print(f"JQ_REPO_PATHS={joined}") - return - - lines = [ - "# Generated by scripts/gen-repos.py from repos.yml - do not edit.", - "#", - "# One read-only bind mount per monitored checkout, at the canonical", - "# /repos// the collector looks for, plus the matching", - "# JQ_REPOS list so the GitHub half sees exactly the same fleet.", - "", - "services:", - " collector:", - " environment:", - f' JQ_REPOS: "{",".join(resolved)}"', - ] - mounts = {name: path for name, path in resolved.items() if path is not None} - if mounts: - lines.append(" volumes:") - lines += [f' - "{path}:/repos/{name}:ro"' for name, path in mounts.items()] - TARGET.write_text("\n".join(lines) + "\n", encoding="utf-8") - - without = len(resolved) - len(mounts) - tail = f", {without} without a local checkout" if without else "" - print(f"gen-repos: wrote {TARGET.name} - {len(resolved)} repos{tail}") + # Nothing but the lines themselves on stdout, so they can be exported, + # piped, or appended to a .env. + print(f'JQ_REPOS={",".join(resolved)}') + + paths = {name: path for name, path in resolved.items() if path is not None} + # A comma is the separator, so a path containing one cannot be expressed. + # Refuse rather than print a line the collector would reject or, worse, + # silently misread as two repos. + for name, path in paths.items(): + if "," in str(path): + fail(f"{name}: path contains a comma, which JQ_REPO_PATHS cannot express: {path}") + if paths: + joined = ",".join(f"{name}={path}" for name, path in paths.items()) + print(f"JQ_REPO_PATHS={joined}") if __name__ == "__main__": diff --git a/scripts/up.sh b/scripts/up.sh index ac0579f..e16e40f 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -1,5 +1,10 @@ #!/usr/bin/env bash -# Bring the stack up, minting a GitHub token from the gh CLI if there isn't one. +# Bring both halves up: Prometheus and Grafana in Docker, the collector on this +# machine. Mints a GitHub token from the gh CLI if there isn't one. +# +# The collector is not containerised - see scripts/collector.sh for why. On +# macOS it is installed as a launchd agent so it comes back after a reboot; +# elsewhere this script says how to run it and leaves supervision to you. set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -31,16 +36,66 @@ if ! grep -qE '^GITHUB_TOKEN=.+' .env; then echo "wrote a token from 'gh auth token' into monitoring/.env" fi -# repos.yml is the fleet. Regenerate the mounts every time, so editing the list -# and running ./scripts/up.sh is the whole workflow. -if command -v uv >/dev/null 2>&1; then - uv run --quiet --with pyyaml scripts/gen-repos.py +command -v uv >/dev/null 2>&1 || { + echo "uv is not installed - the collector needs it; see https://docs.astral.sh/uv/" >&2 + exit 1 +} + +# Fail on a broken repos.yml here, while there is still a human watching, rather +# than inside a launchd agent whose output nobody is reading. The collector +# recomputes this at every launch; this run is only a check. +uv run --quiet --with pyyaml python scripts/gen-repos.py >/dev/null + +docker compose up -d + +# -- the collector ----------------------------------------------------------- +label="com.jebel-quant.jq-collector" +if [[ "$(uname -s)" == "Darwin" ]]; then + plist="$HOME/Library/LaunchAgents/$label.plist" + # A launchd agent inherits almost no PATH - /usr/bin:/bin:/usr/sbin:/sbin and + # nothing else - so uv, which lives under /opt/homebrew or ~/.local, is simply + # not found. Pin the directory it is actually in rather than guessing at + # either location. + uv_dir="$(dirname "$(command -v uv)")" + logs="$here/.collector-logs" + mkdir -p "$HOME/Library/LaunchAgents" "$logs" + # Regenerated every time: the paths are absolute, so a moved checkout of this + # repo would otherwise leave a plist pointing at where it used to be. + cat > "$plist" < + + + + Label$label + ProgramArguments + $here/scripts/collector.sh + WorkingDirectory$here + EnvironmentVariables + + PATH$uv_dir:/usr/bin:/bin:/usr/sbin:/sbin + + RunAtLoad + KeepAlive + + ThrottleInterval10 + StandardOutPath$logs/collector.log + StandardErrorPath$logs/collector.log + + +PLIST + # bootout first so an edited plist is actually re-read; it fails when nothing + # is loaded, which is the normal first run. + launchctl bootout "gui/$UID/$label" 2>/dev/null || true + launchctl bootstrap "gui/$UID" "$plist" + echo "collector: loaded as a launchd agent - logs in .collector-logs/collector.log" else - python3 scripts/gen-repos.py + echo "collector: not started. Run it yourself, e.g." + echo " ./scripts/collector.sh # foreground" + echo " or install it as a systemd --user service running that script." fi -docker compose -f docker-compose.yml -f docker-compose.repos.yml up -d --build - echo echo "Grafana http://localhost:3000/d/jq-fleet (anonymous read-only; admin/admin to edit)" echo "Prometheus http://localhost:9090"