From 33dcceb7fe5876db15ae4666afe6cf841daba2a3 Mon Sep 17 00:00:00 2001 From: Thomas Schmelzer Date: Sun, 30 Aug 2026 09:42:54 +0400 Subject: [PATCH] List the monitored repos explicitly, in repos.yml The fleet was assembled two ways at once, and neither was under your control. A whole-org GitHub sweep decided membership from the org side - a new repo appeared unasked, and a shared org like cvxgrp dragged in 100+ repos that were not yours - while a directory walk under one mounted root decided it from the disk side, admitting any checkout that happened to sit there with an origin that looked right. Both are replaced by repos.yml: one entry per repo, `path` to a checkout with owner/name read from its origin, or a bare `repo:` for one you have not cloned. scripts/gen-repos.py turns it into docker-compose.repos.yml - a read-only bind mount per checkout at the canonical /repos//, plus the matching JQ_REPOS - so an unlisted repo is not merely filtered out, it is never visible to the container. Both halves of the collector read that one list, so the GitHub panels and the working-copy panels cannot disagree about who is in scope. The server stack keeps naming its fleet in JQ_REPOS, since there are no checkouts there to derive it from, and now refuses to start without it rather than serving an empty board. repos.yml and the generated override are gitignored: they describe the folder layout of one machine. That, plus the MIT license, is what this repo needed to be public. Verified against the live stack: 3 repos exported, 2 with working copies, local refresh 0.1s. Co-Authored-By: Claude Opus 5 (1M context) --- .env.example | 17 +-- .github/workflows/ci.yml | 33 ++++- .gitignore | 7 ++ LICENSE | 21 ++++ README.md | 78 +++++++++--- collector/jq_collector/config.py | 45 +++---- collector/jq_collector/github.py | 26 ++-- collector/jq_collector/localgit.py | 76 ++++++----- collector/tests/test_fleet.py | 194 +++++++++++++++++++++++++++++ docker-compose.server.yml | 9 +- docker-compose.yml | 25 ++-- repos.example.yml | 28 +++++ scripts/down.sh | 6 +- scripts/gen-repos.py | 139 +++++++++++++++++++++ scripts/up.sh | 17 ++- 15 files changed, 589 insertions(+), 132 deletions(-) create mode 100644 LICENSE create mode 100644 collector/tests/test_fleet.py create mode 100644 repos.example.yml create mode 100755 scripts/gen-repos.py diff --git a/.env.example b/.env.example index 31526a0..b2e8b7a 100644 --- a/.env.example +++ b/.env.example @@ -1,27 +1,22 @@ # Copy to .env, or let scripts/up.sh generate it from `gh auth token`. +# +# Which repos are monitored is NOT set here - that lives in repos.yml, one +# entry per checkout. This file holds credentials and cadences only. # Needs `repo` (private repo metadata) and `read:org`. A gh OAuth token works. +# It has to be able to read every repo listed in repos.yml. GITHUB_TOKEN= -# The org whose repos are monitored. Clones whose origin points elsewhere - an -# upstream numpy checkout, say - are skipped automatically. -JQ_ORG=Jebel-Quant - # The repo whose releases define "up to date" for the template pointer. -JQ_TEMPLATE_REPO=rhiza +JQ_TEMPLATE_REPO=Jebel-Quant/rhiza -# Comma-separated repo names to leave out entirely. +# Comma-separated repo names to drop without removing them from repos.yml. JQ_IGNORE= # Drop private repos completely - their names, workflows, PR titles and local # branches are all disclosure. Set this before serving the board publicly. JQ_PUBLIC_ONLY=true -# Where your clones live on the host, relative to this file or absolute. -# Scanned to JQ_SCAN_DEPTH levels, so ~/repos// is found. -JQ_REPO_ROOT_HOST=../.. -JQ_SCAN_DEPTH=2 - # Refresh cadences, in seconds. JQ_GITHUB_INTERVAL=300 JQ_LOCAL_INTERVAL=60 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 701daf0..ad8b067 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,19 +59,40 @@ jobs: name: compose files parse runs-on: ubuntu-latest # The overlays only patch services, so they are not valid on their own - - # they are checked the way they are actually used. GITHUB_TOKEN is a - # required variable in the base file; any value satisfies interpolation. + # they are checked the way they are actually used. GITHUB_TOKEN and, for + # the server stack, JQ_REPOS are required variables; any value satisfies + # interpolation. env: GITHUB_TOKEN: dummy-value-for-interpolation-only GF_ADMIN_PASSWORD: dummy-value-for-interpolation-only + JQ_REPOS: Jebel-Quant/rhiza steps: - uses: actions/checkout@v4 - - name: Base - run: docker compose -f docker-compose.yml config --quiet + + # repos.yml is gitignored, so CI builds one the way a new user would and + # proves gen-repos.py emits an override that actually merges. Nothing + # else in CI reads it, so a malformed one would otherwise only surface + # on somebody's laptop. + - name: Generate the fleet override + run: | + pip install --quiet pyyaml + printf 'repos:\n' > repos.yml + for full in Jebel-Quant/rhiza cvxgrp/cvxsimulator; do + git init -q -b main "ci-checkouts/$full" + git -C "ci-checkouts/$full" remote add origin "https://github.com/$full.git" + printf ' - path: ci-checkouts/%s\n' "$full" >> repos.yml + done + # A repo with no checkout: GitHub panels only, no bind mount. + printf ' - repo: Jebel-Quant/actions\n' >> repos.yml + python3 scripts/gen-repos.py + cat docker-compose.repos.yml + + - name: Base + fleet override + run: docker compose -f docker-compose.yml -f docker-compose.repos.yml config --quiet - name: Base + admin overlay - run: docker compose -f docker-compose.yml -f docker-compose.admin.yml config --quiet + run: docker compose -f docker-compose.yml -f docker-compose.repos.yml -f docker-compose.admin.yml config --quiet - name: Base + public overlay - run: docker compose -f docker-compose.yml -f docker-compose.public.yml config --quiet + run: docker compose -f docker-compose.yml -f docker-compose.repos.yml -f docker-compose.public.yml config --quiet - name: Server stack run: docker compose -f docker-compose.server.yml config --quiet - name: Server + TLS overlay diff --git a/.gitignore b/.gitignore index 09831da..7ff2ff7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ .env +# Your fleet and the mounts generated from it: both describe the folder layout +# of one machine, which has no business in a public repo. Start from +# repos.example.yml. +repos.yml +docker-compose.repos.yml + # Left behind by running the collector natively instead of in its container. .venv/ .ruff_cache/ +.pytest_cache/ __pycache__/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7b9457b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Jebel Quant Research + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index f566700..9545c26 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,35 @@ A Grafana board for the state of your repo fleet — template drift, CI on the default branch, open pull requests, and the working copies on this machine — with Prometheus keeping the history and six alert rules on top. -Two kinds of repo are in scope: **whole orgs** (`Jebel-Quant`, swept in full) and -**individually named repos** (`cvxgrp/cvxrisk`, `cvxgrp/simulator`, -`cvxgrp/cvxcla`, `cvxgrp/cvxmarkowitz`). The second exists because cvxgrp has -100+ repos and only four are yours. +**The fleet is an explicit list.** `repos.yml` names every monitored repo, one +entry per checkout on this machine: + +```yaml +repos: + - path: ~/repos/jebel-quant/rhiza + - path: ~/repos/cvxgrp/cvxsimulator + - repo: Jebel-Quant/actions # monitored, but not checked out here +``` -**Archived repos are never monitored.** They are dropped from the GitHub sweep -*and* their local clones are skipped, so a checkout left on disk cannot keep a +`owner/name` is read from each checkout's `origin` remote, so the path is all +you write. Nothing is discovered: a repo is on the board because it is in this +file, and for no other reason. `scripts/up.sh` turns the file into +`docker-compose.repos.yml`, which mounts each checkout **read-only** at +`/repos//` — so an unlisted repo is not merely filtered out, it is +never visible to the container at all. + +This replaced a whole-org GitHub sweep plus a directory walk under one mounted +root. Both decided membership on their own: a new repo in the org arrived +unasked, a shared org like cvxgrp dragged in 100+ repos that were not yours, and +any checkout that happened to sit under the root joined the board because its +origin looked right. + +Edit `repos.yml`, run `./scripts/up.sh` again, and the fleet is whatever you +just wrote. Both halves of the collector read the same list, so the GitHub +panels and the working-copy panels can never disagree about who is in scope. + +**Archived repos are never monitored.** They are dropped from the GitHub half +*and* their local checkouts are skipped, so a checkout left on disk cannot keep a dead repo on the board — that gap kept `rhiza-brainbug` showing as the fleet's one red repo for a while after it was archived. Set `JQ_INCLUDE_ARCHIVED=true` to opt back in. @@ -37,10 +59,15 @@ path. The script reports both numbers so the difference is visible rather than alarming. ```bash +cp repos.example.yml repos.yml # then list your checkouts ./scripts/up.sh # builds, mints a token from `gh auth token`, starts everything ./scripts/down.sh # stop; add --volumes to discard the history too ``` +`up.sh` creates `repos.yml` from the example on a first run and stops so you can +edit it. Both `repos.yml` and the generated `docker-compose.repos.yml` are +gitignored: they describe the folder layout of one machine. + | | | |---|---| | Dashboard | | @@ -207,20 +234,35 @@ your checkout is behind, not the repo. ## Configuration -Everything lives in `.env` (see `.env.example`). The useful knobs: +**Which repos** is `repos.yml`. **Everything else** is `.env` (see +`.env.example`). + +### repos.yml + +| Key | | | +|---|---|---| +| `path` | | A checkout on this machine. `~` and paths relative to the repo both work. | +| `repo` | | `owner/name`. Optional next to a `path` — it overrides the origin, which is what you want for a fork whose board should follow upstream. On its own it monitors a repo you have not cloned: GitHub panels are gathered, the working-copy panels stay empty for that row. | + +A bare string is shorthand for `path`. Duplicate entries, a path that is not a +checkout, and an entry with neither key are all refused at generate time — +better a refusal than a board that is quietly one repo short. + +### .env | Variable | Default | | |---|---|---| -| `JQ_ORGS` | `Jebel-Quant` | Orgs swept in full, comma-separated. | -| `JQ_REPOS` | the four cvxgrp repos | Individually named repos, as `owner/name`. Use this for orgs where only some repos are yours. | +| `GITHUB_TOKEN` | — | Must be able to read every repo in `repos.yml`. `up.sh` mints one from `gh auth token`. | | `JQ_TEMPLATE_REPO` | `Jebel-Quant/rhiza` | Whose releases define "up to date", as `owner/name`. | -| `JQ_IGNORE` | — | Repos to leave out, as bare names or `owner/name`. Applies to both the GitHub sweep and the local scan. | -| `JQ_INCLUDE_ARCHIVED` | `false` | Archived repos are dropped from both the GitHub sweep and the local scan. | -| `JQ_REPO_ROOT_HOST` | `../..` | Which host folder to mount. Defaults to all of `~/repos`. | -| `JQ_SCAN_DEPTH` | `2` | How deep to look for clones, so `~/repos//` is found. | +| `JQ_IGNORE` | — | Repos to drop without editing `repos.yml`, as bare names or `owner/name`. Applies to both halves. | +| `JQ_INCLUDE_ARCHIVED` | `false` | Archived repos are dropped from both halves. | +| `JQ_PUBLIC_ONLY` | `false` | Drop private repos entirely — not just their details, their existence. | | `JQ_GITHUB_INTERVAL` | `300` | Seconds between GitHub refreshes. | | `PROM_RETENTION` | `180d` | How much history to keep. | +On the server stack there are no checkouts and so no `repos.yml`: the fleet is +named directly in `JQ_REPOS`, a comma-separated list of `owner/name`. + ### API budget A refresh costs roughly `3 × repos + open PRs` REST calls. Measured on this @@ -308,6 +350,7 @@ copies here, and strangers can reach it": | | | |---|---| | `JQ_REPO_ROOT` empty | local scanning skipped entirely — a clean no-op, not an error every minute | +| the fleet is `JQ_REPOS` | no checkouts to derive it from, so the list is named directly rather than in `repos.yml` | | `JQ_PUBLIC_ONLY` forced on | private repos are never gathered, so they cannot leak | | only Grafana publishes a port | Prometheus and the collector talk over the compose network and are unreachable from outside | | anonymous off, public dashboards on | a public link serves one dashboard's queries with no datasource behind it | @@ -331,9 +374,9 @@ GITHUB_TOKEN=github_pat_... # public_repo scope is enough GF_ADMIN_PASSWORD=... # 16+ chars; openssl rand -base64 24 FLEET_DOMAIN=fleet.example.com ACME_EMAIL=you@example.com -# Repos outside the swept org. Leave this out and they are silently absent - -# the board simply reports a smaller fleet, with nothing to say it is short. -JQ_REPOS=cvxgrp/cvxrisk,cvxgrp/simulator,cvxgrp/cvxcla,cvxgrp/cvxmarkowitz +# The fleet. Required - there are no checkouts here to derive it from, and the +# stack refuses to start without it rather than serving an empty board. +JQ_REPOS=Jebel-Quant/rhiza,Jebel-Quant/actions,cvxgrp/cvxsimulator SETTINGS chmod 600 .env @@ -345,7 +388,8 @@ it automatically, and it is gitignored. Note the token has to be able to read every repo named in `JQ_REPOS`. A fine-grained token scoped to one org cannot see another's, and the collector -logs `named repo ... is not readable` when that happens. +logs `listed repo ... is not readable` when that happens — one unreadable entry +costs one row, not the whole board. Then reach Grafana, sign in as `admin`, open **Jebel-Quant Fleet (public)** → *Share* → *Public dashboard*, and share only that link. diff --git a/collector/jq_collector/config.py b/collector/jq_collector/config.py index 8670ac8..d403a93 100644 --- a/collector/jq_collector/config.py +++ b/collector/jq_collector/config.py @@ -24,23 +24,29 @@ def _csv(name: str, default: tuple[str, ...] = ()) -> tuple[str, ...]: class Config: """Everything the collector needs to know about its environment. - Repos arrive two ways: whole-org sweeps (``JQ_ORGS``) and individually named - repos (``JQ_REPOS``). The second exists because you rarely want *all* of a - large shared org - cvxgrp has 100+ repos and only a handful are yours. + The fleet is an explicit list: ``JQ_REPOS`` names every monitored repo as + ``owner/name``, and nothing else is ever gathered. There used to be a + whole-org sweep as well, which meant the board's contents were decided by + GitHub rather than by you - a new repo in the org appeared unasked, and a + shared org like cvxgrp dragged in a hundred repos that were not yours. + + On a laptop the list is generated from ``repos.yml`` by + ``scripts/gen-repos.py``, which also mounts each checkout at + ``$JQ_REPO_ROOT//``; on a server it is set by hand and no + checkouts exist. Both halves read the same list, so the GitHub panels and + the working-copy panels can never disagree about who is in the fleet. """ - orgs: tuple[str, ...] = field(default_factory=lambda: _csv("JQ_ORGS", ("Jebel-Quant",))) - extra_repos: tuple[str, ...] = field(default_factory=lambda: _csv("JQ_REPOS")) + repos: tuple[str, ...] = field(default_factory=lambda: _csv("JQ_REPOS")) token: str = os.environ.get("GITHUB_TOKEN", "") api: str = os.environ.get("GITHUB_API", "https://api.github.com") - # Where the clones are mounted (read-only) inside the container. Scanned to - # a depth of two, so both ~/repos/ and ~/repos// are found. - # Set empty to skip local scanning entirely - the right setting on a server, - # where there are no working copies and the local panels do not apply. + # Where the checkouts are mounted (read-only) inside the container, one per + # repo at //. Set empty to skip local scanning + # entirely - the right setting on a server, where there are no working + # copies and the local panels do not apply. repo_root: str = os.environ.get("JQ_REPO_ROOT", "/repos") - scan_depth: int = _int("JQ_SCAN_DEPTH", 2) # owner/name of the repo whose releases define "up to date". template_repo: str = os.environ.get("JQ_TEMPLATE_REPO", "Jebel-Quant/rhiza") @@ -63,7 +69,10 @@ class Config: # could crowd out entries the fleet-wide list should have shown. recent_merges_per_repo: int = _int("JQ_RECENT_MERGES_PER_REPO", 10) - # Repos to leave out, as bare names or as owner/name. + # Repos to leave out, as bare names or as owner/name. Redundant now that + # the fleet is an explicit list - deleting the line from repos.yml is the + # obvious move - but it stays for the server, where the list is an env var + # and commenting one entry out is not possible. ignore: tuple[str, ...] = field(default_factory=lambda: _csv("JQ_IGNORE")) include_archived: bool = os.environ.get("JQ_INCLUDE_ARCHIVED", "false").lower() == "true" @@ -73,19 +82,5 @@ class Config: # names, its PR titles and its local branch names are all disclosure. public_only: bool = os.environ.get("JQ_PUBLIC_ONLY", "false").lower() == "true" - @property - def owners(self) -> frozenset[str]: - """Every owner we might accept a clone from, lowercased.""" - from_extras = (r.split("/", 1)[0] for r in self.extra_repos if "/" in r) - return frozenset(o.lower() for o in (*self.orgs, *from_extras)) - def is_ignored(self, owner: str, name: str) -> bool: return name in self.ignore or f"{owner}/{name}" in self.ignore - - def wants(self, owner: str, name: str) -> bool: - """Is this repo in scope - by its org, or by being named explicitly?""" - if self.is_ignored(owner, name): - return False - if owner.lower() in {o.lower() for o in self.orgs}: - return True - return f"{owner}/{name}".lower() in {r.lower() for r in self.extra_repos} diff --git a/collector/jq_collector/github.py b/collector/jq_collector/github.py index 7690de0..cb2ed56 100644 --- a/collector/jq_collector/github.py +++ b/collector/jq_collector/github.py @@ -107,31 +107,25 @@ def _paginate(self, path: str, **params: object) -> list[dict]: # -- fleet-level ----------------------------------------------------- def list_repos(self) -> list[dict]: - """Every in-scope repo: whole-org sweeps plus individually named ones.""" + """The repos named in the config, in the order they were listed. + + One call each, and no org sweep: the fleet is whatever you wrote down. + A repo that cannot be read is dropped with a warning rather than + failing the refresh, so one bad line does not blank the whole board. + """ repos: list[dict] = [] seen: set[str] = set() - for org in self._cfg.orgs: - found = self._paginate(f"/orgs/{org}/repos", type="all", sort="full_name") - if not found: - # A user account rather than an org, or no org read access. - owned = self._paginate("/user/repos", affiliation="owner", sort="full_name") - found = [r for r in owned if (r.get("owner") or {}).get("login") == org] - for raw in found: - if raw.get("full_name") not in seen: - seen.add(raw["full_name"]) - repos.append(raw) - - for full_name in self._cfg.extra_repos: - if full_name in seen or "/" not in full_name: + for full_name in self._cfg.repos: + if "/" not in full_name or full_name in seen: continue raw = self._json(f"/repos/{full_name}") - if isinstance(raw, dict): + if isinstance(raw, dict) and raw.get("full_name"): seen.add(raw["full_name"]) repos.append(raw) else: log.warning( - "named repo %s is not readable - check the token's scopes", + "listed repo %s is not readable - check the name and the token's scopes", full_name, ) diff --git a/collector/jq_collector/localgit.py b/collector/jq_collector/localgit.py index 623470e..fae2a23 100644 --- a/collector/jq_collector/localgit.py +++ b/collector/jq_collector/localgit.py @@ -8,6 +8,11 @@ last fetch *you* ran, which is why ``fetch_age`` is reported alongside them; for a fetch-independent answer the exporter compares the local default-branch sha with the one GitHub reports. + +Nothing here searches for checkouts either. Each monitored repo is expected at +``//``, which is exactly where the generated compose +override mounts it. The fleet is decided by ``repos.yml``, not by whatever +happened to be lying around under a scanned directory. """ from __future__ import annotations @@ -24,9 +29,6 @@ _TIMEOUT = 20 -# Never descend into these while looking for clones. -_SKIP_DIRS = frozenset({".git", ".venv", "venv", "node_modules", "__pycache__", ".tox", "target"}) - def _git(path: str, *args: str) -> str | None: """Run a read-only git command, returning None if it fails.""" @@ -108,31 +110,6 @@ def _ahead_behind(path: str) -> tuple[int | None, int | None]: return None, None -def _find_clones(root: str, max_depth: int) -> list[str]: - """Directories under ``root`` that are git working copies, breadth first.""" - found: list[str] = [] - frontier = [(root, 0)] - while frontier: - path, depth = frontier.pop(0) - if depth > max_depth: - continue - try: - entries = sorted(os.listdir(path)) - except OSError: - continue - if ".git" in entries: - # A clone is a leaf: submodules and nested repos are not the fleet. - found.append(path) - continue - for entry in entries: - if entry in _SKIP_DIRS or entry.startswith("."): - continue - child = os.path.join(path, entry) - if os.path.isdir(child) and not os.path.islink(child): - frontier.append((child, depth + 1)) - return found - - def scan_repo(name: str, owner: str, path: str, cfg: Config) -> LocalRepo: status = _git(path, "status", "--porcelain=v1") or "" lines = [line for line in status.splitlines() if line.strip()] @@ -170,10 +147,13 @@ def scan( default_branches: dict[str, str], skip: frozenset[str] = frozenset(), ) -> dict[str, LocalRepo]: - """Scan every in-scope clone under the repo root, keyed by ``owner/name``. + """Read every listed repo's working copy, keyed by ``owner/name``. + + A repo with no checkout on this machine is not an error: the fleet is the + list, and the local panels simply have nothing to say about that row. ``skip`` holds repos deliberately dropped - archived on GitHub, or named - in JQ_IGNORE. Without it a clone would keep a dropped repo on the board + in JQ_IGNORE. Without it a checkout would keep a dropped repo on the board after the GitHub half had stopped reporting it. """ found: dict[str, LocalRepo] = {} @@ -185,21 +165,35 @@ def scan( log.error("repo root %s is not a directory", cfg.repo_root) return found - for path in _find_clones(cfg.repo_root, cfg.scan_depth): - origin = origin_owner_name(path) - if origin is None: - log.debug("skipping %s: no origin remote", path) + for key in cfg.repos: + if "/" not in key or key in skip: continue - owner, repo_name = origin - if not cfg.wants(owner, repo_name): - # e.g. an upstream numpy clone, or another org's repo parked nearby. - log.debug("skipping %s: %s/%s is out of scope", path, owner, repo_name) + owner, repo_name = key.split("/", 1) + path = os.path.join(cfg.repo_root, owner, repo_name) + # `.git` is a directory in a plain checkout and a file in a worktree. + if not os.path.exists(os.path.join(path, ".git")): + # Not mounted, or mounted somewhere else. Normal on a server, and + # normal for a repo you monitor but have not checked out. + log.debug("no working copy for %s at %s", key, path) continue - key = f"{owner}/{repo_name}" - if key in skip: - log.debug("skipping %s: archived or ignored", key) + # The mount point claims to be this repo; the origin remote is the only + # thing that can confirm it. A wrong path in repos.yml would otherwise + # report one repo's dirty files under another repo's name. + origin = origin_owner_name(path) + if origin is not None and (origin[0].lower(), origin[1].lower()) != ( + owner.lower(), + repo_name.lower(), + ): + log.warning( + "%s is a checkout of %s/%s, not %s - check repos.yml", + path, + origin[0], + origin[1], + key, + ) continue + local = scan_repo(repo_name, owner, path, cfg) default_branch = default_branches.get(key, "main") sha = _git(path, "rev-parse", "--verify", "--quiet", default_branch) or "" diff --git a/collector/tests/test_fleet.py b/collector/tests/test_fleet.py new file mode 100644 index 0000000..6d2bedb --- /dev/null +++ b/collector/tests/test_fleet.py @@ -0,0 +1,194 @@ +"""The fleet is the list, and only the list. + +The board used to be assembled by a whole-org GitHub sweep plus a directory +walk under one mounted root. Both decided membership on their own: a new repo +in the org appeared unasked, and any checkout that happened to sit under the +root joined the board because its origin looked right. These tests pin the +replacement - a repo is monitored because it is named in the config, and for no +other reason - and pin the two ways that can go quietly wrong: a listed repo +with no checkout must be a no-op, and a checkout of the wrong repo must be +refused rather than reported under the listed repo's name. +""" + +from __future__ import annotations + +import importlib.util +import pathlib +import subprocess + +import pytest + +from jq_collector import localgit +from jq_collector.config import Config + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] + + +def make_checkout( + root: pathlib.Path, owner: str, name: str, origin: str | None = None +) -> pathlib.Path: + """A real git checkout - the code shells out to git, so fixtures must too.""" + path = root / owner / name + path.mkdir(parents=True) + git = ["git", "-C", str(path)] + subprocess.run([*git[:1], "init", "-q", "-b", "main", str(path)], check=True) + subprocess.run([*git, "config", "user.email", "t@example.com"], check=True) + subprocess.run([*git, "config", "user.name", "T"], check=True) + (path / "README.md").write_text("x\n") + subprocess.run([*git, "add", "-A"], check=True) + subprocess.run([*git, "commit", "-qm", "init"], check=True) + subprocess.run( + [*git, "remote", "add", "origin", origin or f"git@github.com:{owner}/{name}.git"], + check=True, + ) + return path + + +def config(repos: tuple[str, ...], repo_root: pathlib.Path) -> Config: + cfg = Config() + object.__setattr__(cfg, "repos", repos) + object.__setattr__(cfg, "repo_root", str(repo_root)) + return cfg + + +# -- the local half ---------------------------------------------------------- + + +def test_only_listed_repos_are_read(tmp_path): + """An unlisted checkout sitting right next to a listed one is not the fleet.""" + make_checkout(tmp_path, "Jebel-Quant", "rhiza") + make_checkout(tmp_path, "Jebel-Quant", "not-listed") + + found = localgit.scan(config(("Jebel-Quant/rhiza",), tmp_path), {}) + + assert set(found) == {"Jebel-Quant/rhiza"} + + +def test_a_listed_repo_without_a_checkout_is_a_quiet_no_op(tmp_path): + """You may monitor a repo you have never cloned; the GitHub half still works.""" + make_checkout(tmp_path, "Jebel-Quant", "rhiza") + + found = localgit.scan(config(("Jebel-Quant/rhiza", "Jebel-Quant/actions"), tmp_path), {}) + + assert set(found) == {"Jebel-Quant/rhiza"} + + +def test_a_checkout_of_the_wrong_repo_is_refused(tmp_path, caplog): + """A bad path in repos.yml must not file one repo's dirt under another's name.""" + # Mounted where Jebel-Quant/rhiza is expected, but it is really cvxgrp/cvxpy. + make_checkout(tmp_path, "Jebel-Quant", "rhiza", origin="git@github.com:cvxgrp/cvxpy.git") + + found = localgit.scan(config(("Jebel-Quant/rhiza",), tmp_path), {}) + + assert found == {} + assert "check repos.yml" in caplog.text + + +def test_an_ignored_repo_is_skipped_even_when_checked_out(tmp_path): + make_checkout(tmp_path, "Jebel-Quant", "rhiza") + cfg = config(("Jebel-Quant/rhiza",), tmp_path) + + found = localgit.scan(cfg, {}, skip=frozenset({"Jebel-Quant/rhiza"})) + + assert found == {} + + +def test_the_checkout_is_read_at_the_canonical_path(tmp_path): + """//, which is exactly where the override mounts it.""" + path = make_checkout(tmp_path, "cvxgrp", "cvxsimulator") + + found = localgit.scan(config(("cvxgrp/cvxsimulator",), tmp_path), {}) + + assert found["cvxgrp/cvxsimulator"].path == str(path) + assert found["cvxgrp/cvxsimulator"].branch == "main" + + +# -- the GitHub half --------------------------------------------------------- + + +def test_list_repos_asks_only_for_listed_repos(make_client, cfg): + """No /orgs//repos call: GitHub is never asked what the fleet contains.""" + object.__setattr__(cfg, "repos", ("Jebel-Quant/rhiza", "cvxgrp/cvxsimulator")) + client = make_client( + { + "/repos/Jebel-Quant/rhiza": {"full_name": "Jebel-Quant/rhiza"}, + "/repos/cvxgrp/cvxsimulator": {"full_name": "cvxgrp/cvxsimulator"}, + } + ) + + repos = client.list_repos() + + assert [r["full_name"] for r in repos] == ["Jebel-Quant/rhiza", "cvxgrp/cvxsimulator"] + assert not [c for c in client.calls if c.startswith("/orgs/")] + + +def test_an_unreadable_repo_does_not_lose_the_others(make_client, cfg, caplog): + """One typo in the list must cost one row, not the whole board.""" + object.__setattr__(cfg, "repos", ("Jebel-Quant/typo", "Jebel-Quant/rhiza")) + client = make_client({"/repos/Jebel-Quant/rhiza": {"full_name": "Jebel-Quant/rhiza"}}) + + repos = client.list_repos() + + assert [r["full_name"] for r in repos] == ["Jebel-Quant/rhiza"] + assert "Jebel-Quant/typo" in caplog.text + + +# -- the generator ----------------------------------------------------------- + + +@pytest.fixture +def gen_repos(): + """scripts/gen-repos.py, loaded by path - it is a script, not a package.""" + pytest.importorskip("yaml") + spec = importlib.util.spec_from_file_location( + "gen_repos", REPO_ROOT / "scripts" / "gen-repos.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_the_generator_names_a_checkout_from_its_origin(gen_repos, tmp_path): + path = make_checkout(tmp_path, "cvxgrp", "cvxsimulator") + + assert gen_repos.resolve({"path": str(path)}, 1) == ("cvxgrp/cvxsimulator", path) + + +def test_an_explicit_repo_overrides_the_origin(gen_repos, tmp_path): + """For a fork you want the board to follow upstream, not your copy.""" + path = make_checkout(tmp_path, "me", "cvxpy") + + assert gen_repos.resolve({"path": str(path), "repo": "cvxpy/cvxpy"}, 1) == ( + "cvxpy/cvxpy", + path, + ) + + +def test_an_entry_may_name_a_repo_with_no_checkout(gen_repos): + assert gen_repos.resolve({"repo": "Jebel-Quant/actions"}, 1) == ("Jebel-Quant/actions", None) + + +def test_a_bare_string_entry_is_a_path(gen_repos, tmp_path): + path = make_checkout(tmp_path, "Jebel-Quant", "rhiza") + + assert gen_repos.resolve(str(path), 1) == ("Jebel-Quant/rhiza", path) + + +@pytest.mark.parametrize( + "entry", + [ + {"repo": "no-slash"}, # not owner/name, and no path to derive it from + {}, # neither + {"path": "/definitely/not/here"}, + ], +) +def test_a_broken_entry_fails_loudly(gen_repos, entry): + """Better a refusal at generate time than a board that is quietly short a repo.""" + with pytest.raises(SystemExit): + gen_repos.resolve(entry, 1) + + +def test_a_directory_that_is_not_a_checkout_fails(gen_repos, tmp_path): + (tmp_path / "empty").mkdir() + with pytest.raises(SystemExit): + gen_repos.resolve({"path": str(tmp_path / "empty")}, 1) diff --git a/docker-compose.server.yml b/docker-compose.server.yml index 7912499..8aa02a7 100644 --- a/docker-compose.server.yml +++ b/docker-compose.server.yml @@ -7,7 +7,9 @@ # strangers can reach it": # # * no bind mount and JQ_REPO_ROOT empty, so local scanning is skipped -# entirely rather than erroring once a minute +# entirely rather than erroring once a minute. There is no repos.yml here +# either: with no checkouts to point at, the fleet is named directly in +# JQ_REPOS as a comma-separated list of owner/name. # * JQ_PUBLIC_ONLY is forced on, so private repos are never gathered at all # * only Grafana publishes a port; Prometheus and the collector talk over the # compose network and are unreachable from outside @@ -25,8 +27,9 @@ services: restart: unless-stopped environment: GITHUB_TOKEN: ${GITHUB_TOKEN:?set GITHUB_TOKEN in the environment or a .env file} - JQ_ORGS: ${JQ_ORGS:-Jebel-Quant} - JQ_REPOS: ${JQ_REPOS:-} + # The fleet, as owner/name. Explicit here as on a laptop - the only + # difference is that a server has no checkouts to derive it from. + JQ_REPOS: ${JQ_REPOS:?set JQ_REPOS to a comma-separated list of owner/name} JQ_TEMPLATE_REPO: ${JQ_TEMPLATE_REPO:-Jebel-Quant/rhiza} JQ_GITHUB_INTERVAL: ${JQ_GITHUB_INTERVAL:-300} # No clones on a server. Empty means "skip local scanning". diff --git a/docker-compose.yml b/docker-compose.yml index 80e61e1..8659989 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,9 @@ +# The laptop stack. Not usable on its own - the fleet lives in repos.yml, +# so bring it up with scripts/up.sh, or with: +# +# python3 scripts/gen-repos.py +# docker compose -f docker-compose.yml -f docker-compose.repos.yml up -d + name: jq-monitoring services: @@ -7,24 +13,21 @@ services: restart: unless-stopped environment: GITHUB_TOKEN: ${GITHUB_TOKEN:?set GITHUB_TOKEN in monitoring/.env - scripts/up.sh does it for you} - # Whole-org sweeps, plus individually named repos for orgs where only - # some of the repos are yours. - JQ_ORGS: ${JQ_ORGS:-Jebel-Quant} - JQ_REPOS: ${JQ_REPOS:-} + # JQ_REPOS and the per-repo bind mounts both come from + # docker-compose.repos.yml, generated from repos.yml. This file has no + # opinion about which repos are yours. JQ_REPO_ROOT: /repos - JQ_SCAN_DEPTH: ${JQ_SCAN_DEPTH:-2} JQ_TEMPLATE_REPO: ${JQ_TEMPLATE_REPO:-Jebel-Quant/rhiza} JQ_GITHUB_INTERVAL: ${JQ_GITHUB_INTERVAL:-300} JQ_LOCAL_INTERVAL: ${JQ_LOCAL_INTERVAL:-60} JQ_IGNORE: ${JQ_IGNORE:-} JQ_PUBLIC_ONLY: ${JQ_PUBLIC_ONLY:-false} JQ_LOG_LEVEL: ${JQ_LOG_LEVEL:-INFO} - volumes: - # The whole of ~/repos, so clones under any org folder are visible; the - # scan keeps only repos that are in scope. Read-only on purpose: the - # collector reports on your working copies and must never be able to touch - # them. Every git call is --no-optional-locks for the same reason. - - ${JQ_REPO_ROOT_HOST:-../..}:/repos:ro + # No volumes here on purpose. Every checkout is mounted individually by + # docker-compose.repos.yml, read-only: the collector reports on your working + # copies and must never be able to touch them, and mounting only what is + # listed means an unlisted repo is not merely filtered out - it is not even + # visible to the container. Every git call is --no-optional-locks too. ports: - "127.0.0.1:9109:9109" diff --git a/repos.example.yml b/repos.example.yml new file mode 100644 index 0000000..a55d074 --- /dev/null +++ b/repos.example.yml @@ -0,0 +1,28 @@ +# The fleet. Copy to repos.yml and edit - repos.yml is gitignored, because the +# folder layout of your machine is nobody else's business. +# +# One entry per monitored repo. `scripts/gen-repos.py` reads this file and +# writes docker-compose.repos.yml, which mounts each checkout read-only at +# /repos// and hands the collector the matching JQ_REPOS list. +# scripts/up.sh runs it for you; run it again after editing this file. +# +# Nothing is discovered. A repo appears on the board because it is listed here, +# and for no other reason. + +repos: + # The usual entry: a path to a checkout. `~` and paths relative to this file + # both work, and owner/name is read from the checkout's origin remote. + - path: ~/repos/jebel-quant/monitoring + - path: ~/repos/jebel-quant/rhiza + + # Only some of a large shared org is yours, so name those repos one by one. + - path: ~/repos/cvxgrp/cvxsimulator + + # Override the origin when the checkout is a fork and you want the board to + # follow upstream rather than your copy. + # - path: ~/repos/forks/cvxpy + # repo: cvxpy/cvxpy + + # A repo with no checkout on this machine. GitHub panels (CI, pull requests, + # template drift) are gathered; the working-copy panels stay empty for it. + # - repo: Jebel-Quant/actions diff --git a/scripts/down.sh b/scripts/down.sh index 5e9fee2..7042271 100755 --- a/scripts/down.sh +++ b/scripts/down.sh @@ -2,4 +2,8 @@ # Stop the stack. Add --volumes to also throw away the Prometheus history. set -euo pipefail cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -docker compose down "$@" +# The generated override may be missing on a fresh clone; the project name in +# docker-compose.yml is enough to find the running containers either way. +files=(-f docker-compose.yml) +[[ -f docker-compose.repos.yml ]] && files+=(-f docker-compose.repos.yml) +docker compose "${files[@]}" down "$@" diff --git a/scripts/gen-repos.py b/scripts/gen-repos.py new file mode 100755 index 0000000..414f3cf --- /dev/null +++ b/scripts/gen-repos.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Turn repos.yml into the compose override that mounts the fleet. + +The collector never searches for checkouts. It expects each repo at +/repos//, so this script resolves every listed path, asks its +origin remote who it is, and writes one read-only bind mount per repo plus the +matching JQ_REPOS list. Both halves of the collector then read the same list, +and the board's contents are decided by repos.yml alone. + +Run it after editing repos.yml; scripts/up.sh does that for you. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SOURCE = ROOT / "repos.yml" +TARGET = ROOT / "docker-compose.repos.yml" + + +def fail(message: str) -> None: + print(f"gen-repos: {message}", file=sys.stderr) + sys.exit(1) + + +def origin_owner_name(path: Path) -> tuple[str, str] | None: + """The ``(owner, name)`` a checkout's origin points at.""" + try: + url = subprocess.run( + ["git", "--no-optional-locks", "-C", str(path), "remote", "get-url", "origin"], + capture_output=True, + text=True, + timeout=20, + check=False, + ).stdout.strip() + except (OSError, subprocess.TimeoutExpired): + return None + if not url: + return None + url = url.removesuffix(".git") + if url.startswith("git@"): + tail = url.partition(":")[2] + elif "://" in url: + tail = url.split("://", 1)[1].split("/", 1)[-1] + else: + tail = url + parts = [p for p in tail.split("/") if p] + return (parts[-2], parts[-1]) if len(parts) >= 2 else None + + +def load_entries() -> list[dict]: + if not SOURCE.exists(): + fail(f"{SOURCE.name} not found - copy repos.example.yml to repos.yml and list your repos") + try: + import yaml + except ModuleNotFoundError: + fail( + "PyYAML is not installed - `uv run --with pyyaml scripts/gen-repos.py`, or pip install pyyaml" + ) + data = yaml.safe_load(SOURCE.read_text(encoding="utf-8")) or {} + entries = data.get("repos") if isinstance(data, dict) else None + if not isinstance(entries, list) or not entries: + fail(f"{SOURCE.name} lists no repos under a top-level `repos:` key") + return entries + + +def resolve(entry: object, index: int) -> tuple[str, Path | None]: + """One entry -> ``(owner/name, checkout path or None)``.""" + # `- ~/repos/foo` is accepted as shorthand for `- path: ~/repos/foo`. + if isinstance(entry, str): + entry = {"path": entry} + if not isinstance(entry, dict): + fail(f"entry {index} is neither a path nor a mapping: {entry!r}") + + named = str(entry.get("repo") or "").strip() + raw_path = entry.get("path") + + if raw_path is None: + if "/" not in named: + fail(f"entry {index} needs a `path`, or a `repo:` of the form owner/name") + return named, None + + # Relative paths are relative to this repo, so a checked-in repos.yml on a + # colleague's machine means the same thing as it does here. + path = Path(os.path.expanduser(str(raw_path))) + path = (path if path.is_absolute() else ROOT / path).resolve() + + if not path.is_dir(): + fail(f"{raw_path} is not a directory") + # `.git` is a directory in a plain checkout and a file in a worktree. + if not (path / ".git").exists(): + fail(f"{path} is not a git checkout") + + if "/" in named: + return named, path + + origin = origin_owner_name(path) + if origin is None: + fail(f"{path} has no usable origin remote - add an explicit `repo: owner/name`") + return f"{origin[0]}/{origin[1]}", path + + +def main() -> None: + resolved: dict[str, Path | None] = {} + for index, entry in enumerate(load_entries(), start=1): + full_name, path = resolve(entry, index) + if full_name in resolved: + fail(f"{full_name} is listed twice") + resolved[full_name] = path + + lines = [ + "# Generated by scripts/gen-repos.py from repos.yml - do not edit.", + "#", + "# One read-only bind mount per monitored checkout, at the canonical", + "# /repos// the collector looks for, plus the matching", + "# JQ_REPOS list so the GitHub half sees exactly the same fleet.", + "", + "services:", + " collector:", + " environment:", + f' JQ_REPOS: "{",".join(resolved)}"', + ] + mounts = {name: path for name, path in resolved.items() if path is not None} + if mounts: + lines.append(" volumes:") + lines += [f' - "{path}:/repos/{name}:ro"' for name, path in mounts.items()] + TARGET.write_text("\n".join(lines) + "\n", encoding="utf-8") + + without = len(resolved) - len(mounts) + tail = f", {without} without a local checkout" if without else "" + print(f"gen-repos: wrote {TARGET.name} - {len(resolved)} repos{tail}") + + +if __name__ == "__main__": + main() diff --git a/scripts/up.sh b/scripts/up.sh index ee19383..ac0579f 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -10,6 +10,13 @@ if [[ ! -f .env ]]; then echo "created monitoring/.env from the example" fi +if [[ ! -f repos.yml ]]; then + cp repos.example.yml repos.yml + echo "created monitoring/repos.yml from the example - EDIT IT, then run this again" + echo "it lists the checkouts to monitor; the example points at paths you may not have" + exit 1 +fi + # Only fill the token if it is still blank - never clobber one you set by hand. if ! grep -qE '^GITHUB_TOKEN=.+' .env; then if ! command -v gh >/dev/null 2>&1; then @@ -24,7 +31,15 @@ if ! grep -qE '^GITHUB_TOKEN=.+' .env; then echo "wrote a token from 'gh auth token' into monitoring/.env" fi -docker compose up -d --build +# repos.yml is the fleet. Regenerate the mounts every time, so editing the list +# and running ./scripts/up.sh is the whole workflow. +if command -v uv >/dev/null 2>&1; then + uv run --quiet --with pyyaml scripts/gen-repos.py +else + python3 scripts/gen-repos.py +fi + +docker compose -f docker-compose.yml -f docker-compose.repos.yml up -d --build echo echo "Grafana http://localhost:3000/d/jq-fleet (anonymous read-only; admin/admin to edit)"