List the monitored repos explicitly, in repos.yml

.env.example

@@ -1,27 +1,22 @@
 # Copy to .env, or let scripts/up.sh generate it from `gh auth token`.
+#
+# Which repos are monitored is NOT set here - that lives in repos.yml, one
+# entry per checkout. This file holds credentials and cadences only.
 
 # Needs `repo` (private repo metadata) and `read:org`. A gh OAuth token works.
+# It has to be able to read every repo listed in repos.yml.
 GITHUB_TOKEN=
 
-# The org whose repos are monitored. Clones whose origin points elsewhere - an
-# upstream numpy checkout, say - are skipped automatically.
-JQ_ORG=Jebel-Quant
-
 # The repo whose releases define "up to date" for the template pointer.
-JQ_TEMPLATE_REPO=rhiza
+JQ_TEMPLATE_REPO=Jebel-Quant/rhiza
 
-# Comma-separated repo names to leave out entirely.
+# Comma-separated repo names to drop without removing them from repos.yml.
 JQ_IGNORE=
 
 # Drop private repos completely - their names, workflows, PR titles and local
 # branches are all disclosure. Set this before serving the board publicly.
 JQ_PUBLIC_ONLY=true
 
-# Where your clones live on the host, relative to this file or absolute.
-# Scanned to JQ_SCAN_DEPTH levels, so ~/repos/<org>/<repo> is found.
-JQ_REPO_ROOT_HOST=../..
-JQ_SCAN_DEPTH=2
-
 # Refresh cadences, in seconds.
 JQ_GITHUB_INTERVAL=300
 JQ_LOCAL_INTERVAL=60

.github/workflows/ci.yml

@@ -59,19 +59,40 @@ jobs:
     name: compose files parse
     runs-on: ubuntu-latest
     # The overlays only patch services, so they are not valid on their own -
-    # they are checked the way they are actually used. GITHUB_TOKEN is a
-    # required variable in the base file; any value satisfies interpolation.
+    # they are checked the way they are actually used. GITHUB_TOKEN and, for
+    # the server stack, JQ_REPOS are required variables; any value satisfies
+    # interpolation.
     env:
       GITHUB_TOKEN: dummy-value-for-interpolation-only
       GF_ADMIN_PASSWORD: dummy-value-for-interpolation-only
+      JQ_REPOS: Jebel-Quant/rhiza
     steps:
       - uses: actions/checkout@v4
-      - name: Base
-        run: docker compose -f docker-compose.yml config --quiet
+
+      # repos.yml is gitignored, so CI builds one the way a new user would and
+      # proves gen-repos.py emits an override that actually merges. Nothing
+      # else in CI reads it, so a malformed one would otherwise only surface
+      # on somebody's laptop.
+      - name: Generate the fleet override
+        run: |
+          pip install --quiet pyyaml
+          printf 'repos:\n' > repos.yml
+          for full in Jebel-Quant/rhiza cvxgrp/cvxsimulator; do
+            git init -q -b main "ci-checkouts/$full"
+            git -C "ci-checkouts/$full" remote add origin "https://github.com/$full.git"
+            printf '  - path: ci-checkouts/%s\n' "$full" >> repos.yml
+          done
+          # A repo with no checkout: GitHub panels only, no bind mount.
+          printf '  - repo: Jebel-Quant/actions\n' >> repos.yml
+          python3 scripts/gen-repos.py
+          cat docker-compose.repos.yml
+
+      - name: Base + fleet override
+        run: docker compose -f docker-compose.yml -f docker-compose.repos.yml config --quiet
       - name: Base + admin overlay
-        run: docker compose -f docker-compose.yml -f docker-compose.admin.yml config --quiet
+        run: docker compose -f docker-compose.yml -f docker-compose.repos.yml -f docker-compose.admin.yml config --quiet
       - name: Base + public overlay
-        run: docker compose -f docker-compose.yml -f docker-compose.public.yml config --quiet
+        run: docker compose -f docker-compose.yml -f docker-compose.repos.yml -f docker-compose.public.yml config --quiet
       - name: Server stack
         run: docker compose -f docker-compose.server.yml config --quiet
       - name: Server + TLS overlay

.gitignore

@@ -1,6 +1,13 @@
 .env
 
+# Your fleet and the mounts generated from it: both describe the folder layout
+# of one machine, which has no business in a public repo. Start from
+# repos.example.yml.
+repos.yml
+docker-compose.repos.yml
+
 # Left behind by running the collector natively instead of in its container.
 .venv/
 .ruff_cache/
+.pytest_cache/
 __pycache__/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Jebel Quant Research
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -4,13 +4,35 @@ A Grafana board for the state of your repo fleet — template drift, CI on the
 default branch, open pull requests, and the working copies on this machine —
 with Prometheus keeping the history and six alert rules on top.
 
-Two kinds of repo are in scope: **whole orgs** (`Jebel-Quant`, swept in full) and
-**individually named repos** (`cvxgrp/cvxrisk`, `cvxgrp/simulator`,
-`cvxgrp/cvxcla`, `cvxgrp/cvxmarkowitz`). The second exists because cvxgrp has
-100+ repos and only four are yours.
+**The fleet is an explicit list.** `repos.yml` names every monitored repo, one
+entry per checkout on this machine:
+
+```yaml
+repos:
+  - path: ~/repos/jebel-quant/rhiza
+  - path: ~/repos/cvxgrp/cvxsimulator
+  - repo: Jebel-Quant/actions      # monitored, but not checked out here
+```
 
-**Archived repos are never monitored.** They are dropped from the GitHub sweep
-*and* their local clones are skipped, so a checkout left on disk cannot keep a
+`owner/name` is read from each checkout's `origin` remote, so the path is all
+you write. Nothing is discovered: a repo is on the board because it is in this
+file, and for no other reason. `scripts/up.sh` turns the file into
+`docker-compose.repos.yml`, which mounts each checkout **read-only** at
+`/repos/<owner>/<name>` — so an unlisted repo is not merely filtered out, it is
+never visible to the container at all.
+
+This replaced a whole-org GitHub sweep plus a directory walk under one mounted
+root. Both decided membership on their own: a new repo in the org arrived
+unasked, a shared org like cvxgrp dragged in 100+ repos that were not yours, and
+any checkout that happened to sit under the root joined the board because its
+origin looked right.
+
+Edit `repos.yml`, run `./scripts/up.sh` again, and the fleet is whatever you
+just wrote. Both halves of the collector read the same list, so the GitHub
+panels and the working-copy panels can never disagree about who is in scope.
+
+**Archived repos are never monitored.** They are dropped from the GitHub half
+*and* their local checkouts are skipped, so a checkout left on disk cannot keep a
 dead repo on the board — that gap kept `rhiza-brainbug` showing as the fleet's
 one red repo for a while after it was archived. Set `JQ_INCLUDE_ARCHIVED=true`
 to opt back in.
@@ -37,10 +59,15 @@ path. The script reports both numbers so the difference is visible rather than
 alarming.
 
 ```bash
+cp repos.example.yml repos.yml   # then list your checkouts
 ./scripts/up.sh     # builds, mints a token from `gh auth token`, starts everything
 ./scripts/down.sh   # stop; add --volumes to discard the history too
 ```
 
+`up.sh` creates `repos.yml` from the example on a first run and stops so you can
+edit it. Both `repos.yml` and the generated `docker-compose.repos.yml` are
+gitignored: they describe the folder layout of one machine.
+
 | | |
 |---|---|
 | Dashboard | <http://localhost:3000/d/jq-fleet> |
@@ -207,20 +234,35 @@ your checkout is behind, not the repo.
 
 ## Configuration
 
-Everything lives in `.env` (see `.env.example`). The useful knobs:
+**Which repos** is `repos.yml`. **Everything else** is `.env` (see
+`.env.example`).
+
+### repos.yml
+
+| Key | | |
+|---|---|---|
+| `path` | | A checkout on this machine. `~` and paths relative to the repo both work. |
+| `repo` | | `owner/name`. Optional next to a `path` — it overrides the origin, which is what you want for a fork whose board should follow upstream. On its own it monitors a repo you have not cloned: GitHub panels are gathered, the working-copy panels stay empty for that row. |
+
+A bare string is shorthand for `path`. Duplicate entries, a path that is not a
+checkout, and an entry with neither key are all refused at generate time —
+better a refusal than a board that is quietly one repo short.
+
+### .env
 
 | Variable | Default | |
 |---|---|---|
-| `JQ_ORGS` | `Jebel-Quant` | Orgs swept in full, comma-separated. |
-| `JQ_REPOS` | the four cvxgrp repos | Individually named repos, as `owner/name`. Use this for orgs where only some repos are yours. |
+| `GITHUB_TOKEN` | — | Must be able to read every repo in `repos.yml`. `up.sh` mints one from `gh auth token`. |
 | `JQ_TEMPLATE_REPO` | `Jebel-Quant/rhiza` | Whose releases define "up to date", as `owner/name`. |
-| `JQ_IGNORE` | — | Repos to leave out, as bare names or `owner/name`. Applies to both the GitHub sweep and the local scan. |
-| `JQ_INCLUDE_ARCHIVED` | `false` | Archived repos are dropped from both the GitHub sweep and the local scan. |
-| `JQ_REPO_ROOT_HOST` | `../..` | Which host folder to mount. Defaults to all of `~/repos`. |
-| `JQ_SCAN_DEPTH` | `2` | How deep to look for clones, so `~/repos/<org>/<repo>` is found. |
+| `JQ_IGNORE` | — | Repos to drop without editing `repos.yml`, as bare names or `owner/name`. Applies to both halves. |
+| `JQ_INCLUDE_ARCHIVED` | `false` | Archived repos are dropped from both halves. |
+| `JQ_PUBLIC_ONLY` | `false` | Drop private repos entirely — not just their details, their existence. |
 | `JQ_GITHUB_INTERVAL` | `300` | Seconds between GitHub refreshes. |
 | `PROM_RETENTION` | `180d` | How much history to keep. |
 
+On the server stack there are no checkouts and so no `repos.yml`: the fleet is
+named directly in `JQ_REPOS`, a comma-separated list of `owner/name`.
+
 ### API budget
 
 A refresh costs roughly `3 × repos + open PRs` REST calls. Measured on this
@@ -308,6 +350,7 @@ copies here, and strangers can reach it":
 | | |
 |---|---|
 | `JQ_REPO_ROOT` empty | local scanning skipped entirely — a clean no-op, not an error every minute |
+| the fleet is `JQ_REPOS` | no checkouts to derive it from, so the list is named directly rather than in `repos.yml` |
 | `JQ_PUBLIC_ONLY` forced on | private repos are never gathered, so they cannot leak |
 | only Grafana publishes a port | Prometheus and the collector talk over the compose network and are unreachable from outside |
 | anonymous off, public dashboards on | a public link serves one dashboard's queries with no datasource behind it |
@@ -331,9 +374,9 @@ GITHUB_TOKEN=github_pat_...        # public_repo scope is enough
 GF_ADMIN_PASSWORD=...              # 16+ chars; openssl rand -base64 24
 FLEET_DOMAIN=fleet.example.com
 ACME_EMAIL=you@example.com
-# Repos outside the swept org. Leave this out and they are silently absent -
-# the board simply reports a smaller fleet, with nothing to say it is short.
-JQ_REPOS=cvxgrp/cvxrisk,cvxgrp/simulator,cvxgrp/cvxcla,cvxgrp/cvxmarkowitz
+# The fleet. Required - there are no checkouts here to derive it from, and the
+# stack refuses to start without it rather than serving an empty board.
+JQ_REPOS=Jebel-Quant/rhiza,Jebel-Quant/actions,cvxgrp/cvxsimulator
 SETTINGS
 chmod 600 .env
 
@@ -345,7 +388,8 @@ it automatically, and it is gitignored.
 
 Note the token has to be able to read every repo named in `JQ_REPOS`. A
 fine-grained token scoped to one org cannot see another's, and the collector
-logs `named repo ... is not readable` when that happens.
+logs `listed repo ... is not readable` when that happens — one unreadable entry
+costs one row, not the whole board.
 
 Then reach Grafana, sign in as `admin`, open **Jebel-Quant Fleet (public)** →
 *Share* → *Public dashboard*, and share only that link.

collector/jq_collector/config.py

@@ -24,23 +24,29 @@ def _csv(name: str, default: tuple[str, ...] = ()) -> tuple[str, ...]:
 class Config:
     """Everything the collector needs to know about its environment.
 
-    Repos arrive two ways: whole-org sweeps (``JQ_ORGS``) and individually named
-    repos (``JQ_REPOS``). The second exists because you rarely want *all* of a
-    large shared org - cvxgrp has 100+ repos and only a handful are yours.
+    The fleet is an explicit list: ``JQ_REPOS`` names every monitored repo as
+    ``owner/name``, and nothing else is ever gathered. There used to be a
+    whole-org sweep as well, which meant the board's contents were decided by
+    GitHub rather than by you - a new repo in the org appeared unasked, and a
+    shared org like cvxgrp dragged in a hundred repos that were not yours.
+
+    On a laptop the list is generated from ``repos.yml`` by
+    ``scripts/gen-repos.py``, which also mounts each checkout at
+    ``$JQ_REPO_ROOT/<owner>/<name>``; on a server it is set by hand and no
+    checkouts exist. Both halves read the same list, so the GitHub panels and
+    the working-copy panels can never disagree about who is in the fleet.
     """
 
-    orgs: tuple[str, ...] = field(default_factory=lambda: _csv("JQ_ORGS", ("Jebel-Quant",)))
-    extra_repos: tuple[str, ...] = field(default_factory=lambda: _csv("JQ_REPOS"))
+    repos: tuple[str, ...] = field(default_factory=lambda: _csv("JQ_REPOS"))
 
     token: str = os.environ.get("GITHUB_TOKEN", "")
     api: str = os.environ.get("GITHUB_API", "https://api.github.com")
 
-    # Where the clones are mounted (read-only) inside the container. Scanned to
-    # a depth of two, so both ~/repos/<repo> and ~/repos/<org>/<repo> are found.
-    # Set empty to skip local scanning entirely - the right setting on a server,
-    # where there are no working copies and the local panels do not apply.
+    # Where the checkouts are mounted (read-only) inside the container, one per
+    # repo at <repo_root>/<owner>/<name>. Set empty to skip local scanning
+    # entirely - the right setting on a server, where there are no working
+    # copies and the local panels do not apply.
     repo_root: str = os.environ.get("JQ_REPO_ROOT", "/repos")
-    scan_depth: int = _int("JQ_SCAN_DEPTH", 2)
 
     # owner/name of the repo whose releases define "up to date".
     template_repo: str = os.environ.get("JQ_TEMPLATE_REPO", "Jebel-Quant/rhiza")
@@ -63,7 +69,10 @@ class Config:
     # could crowd out entries the fleet-wide list should have shown.
     recent_merges_per_repo: int = _int("JQ_RECENT_MERGES_PER_REPO", 10)
 
-    # Repos to leave out, as bare names or as owner/name.
+    # Repos to leave out, as bare names or as owner/name. Redundant now that
+    # the fleet is an explicit list - deleting the line from repos.yml is the
+    # obvious move - but it stays for the server, where the list is an env var
+    # and commenting one entry out is not possible.
     ignore: tuple[str, ...] = field(default_factory=lambda: _csv("JQ_IGNORE"))
 
     include_archived: bool = os.environ.get("JQ_INCLUDE_ARCHIVED", "false").lower() == "true"
@@ -73,19 +82,5 @@ class Config:
     # names, its PR titles and its local branch names are all disclosure.
     public_only: bool = os.environ.get("JQ_PUBLIC_ONLY", "false").lower() == "true"
 
-    @property
-    def owners(self) -> frozenset[str]:
-        """Every owner we might accept a clone from, lowercased."""
-        from_extras = (r.split("/", 1)[0] for r in self.extra_repos if "/" in r)
-        return frozenset(o.lower() for o in (*self.orgs, *from_extras))
-
     def is_ignored(self, owner: str, name: str) -> bool:
         return name in self.ignore or f"{owner}/{name}" in self.ignore
-
-    def wants(self, owner: str, name: str) -> bool:
-        """Is this repo in scope - by its org, or by being named explicitly?"""
-        if self.is_ignored(owner, name):
-            return False
-        if owner.lower() in {o.lower() for o in self.orgs}:
-            return True
-        return f"{owner}/{name}".lower() in {r.lower() for r in self.extra_repos}

collector/jq_collector/github.py

@@ -107,31 +107,25 @@ def _paginate(self, path: str, **params: object) -> list[dict]:
     # -- fleet-level -----------------------------------------------------
 
     def list_repos(self) -> list[dict]:
-        """Every in-scope repo: whole-org sweeps plus individually named ones."""
+        """The repos named in the config, in the order they were listed.
+
+        One call each, and no org sweep: the fleet is whatever you wrote down.
+        A repo that cannot be read is dropped with a warning rather than
+        failing the refresh, so one bad line does not blank the whole board.
+        """
         repos: list[dict] = []
         seen: set[str] = set()
 
-        for org in self._cfg.orgs:
-            found = self._paginate(f"/orgs/{org}/repos", type="all", sort="full_name")
-            if not found:
-                # A user account rather than an org, or no org read access.
-                owned = self._paginate("/user/repos", affiliation="owner", sort="full_name")
-                found = [r for r in owned if (r.get("owner") or {}).get("login") == org]
-            for raw in found:
-                if raw.get("full_name") not in seen:
-                    seen.add(raw["full_name"])
-                    repos.append(raw)
-
-        for full_name in self._cfg.extra_repos:
-            if full_name in seen or "/" not in full_name:
+        for full_name in self._cfg.repos:
+            if "/" not in full_name or full_name in seen:
                 continue
             raw = self._json(f"/repos/{full_name}")
-            if isinstance(raw, dict):
+            if isinstance(raw, dict) and raw.get("full_name"):
                 seen.add(raw["full_name"])
                 repos.append(raw)
             else:
                 log.warning(
-                    "named repo %s is not readable - check the token's scopes",
+                    "listed repo %s is not readable - check the name and the token's scopes",
                     full_name,
                 )