From f719c7b828e83cce0a0225df1f3f41248918af9c Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Thu, 10 Sep 2026 15:32:54 -0700 Subject: [PATCH 1/3] feat(run-task): apply git sparse-checkout patterns from the task environment The patterns travel in `_SPARSE_PATTERNS` as a JSON list of native git patterns, so the cloned revision does not need to contain the profile. A JSON list because generic-worker on Windows sets task environment variables through a batch `set` line, which cuts a value at its first newline. The option and the variable have to come together, and a checkout is required for either. Patterns go through stdin so a name starting with a dash is not taken for an option. A full task on a sparse cache drops the sparse checkout first. On the Mercurial path the option goes to robustcheckout as --sparseprofile. A sparse clone is blobless (--filter=blob:none) and defers its checkout until the patterns are set, so only files inside the profile are written or fetched. Git keeps that filter on the origin remote and a fetch only inherits it when it names that remote, so run-task fetches through origin whenever the head repository is the repository it cloned, and points origin at the task's repository first so a cached clone follows a repository that moved. git 2.49 keeps the filter for a fetch by matching URL as well, but that is not documented and the workers run other versions. Refs #1025 --- src/taskgraph/run-task/run-task | 187 ++++++++++++++++++++++++++++---- test/test_scripts_run_task.py | 154 ++++++++++++++++++++++++++ 2 files changed, 317 insertions(+), 24 deletions(-) diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index 34937bec4..efa28d9a5 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -32,7 +32,7 @@ import time import urllib.error import urllib.request from pathlib import Path -from typing import Dict, Optional +from typing import Dict, List, NamedTuple, Optional SECRET_BASEURL_TPL = "{}/secrets/v1/secret/{{}}".format( os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip("/") @@ -234,8 +234,8 @@ def remove(path): _call_windows_retry(shutil.rmtree, (path,)) -def run_required_command(prefix, args, *, extra_env=None, cwd=None): - res = run_command(prefix, args, extra_env=extra_env, cwd=cwd) +def run_required_command(prefix, args, *, extra_env=None, cwd=None, stdin_data=None): + res = run_command(prefix, args, extra_env=extra_env, cwd=cwd, stdin_data=stdin_data) if res: sys.exit(res) @@ -253,7 +253,7 @@ def retry_required_command(prefix, args, *, extra_env=None, cwd=None, retries=2) time.sleep(backoff) -def run_command(prefix, args, *, extra_env=None, cwd=None): +def run_command(prefix, args, *, extra_env=None, cwd=None, stdin_data=None): """Runs a process and prefixes its output with the time. Returns the process exit code. @@ -282,11 +282,15 @@ def run_command(prefix, args, *, extra_env=None, cwd=None): bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - stdin=sys.stdin.fileno(), + stdin=subprocess.PIPE if stdin_data is not None else sys.stdin.fileno(), cwd=cwd, env=env, ) + if stdin_data is not None: + p.stdin.write(stdin_data) + p.stdin.close() + stdout = io.TextIOWrapper(p.stdout, encoding="latin1") if os.getpid() == 1: @@ -571,8 +575,9 @@ def configure_volume_posix(volume, user, group, running_as_root): class PerfRecorder: """Collects operation timings and emits them as PERFHERDER_DATA.""" - def __init__(self, framework: str): + def __init__(self, framework: str, extra_options: Optional[List[str]] = None): self.framework = framework + self.extra_options = list(extra_options or []) self.optimes = [] self._stack = [] @@ -610,7 +615,7 @@ class PerfRecorder: "value": duration, "lowerIsBetter": True, "shouldAlert": False, - "extraOptions": [instance_type], + "extraOptions": [instance_type, *self.extra_options], "subtests": [], } ) @@ -673,6 +678,52 @@ def _clean_git_checkout(destination_path): print_line(b"vcs", b"successfully cleaned git checkout!\n") +def _git_config(destination_path: str, key: str) -> Optional[str]: + p = subprocess.run( + ["git", "config", "--get", key], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=destination_path, + env=os.environ, + encoding="utf-8", + ) + if p.returncode == 1: + return None + if p.returncode != 0: + raise RuntimeError(f"git config --get {key} failed: {p.stderr.strip()}") + return p.stdout.strip() + + +class SparsePlan(NamedTuple): + """The one `git sparse-checkout` command a checkout needs, if any.""" + + op: Optional[str] + args: List[str] + entries: List[str] + message: str + + +def _plan_sparse_checkout( + is_sparse: bool, + sparse_profile: Optional[str], + sparse_patterns: Optional[List[str]], +) -> SparsePlan: + profile = sparse_profile or "" + if not sparse_patterns: + if is_sparse: + return SparsePlan( + "sparse_disable", + ["git", "sparse-checkout", "disable"], + [], + "widening the sparse checkout to the full tree", + ) + return SparsePlan(None, [], [], "") + + args = ["git", "sparse-checkout", "set", "--no-cone", "--stdin"] + message = f"sparse_set: sparse profile {profile}, {len(sparse_patterns)} entries" + return SparsePlan("sparse_set", args, sparse_patterns, message) + + def shortref(ref: str) -> str: """Normalize a git ref to its short form. @@ -696,6 +747,8 @@ def git_checkout( ssh_key_file: Optional[Path], ssh_known_hosts_file: Optional[Path], shallow: bool = False, + sparse_profile: Optional[str] = None, + sparse_patterns: Optional[List[str]] = None, ): assert head_ref or head_rev @@ -725,20 +778,28 @@ def git_checkout( ) has_repo = os.path.exists(destination_path) - perf = PerfRecorder("vcs") - with perf.record("overall", "overall_pull" if has_repo else "overall_clone"): - # Bypass Git's "safe directory" feature as the destination could be - # coming from a cache and therefore cloned by a different user. - args = [ - "git", - "config", - "--global", - "--add", - "safe.directory", - Path(destination_path).as_posix(), - ] - retry_required_command(b"vcs", args, extra_env=env) + # Bypass Git's "safe directory" feature as the destination could be + # coming from a cache and therefore cloned by a different user. + args = [ + "git", + "config", + "--global", + "--add", + "safe.directory", + Path(destination_path).as_posix(), + ] + retry_required_command(b"vcs", args, extra_env=env) + + is_sparse = ( + has_repo and _git_config(destination_path, "core.sparseCheckout") == "true" + ) + sparse = bool(sparse_patterns) + + action = "pull" if has_repo else "clone" + mode = "sparsecheckout" if sparse else "fullcheckout" + perf = PerfRecorder("vcs") + with perf.record("overall", f"overall_{action}", f"overall_{action}_{mode}"): if not has_repo: # Repository doesn't already exist, needs to be cloned args = [ @@ -747,7 +808,13 @@ def git_checkout( ] if shallow: - args.extend(["--depth=1", "--no-checkout"]) + args.append("--depth=1") + + if sparse: + args.append("--filter=blob:none") + + if shallow or sparse: + args.append("--no-checkout") args.extend( [ @@ -758,6 +825,11 @@ def git_checkout( with perf.record("clone"): retry_required_command(b"vcs", args, extra_env=env) + else: + # Fetches go through origin by name, so a cached clone has to + # point origin at the repository this task wants. + args = ["git", "remote", "set-url", "origin", base_repo or head_repo] + run_required_command(b"vcs", args, cwd=destination_path) # For Github based repos, base_rev often doesn't refer to an ancestor of # head_rev simply due to Github not providing that information in their @@ -786,16 +858,38 @@ def git_checkout( # shallow clone, head_rev needs to be fetched independently regardless. targets.append(head_rev) + # A partial clone keeps its blob filter on the origin remote, and a + # fetch only inherits the filter when it names that remote. Fetching + # the same repository by URL would download every blob of the commit. + remote = "origin" if head_repo == (base_repo or head_repo) else head_repo + with perf.record("pull"): git_fetch( destination_path, *targets, - remote=head_repo, + remote=remote, tags=tags, shallow=shallow, env=env, ) + # `git fetch` set `FETCH_HEAD` reference to the last commit of the desired branch + rev = head_rev if head_rev else "FETCH_HEAD" + + plan = _plan_sparse_checkout(is_sparse, sparse_profile, sparse_patterns) + if sparse and sparse_profile: + perf.extra_options.append(os.path.basename(sparse_profile)) + if plan.message: + print_line(b"vcs", f"{plan.message}\n".encode("utf-8")) + if plan.op: + stdin_data = None + if plan.entries: + stdin_data = ("\n".join(plan.entries) + "\n").encode("utf-8") + with perf.record(plan.op): + run_required_command( + b"vcs", plan.args, cwd=destination_path, stdin_data=stdin_data + ) + args = [ "git", "checkout", @@ -805,8 +899,7 @@ def git_checkout( if head_ref: args.extend(["-B", shortref(head_ref)]) - # `git fetch` set `FETCH_HEAD` reference to the last commit of the desired branch - args.append(head_rev if head_rev else "FETCH_HEAD") + args.append(rev) with perf.record("update"): run_required_command(b"vcs", args, cwd=destination_path) @@ -893,6 +986,7 @@ def hg_checkout( store_path: str, branch: Optional[str], revision: Optional[str], + sparse_profile: Optional[str] = None, ): if IS_MACOSX or IS_POSIX: hg_bin = "hg" @@ -916,6 +1010,9 @@ def hg_checkout( if base_repo: args.extend(["--upstream", base_repo]) + if sparse_profile: + args.extend(["--sparseprofile", sparse_profile]) + # Specify method to checkout a revision. This defaults to revisions as # SHA-1 strings, but also supports symbolic revisions like `tip` via the # branch flag. @@ -983,13 +1080,35 @@ def add_vcs_arguments(parser, project, name): action="store_true", help=f"Use shallow clone for {name}", ) + parser.add_argument( + f"--{project}-sparse-profile", + help=f"Path to sparse profile for {name} checkout. For git checkouts " + f"the patterns come from the {project.upper()}_SPARSE_PATTERNS " + "environment variable, a JSON list of git sparse-checkout patterns.", + ) def collect_vcs_options(args, project, name): checkout = getattr(args, f"{project}_checkout") shallow_clone = getattr(args, f"{project}_shallow_clone") + sparse_profile = getattr(args, f"{project}_sparse_profile", None) env_prefix = project.upper() + sparse_patterns = None + if f"{env_prefix}_SPARSE_PATTERNS" in os.environ: + try: + sparse_patterns = json.loads(os.environ[f"{env_prefix}_SPARSE_PATTERNS"]) + except ValueError as e: + raise RuntimeError( + f"{env_prefix}_SPARSE_PATTERNS is not a JSON list of patterns: {e}" + ) from None + if not isinstance(sparse_patterns, list) or not all( + isinstance(p, str) and p.strip() for p in sparse_patterns + ): + raise RuntimeError( + f"{env_prefix}_SPARSE_PATTERNS must be a JSON list of non-empty " + "pattern strings" + ) repo_type = os.environ.get(f"{env_prefix}_REPOSITORY_TYPE") base_repo = os.environ.get(f"{env_prefix}_BASE_REPOSITORY") @@ -1032,6 +1151,8 @@ def collect_vcs_options(args, project, name): "ssh-secret-name": private_key_secret, "pip-requirements": pip_requirements, "shallow-clone": shallow_clone, + "sparse-profile": sparse_profile, + "sparse-patterns": sparse_patterns, } @@ -1040,6 +1161,10 @@ def vcs_checkout_from_args(options): if options["head-ref"] and not options["head-rev"]: print("task should be defined in terms of non-symbolic revision") sys.exit(1) + if options["sparse-profile"] or options["sparse-patterns"] is not None: + raise RuntimeError( + f"--{options['project']}-sparse-profile needs --{options['project']}-checkout" + ) return head_ref = options["head-ref"] @@ -1071,6 +1196,17 @@ def vcs_checkout_from_args(options): if not head_ref: print("Providing a ref will improve the performance of this checkout") + if bool(options["sparse-profile"]) != (options["sparse-patterns"] is not None): + raise RuntimeError( + f"--{options['project']}-sparse-profile and the " + f"{options['env-prefix']}_SPARSE_PATTERNS environment variable " + "must be given together for a git checkout" + ) + if options["sparse-patterns"] is not None and not options["sparse-patterns"]: + raise RuntimeError( + f"{options['env-prefix']}_SPARSE_PATTERNS holds no patterns" + ) + revision = git_checkout( options["checkout"], options["head-repo"], @@ -1081,6 +1217,8 @@ def vcs_checkout_from_args(options): ssh_key_file, ssh_known_hosts_file, shallow=options.get("shallow-clone", False), + sparse_profile=options["sparse-profile"], + sparse_patterns=options["sparse-patterns"], ) elif options["repo-type"] == "hg": revision = hg_checkout( @@ -1090,6 +1228,7 @@ def vcs_checkout_from_args(options): options["store-path"], head_ref, head_rev, + sparse_profile=options["sparse-profile"], ) else: raise RuntimeError('Type of VCS must be either "git" or "hg"') diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index a67712731..9306a4c88 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -180,6 +180,17 @@ def test_install_pip_requirements_with_uv( {"shallow-clone": True}, id="git_with_shallow_clone", ), + pytest.param( + {"myrepo_sparse_profile": "profiles/docs"}, + { + "REPOSITORY_TYPE": "git", + "HEAD_REPOSITORY": "https://github.com/test/repo.git", + "HEAD_REV": "abc123", + "SPARSE_PATTERNS": '["/docs", "*.md"]', + }, + {"sparse-profile": "profiles/docs", "sparse-patterns": ["/docs", "*.md"]}, + id="git_with_sparse_profile", + ), ], ) def test_collect_vcs_options( @@ -215,6 +226,8 @@ def test_collect_vcs_options( "head-rev": env.get("HEAD_REV"), "repo-type": env.get("REPOSITORY_TYPE"), "shallow-clone": False, + "sparse-profile": None, + "sparse-patterns": None, "ssh-secret-name": env.get("SSH_SECRET_NAME"), "store-path": env.get("HG_STORE_PATH"), } @@ -710,3 +723,144 @@ def test_main_abspath_environment(mocker, run_main): assert env.get("MOZ_UV_HOME") == "/builds/worker/dir/uv" for key in envvars: assert env[key] == "/builds/worker/file" + + +SPARSE_REPO_FILES = [ + "a/deep/two.txt", + "a/one.txt", + "b/three.txt", + "c/four.md", + "root.txt", +] + + +@pytest.fixture(scope="session") # Tests shouldn't change this repo +def sparse_git_repo(): + "Repository with nested directories for sparse checkouts" + with tempfile.TemporaryDirectory() as repo: + repo_path = str(repo) + subprocess.check_call(["git", "init", "-b", "main"], cwd=repo_path) + subprocess.check_call(["git", "config", "user.name", "pytest"], cwd=repo_path) + subprocess.check_call( + ["git", "config", "user.email", "py@tes.t"], cwd=repo_path + ) + subprocess.check_call( + ["git", "config", "uploadpack.allowFilter", "true"], cwd=repo_path + ) + subprocess.check_call( + ["git", "config", "uploadpack.allowAnySHA1InWant", "true"], cwd=repo_path + ) + for filename in SPARSE_REPO_FILES: + filepath = os.path.join(repo_path, filename) + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w") as fout: + fout.write(filename) + subprocess.check_call(["git", "add", "."], cwd=repo_path) + subprocess.check_call(["git", "commit", "-m", "Initial commit"], cwd=repo_path) + yield {"url": f"file://{repo_path}", "rev": git_current_rev(repo_path)} + + +def materialized_files(destination): + out = subprocess.check_output(["git", "ls-files", "-t", "-z"], cwd=str(destination)) + return sorted( + entry[2:].decode() for entry in out.split(b"\0") if entry and entry[:1] != b"S" + ) + + +def git_config(destination, key): + result = subprocess.run( + ["git", "config", "--get", key], + cwd=str(destination), + capture_output=True, + text=True, + ) + return result.stdout.strip() if result.returncode == 0 else None + + +def sparse_git_checkout(run_task_mod, repo, destination, patterns=None): + run_task_mod.git_checkout( + destination_path=str(destination), + head_repo=repo["url"], + base_repo=repo["url"], + base_rev=None, + head_ref="main", + head_rev=repo["rev"], + ssh_key_file=None, + ssh_known_hosts_file=None, + shallow=True, + sparse_profile="profiles/test" if patterns else None, + sparse_patterns=patterns, + ) + + +def test_git_checkout_sparse(mock_stdin, run_task_mod, sparse_git_repo, tmp_path): + destination = tmp_path / "destination" + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/b", "*.md"]) + + assert materialized_files(destination) == ["b/three.txt", "c/four.md"] + assert git_config(destination, "core.sparseCheckout") == "true" + assert git_config(destination, "core.sparseCheckoutCone") != "true" + assert git_config(destination, "remote.origin.promisor") == "true" + assert git_current_rev(destination) == sparse_git_repo["rev"] + + +@pytest.mark.parametrize( + "args,env,error", + ( + pytest.param( + {"myrepo_sparse_profile": "profiles/test"}, + {}, + "must be given together", + id="option_without_patterns", + ), + pytest.param( + {}, + {"MYREPO_SPARSE_PATTERNS": '["/a"]'}, + "must be given together", + id="patterns_without_option", + ), + pytest.param( + {"myrepo_sparse_profile": "profiles/test"}, + {"MYREPO_SPARSE_PATTERNS": "[]"}, + "holds no patterns", + id="empty_patterns", + ), + pytest.param( + {"myrepo_sparse_profile": "profiles/test"}, + {"MYREPO_SPARSE_PATTERNS": "/a\n*.md\n"}, + "is not a JSON list", + id="patterns_not_json", + ), + pytest.param( + {"myrepo_sparse_profile": "profiles/test"}, + {"MYREPO_SPARSE_PATTERNS": '["/a", ""]'}, + "must be a JSON list of non-empty", + id="empty_pattern_string", + ), + pytest.param( + {"myrepo_sparse_profile": "profiles/test", "myrepo_checkout": None}, + {"MYREPO_SPARSE_PATTERNS": '["/a"]'}, + "needs --myrepo-checkout", + id="no_checkout", + ), + ), +) +def test_vcs_checkout_from_args_sparse_validation( + monkeypatch, run_task_mod, args, env, error +): + environ = { + "MYREPO_REPOSITORY_TYPE": "git", + "MYREPO_HEAD_REPOSITORY": "https://github.com/test/repo.git", + "MYREPO_HEAD_REV": "abc123", + } + environ.update(env) + monkeypatch.setattr(os, "environ", environ) + args.setdefault("myrepo_checkout", "checkout") + args.setdefault("myrepo_shallow_clone", True) + + with pytest.raises(RuntimeError, match=error): + options = run_task_mod.collect_vcs_options( + Namespace(**args), "myrepo", "myrepo" + ) + run_task_mod.vcs_checkout_from_args(options) + From 6784671096c1b5702776cfbe2d0dd69956d922c9 Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Thu, 10 Sep 2026 15:32:54 -0700 Subject: [PATCH 2/3] feat(run-task): only ever widen a cached git sparse checkout Sparse and full tasks in a worker pool share one checkout cache, and this change decides what happens when a task finds a cache another kind of task left behind: the cache only ever grows. A full task on a sparse cache widens it to a full checkout, and every later task on that worker runs full. A sparse task on a full cache runs on it as is. A sparse task on a cache with another sparse profile appends its own patterns with `sparse-checkout add`, so the cache ends up covering both profiles. Appending is why every translated pattern has to be positive. Whatever grows the cache runs after the checkout of the target revision, so a widen or an addition fetches that revision's files and not the old revision's. A cache in cone mode is treated as foreign and widened for now. The next commit starts creating cone caches and takes them over. Refs #1025 --- src/taskgraph/run-task/run-task | 37 ++++++++++++++++++++---- test/test_scripts_run_task.py | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index efa28d9a5..a9a1075a7 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -694,6 +694,12 @@ def _git_config(destination_path: str, key: str) -> Optional[str]: return p.stdout.strip() +def _git_lines(destination_path: str, *args: str) -> List[str]: + return subprocess.check_output( + ["git", *args], cwd=destination_path, env=os.environ, encoding="utf-8" + ).splitlines() + + class SparsePlan(NamedTuple): """The one `git sparse-checkout` command a checkout needs, if any.""" @@ -704,6 +710,8 @@ class SparsePlan(NamedTuple): def _plan_sparse_checkout( + destination_path: str, + has_repo: bool, is_sparse: bool, sparse_profile: Optional[str], sparse_patterns: Optional[List[str]], @@ -719,9 +727,26 @@ def _plan_sparse_checkout( ) return SparsePlan(None, [], [], "") - args = ["git", "sparse-checkout", "set", "--no-cone", "--stdin"] - message = f"sparse_set: sparse profile {profile}, {len(sparse_patterns)} entries" - return SparsePlan("sparse_set", args, sparse_patterns, message) + if has_repo and not is_sparse: + message = ( + f"the cached checkout is a full checkout, sparse profile " + f"{profile} is not applied" + ) + return SparsePlan(None, [], [], message) + + if not has_repo: + args = ["git", "sparse-checkout", "set", "--no-cone", "--stdin"] + op, entries = "sparse_set", sparse_patterns + else: + current = _git_lines(destination_path, "sparse-checkout", "list") + args = ["git", "sparse-checkout", "add", "--stdin"] + op = "sparse_add" + entries = [p for p in sparse_patterns if p not in set(current)] + if not entries: + return SparsePlan(None, [], [], f"sparse profile {profile} already applied") + + message = f"{op}: sparse profile {profile}, {len(entries)} entries" + return SparsePlan(op, args, entries, message) def shortref(ref: str) -> str: @@ -794,7 +819,7 @@ def git_checkout( is_sparse = ( has_repo and _git_config(destination_path, "core.sparseCheckout") == "true" ) - sparse = bool(sparse_patterns) + sparse = bool(sparse_patterns) and (not has_repo or is_sparse) action = "pull" if has_repo else "clone" mode = "sparsecheckout" if sparse else "fullcheckout" @@ -876,7 +901,9 @@ def git_checkout( # `git fetch` set `FETCH_HEAD` reference to the last commit of the desired branch rev = head_rev if head_rev else "FETCH_HEAD" - plan = _plan_sparse_checkout(is_sparse, sparse_profile, sparse_patterns) + plan = _plan_sparse_checkout( + destination_path, has_repo, is_sparse, sparse_profile, sparse_patterns + ) if sparse and sparse_profile: perf.extra_options.append(os.path.basename(sparse_profile)) if plan.message: diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index 9306a4c88..8682ab4f9 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -864,3 +864,54 @@ def test_vcs_checkout_from_args_sparse_validation( ) run_task_mod.vcs_checkout_from_args(options) + +def test_git_checkout_sparse_widens_on_full_task( + mock_stdin, run_task_mod, sparse_git_repo, tmp_path +): + destination = tmp_path / "destination" + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/b", "*.md"]) + sparse_git_checkout(run_task_mod, sparse_git_repo, destination) + + assert git_config(destination, "core.sparseCheckout") != "true" + assert materialized_files(destination) == SPARSE_REPO_FILES + + +def test_git_checkout_sparse_widens_over_modified_files( + mock_stdin, run_task_mod, sparse_git_repo, tmp_path +): + destination = tmp_path / "destination" + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/b", "*.md"]) + (destination / "b" / "three.txt").write_text("modified") + sparse_git_checkout(run_task_mod, sparse_git_repo, destination) + + assert git_config(destination, "core.sparseCheckout") != "true" + assert materialized_files(destination) == SPARSE_REPO_FILES + assert (destination / "b" / "three.txt").read_text() == "b/three.txt" + + +def test_git_checkout_sparse_on_full_cache_stays_full( + mock_stdin, run_task_mod, sparse_git_repo, tmp_path +): + destination = tmp_path / "destination" + sparse_git_checkout(run_task_mod, sparse_git_repo, destination) + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/b", "*.md"]) + + assert git_config(destination, "core.sparseCheckout") is None + assert materialized_files(destination) == SPARSE_REPO_FILES + + +def test_git_checkout_sparse_adds_second_profile( + mock_stdin, run_task_mod, sparse_git_repo, tmp_path +): + destination = tmp_path / "destination" + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/b", "*.md"]) + sparse_git_checkout( + run_task_mod, sparse_git_repo, destination, ["/a/one.txt", "*.md"] + ) + + assert materialized_files(destination) == ["a/one.txt", "b/three.txt", "c/four.md"] + patterns = subprocess.check_output( + ["git", "sparse-checkout", "list"], cwd=str(destination), text=True + ).split() + assert set(patterns) == {"/b", "*.md", "/a/one.txt"} + From 7fab145b48b70a794f1a26f1fd8b272a615a245f Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Thu, 10 Sep 2026 15:32:54 -0700 Subject: [PATCH 3/3] feat(run-task): use cone mode when a sparse profile lists only paths Git tests a no cone pattern against every index entry on every checkout. Cone mode accepts only directories, and in return git matches by prefix and keeps the index itself small with the sparse index. When every pattern is an anchored literal path, which is true for twelve of the eighteen profiles, run-task applies them with `set --cone --sparse-index`. Locally repackage-msi went from 15 to 1 second on a warm cache. A path to a file works as a cone directory because cone mode includes the files of every ancestor directory. The same rule means a cone checkout has a few hundred more files than the profile names, the files directly inside every ancestor up to the root. Never fewer, so tasks are unaffected. Mixing follows the widen only rule. A cone cache takes further cone profiles with `add`. A glob profile on a cone cache converts it to no cone mode and keeps every directory it had plus the files of their ancestors, listed with `git ls-tree` at the target revision, so nothing that was checked out disappears. Refs #1025 --- src/taskgraph/run-task/run-task | 114 ++++++++++++++++++++++++++++---- test/test_scripts_run_task.py | 60 +++++++++++++++++ 2 files changed, 162 insertions(+), 12 deletions(-) diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index a9a1075a7..6566908d7 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -700,9 +700,57 @@ def _git_lines(destination_path: str, *args: str) -> List[str]: ).splitlines() +_PATTERN_METACHARS_RE = re.compile(r"([*?\[\\])") + + +def _escape_pattern(path: str) -> str: + escaped = _PATTERN_METACHARS_RE.sub(r"\\\1", path) + if escaped.endswith(" "): + escaped = escaped[:-1] + "\\ " + return escaped + + +def _cone_ancestor_files( + destination_path: str, rev: str, directories: List[str] +) -> List[str]: + patterns = [] + seen = set() + for directory in directories: + parts = directory.split("/") + for depth in range(len(parts)): + ancestor = "/".join(parts[:depth]) + if ancestor in seen: + continue + seen.add(ancestor) + args = ["ls-tree", "-z", rev] + if ancestor: + args.append(f"{ancestor}/") + out = subprocess.check_output( + ["git", *args], cwd=destination_path, env=os.environ, encoding="utf-8" + ) + for entry in out.split("\0"): + if not entry: + continue + meta, path = entry.split("\t", 1) + if meta.split(" ")[1] == "blob" and "\n" not in path and "\r" not in path: + patterns.append(f"/{_escape_pattern(path)}") + return patterns + + +def _cone_directories(patterns: List[str]) -> Optional[List[str]]: + directories = [] + for pattern in patterns: + body = pattern[1:].rstrip("/") + if not pattern.startswith("/") or not body or any(c in body for c in "*?[\\"): + return None + directories.append(body) + return directories + + class SparsePlan(NamedTuple): """The one `git sparse-checkout` command a checkout needs, if any.""" + cone: bool op: Optional[str] args: List[str] entries: List[str] @@ -715,38 +763,78 @@ def _plan_sparse_checkout( is_sparse: bool, sparse_profile: Optional[str], sparse_patterns: Optional[List[str]], + rev: str, ) -> SparsePlan: profile = sparse_profile or "" if not sparse_patterns: if is_sparse: return SparsePlan( + False, "sparse_disable", ["git", "sparse-checkout", "disable"], [], "widening the sparse checkout to the full tree", ) - return SparsePlan(None, [], [], "") + return SparsePlan(False, None, [], [], "") if has_repo and not is_sparse: message = ( f"the cached checkout is a full checkout, sparse profile " f"{profile} is not applied" ) - return SparsePlan(None, [], [], message) + return SparsePlan(False, None, [], [], message) + cone_directories = _cone_directories(sparse_patterns) if not has_repo: - args = ["git", "sparse-checkout", "set", "--no-cone", "--stdin"] - op, entries = "sparse_set", sparse_patterns + if cone_directories is not None: + args = [ + "git", + "sparse-checkout", + "set", + "--cone", + "--sparse-index", + "--stdin", + ] + cone, entries = True, cone_directories + else: + args = ["git", "sparse-checkout", "set", "--no-cone", "--stdin"] + cone, entries = False, sparse_patterns + op = "sparse_set" else: + is_cone = _git_config(destination_path, "core.sparseCheckoutCone") == "true" current = _git_lines(destination_path, "sparse-checkout", "list") - args = ["git", "sparse-checkout", "add", "--stdin"] - op = "sparse_add" - entries = [p for p in sparse_patterns if p not in set(current)] + if cone_directories is not None and is_cone: + args = ["git", "sparse-checkout", "add", "--stdin"] + cone, op = True, "sparse_add" + entries = [d for d in cone_directories if d not in set(current)] + elif is_cone: + args = [ + "git", + "sparse-checkout", + "set", + "--no-cone", + "--no-sparse-index", + "--stdin", + ] + cone, op = False, "sparse_convert" + entries = list( + dict.fromkeys( + [f"/{_escape_pattern(d)}" for d in current] + + _cone_ancestor_files(destination_path, rev, current) + + sparse_patterns + ) + ) + else: + args = ["git", "sparse-checkout", "add", "--stdin"] + cone, op = False, "sparse_add" + entries = [p for p in sparse_patterns if p not in set(current)] if not entries: - return SparsePlan(None, [], [], f"sparse profile {profile} already applied") + return SparsePlan( + cone, None, [], [], f"sparse profile {profile} already applied" + ) message = f"{op}: sparse profile {profile}, {len(entries)} entries" - return SparsePlan(op, args, entries, message) + return SparsePlan(cone, op, args, entries, message) def shortref(ref: str) -> str: @@ -902,10 +990,12 @@ def git_checkout( rev = head_rev if head_rev else "FETCH_HEAD" plan = _plan_sparse_checkout( - destination_path, has_repo, is_sparse, sparse_profile, sparse_patterns + destination_path, has_repo, is_sparse, sparse_profile, sparse_patterns, rev ) - if sparse and sparse_profile: - perf.extra_options.append(os.path.basename(sparse_profile)) + if sparse: + perf.extra_options.append("cone" if plan.cone else "no-cone") + if sparse_profile: + perf.extra_options.append(os.path.basename(sparse_profile)) if plan.message: print_line(b"vcs", f"{plan.message}\n".encode("utf-8")) if plan.op: diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index 8682ab4f9..209e288bd 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -889,6 +889,23 @@ def test_git_checkout_sparse_widens_over_modified_files( assert (destination / "b" / "three.txt").read_text() == "b/three.txt" +def test_git_checkout_sparse_adds_over_modified_files( + mock_stdin, run_task_mod, sparse_git_repo, tmp_path +): + destination = tmp_path / "destination" + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/b"]) + (destination / "b" / "three.txt").write_text("modified") + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/a/deep"]) + + assert materialized_files(destination) == [ + "a/deep/two.txt", + "a/one.txt", + "b/three.txt", + "root.txt", + ] + assert (destination / "b" / "three.txt").read_text() == "b/three.txt" + + def test_git_checkout_sparse_on_full_cache_stays_full( mock_stdin, run_task_mod, sparse_git_repo, tmp_path ): @@ -915,3 +932,46 @@ def test_git_checkout_sparse_adds_second_profile( ).split() assert set(patterns) == {"/b", "*.md", "/a/one.txt"} + +def test_git_checkout_sparse_cone(mock_stdin, run_task_mod, sparse_git_repo, tmp_path): + destination = tmp_path / "destination" + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/a/deep", "/b"]) + + assert git_config(destination, "core.sparseCheckoutCone") == "true" + assert git_config(destination, "index.sparse") == "true" + assert materialized_files(destination) == [ + "a/deep/two.txt", + "a/one.txt", + "b/three.txt", + "root.txt", + ] + + +def test_git_checkout_sparse_cone_adds_directories( + mock_stdin, run_task_mod, sparse_git_repo, tmp_path +): + destination = tmp_path / "destination" + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/b"]) + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/a/deep"]) + + assert git_config(destination, "core.sparseCheckoutCone") == "true" + assert materialized_files(destination) == [ + "a/deep/two.txt", + "a/one.txt", + "b/three.txt", + "root.txt", + ] + + +def test_git_checkout_sparse_cone_converts_for_globs( + mock_stdin, run_task_mod, sparse_git_repo, tmp_path +): + destination = tmp_path / "destination" + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["/b"]) + before = materialized_files(destination) + sparse_git_checkout(run_task_mod, sparse_git_repo, destination, ["*.md"]) + + assert git_config(destination, "core.sparseCheckoutCone") == "false" + after = materialized_files(destination) + assert set(before) <= set(after) + assert "c/four.md" in after