Add a fetch_files entrypoint so the clone can happen in its own container

updater/bin/fetch_files.rb

@@ -0,0 +1,38 @@
+# typed: strict
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(__dir__ + "/../lib")
+
+$stdout.sync = true
+
+require "dependabot/api_client"
+require "dependabot/environment"
+require "dependabot/service"
+require "dependabot/setup"
+require "dependabot/file_fetcher_command"
+require "debug" if ENV["DEBUG"]
+
+class UpdaterKilledError < StandardError; end
+
+trap("TERM") do
+  puts "Received SIGTERM"
+  error = UpdaterKilledError.new("Updater process killed with SIGTERM")
+  tags = { "gh.dependabot_api.update_job.id": ENV.fetch("DEPENDABOT_JOB_ID", nil) }
+
+  api_client =
+    Dependabot::ApiClient.new(
+      Dependabot::Environment.api_url,
+      Dependabot::Environment.job_id,
+      Dependabot::Environment.job_token
+    )
+  Dependabot::Service.new(client: api_client).capture_exception(error: error, tags: tags)
+  exit
+end
+
+begin
+  # Clones into DEPENDABOT_REPO_CONTENTS_PATH. When that path is a shared volume the
+  # update container reuses the clone instead of making its own.
+  Dependabot::FileFetcherCommand.new(record_ecosystem_versions: false).run
+rescue Dependabot::RunFailure
+  exit 1
+end

updater/bin/run

@@ -3,14 +3,24 @@ set -e
 
 command="$1"
 if [ -z "$command" ]; then
-  echo "usage: run [update_files|update_graph]"
+  echo "usage: run [fetch_files|update_files|update_graph]"
   exit 1
 fi
 
-# ignore fetch_files command for backward compatibility
-if [ "$command" = "fetch_files" ]; then
-  echo "fetch_files command is no longer used directly"
+case "$command" in
+  fetch_files|update_files|update_graph) ;;
+  *)
+    echo "usage: run [fetch_files|update_files|update_graph]"
+    exit 1
+    ;;
+esac
+
+if [ "$command" = "fetch_files" ] && [ "${DEPENDABOT_ENABLE_FETCH_FILES_COMMAND:-}" != "true" ]; then
+  echo "fetch_files command is disabled"
   exit 0
 fi
 
-bundle exec ruby "bin/${command}.rb"
+# exec so the Ruby process' exit code becomes the container's exit code: callers
+# chain `run fetch_files && run update_files` and must be able to tell a fetch
+# failure from a success.
+exec bundle exec ruby "bin/${command}.rb"

updater/lib/dependabot/file_fetcher_command.rb

@@ -8,13 +8,20 @@
 require "dependabot/opentelemetry"
 require "dependabot/updater"
 require "dependabot/file_fetcher_command_connectivity"
+require "dependabot/file_fetcher_command_local_checkout"
 require "octokit"
 require "sorbet-runtime"
 
 module Dependabot
   class FileFetcherCommand < BaseCommand
     extend T::Sig
     include FileFetcherCommandConnectivity
+    include FileFetcherCommandLocalCheckout
+
+    sig { params(record_ecosystem_versions: T::Boolean).void }
+    def initialize(record_ecosystem_versions: true)
+      @record_ecosystem_versions = record_ecosystem_versions
+    end
 
     # BaseCommand does not implement this method, so we should expose
     # the instance variable for error handling to avoid raising a
@@ -38,8 +45,7 @@ def perform_job # rubocop:disable Metrics/AbcSize
         begin
           connectivity_check if ENV["ENABLE_CONNECTIVITY_CHECK"] == "1"
           normalize_single_directory
-          validate_target_branch
-          dependabot_ref_namespace_available?
+          validate_repository
           clone_repo_contents
           @base_commit_sha = file_fetcher.commit
           raise "base commit SHA not found" unless @base_commit_sha
@@ -262,7 +268,7 @@ def dependency_files
     sig { returns(T::Boolean) }
     def should_record_ecosystem_versions?
       # We don't set this flag in GHES because there's no point in recording versions since we can't access that data.
-      Experiments.enabled?(:record_ecosystem_versions)
+      @record_ecosystem_versions && Experiments.enabled?(:record_ecosystem_versions)
     end
 
     sig { params(file_fetcher: Dependabot::FileFetchers::Base).void }
@@ -293,7 +299,7 @@ def with_retries(max_retries: 2, &_block)
       end
     end
 
-    sig { void }
+    sig { override.void }
     def dependabot_ref_namespace_available?
       dependabot_branch = "dependabot"
       begin
@@ -312,7 +318,7 @@ def dependabot_ref_namespace_available?
       end
     end
 
-    sig { void }
+    sig { override.void }
     def validate_target_branch
       return unless job.source.branch
 
@@ -337,12 +343,13 @@ def validate_target_branch
 
     sig { void }
     def clone_repo_contents
+      return if local_checkout_only?
       return unless job.clone?
 
       file_fetcher.clone_repo_contents
     end
 
-    sig { returns(T::Boolean) }
+    sig { override.returns(T::Boolean) }
     def already_cloned?
       return false unless Environment.repo_contents_path
 

updater/lib/dependabot/file_fetcher_command_local_checkout.rb

@@ -0,0 +1,49 @@
+# typed: strong
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module Dependabot
+  module FileFetcherCommandLocalCheckout
+    extend T::Sig
+    extend T::Helpers
+
+    abstract!
+
+    private
+
+    sig { abstract.void }
+    def validate_target_branch; end
+
+    sig { abstract.void }
+    def dependabot_ref_namespace_available?; end
+
+    sig { abstract.returns(T::Boolean) }
+    def already_cloned?; end
+
+    sig { void }
+    def validate_repository
+      validate_local_checkout
+      return if local_checkout_only?
+
+      validate_target_branch
+      dependabot_ref_namespace_available?
+    end
+
+    sig { void }
+    def validate_local_checkout
+      return unless local_checkout_only?
+
+      repo_contents_path = Environment.repo_contents_path
+      Kernel.raise "DEPENDABOT_REPO_CONTENTS_PATH is not set" if repo_contents_path.to_s.empty?
+      return if already_cloned?
+
+      Kernel.raise "Local repository checkout not found at #{repo_contents_path}"
+    end
+
+    sig { returns(T::Boolean) }
+    def local_checkout_only?
+      ENV["DEPENDABOT_LOCAL_CHECKOUT_ONLY"] == "true"
+    end
+  end
+end

updater/spec/bin/run_script_spec.rb

@@ -0,0 +1,54 @@
+# typed: false
+# frozen_string_literal: true
+
+require "open3"
+require "spec_helper"
+require "tmpdir"
+
+class RunScript
+  def initialize(script_path:, updater_root:, environment:)
+    @script_path = script_path
+    @updater_root = updater_root
+    @environment = environment
+  end
+
+  def call(command, environment = {})
+    Open3.capture3(@environment.merge(environment), @script_path, command, chdir: @updater_root)
+  end
+end
+
+RSpec.describe RunScript do
+  let(:updater_root) { File.expand_path("../..", __dir__) }
+  let(:script_path) { File.join(updater_root, "bin/run") }
+  let(:run_script) { described_class.new(script_path:, updater_root:, environment:) }
+  let(:environment) { { "PATH" => "#{bin_directory}:#{ENV.fetch('PATH')}" } }
+  let(:bin_directory) { Dir.mktmpdir }
+
+  before do
+    bundle_path = File.join(bin_directory, "bundle")
+    File.write(bundle_path, "#!/bin/sh\nprintf 'bundle %s\\n' \"$*\"\n")
+    FileUtils.chmod(0o755, bundle_path)
+  end
+
+  after { FileUtils.rm_rf(bin_directory) }
+
+  it "keeps fetch_files as a no-op by default" do
+    stdout, stderr, status = run_script.call("fetch_files")
+
+    expect(status).to be_success
+    expect(stderr).to be_empty
+    expect(stdout).to include("fetch_files command is disabled")
+    expect(stdout).not_to include("bundle exec")
+  end
+
+  it "runs fetch_files when the feature flag is enabled" do
+    stdout, stderr, status = run_script.call(
+      "fetch_files",
+      "DEPENDABOT_ENABLE_FETCH_FILES_COMMAND" => "true"
+    )
+
+    expect(status).to be_success
+    expect(stderr).to be_empty
+    expect(stdout).to eq("bundle exec ruby bin/fetch_files.rb\n")
+  end
+end

updater/spec/dependabot/file_fetcher_command_spec.rb

@@ -12,10 +12,11 @@
 require "dependabot/bundler"
 
 RSpec.describe Dependabot::FileFetcherCommand do
-  subject(:job) { described_class.new }
+  subject(:job) { described_class.new(record_ecosystem_versions:) }
 
   let(:api_client) { double(Dependabot::ApiClient) }
   let(:job_id) { "123123" }
+  let(:record_ecosystem_versions) { true }
 
   before do
     allow(Dependabot::ApiClient).to receive(:new).and_return(api_client)
@@ -62,6 +63,76 @@
       expect(dependency_file.content_encoding).to eq("utf-8")
     end
 
+    context "when ecosystem version recording is disabled" do
+      let(:record_ecosystem_versions) { false }
+
+      it "fetches files without recording ecosystem versions",
+         vcr: { cassette_name: "Dependabot_FileFetcherCommand/_perform_job/fetches_the_files" } do
+        expect(api_client).not_to receive(:record_ecosystem_versions)
+
+        perform_job
+
+        expect(job.files.dependency_files).not_to be_empty
+      end
+    end
+
+    context "when only a local checkout may be used" do
+      let(:repo_contents_path) { Dir.mktmpdir }
+
+      before do
+        allow(Dependabot::Environment).to receive_messages(
+          repo_contents_path: repo_contents_path
+        )
+        stub_const("ENV", ENV.to_h.merge("DEPENDABOT_LOCAL_CHECKOUT_ONLY" => "true"))
+      end
+
+      after { FileUtils.rm_rf(repo_contents_path) unless repo_contents_path.to_s.empty? }
+
+      it "reports an error instead of fetching the repository when the checkout is missing" do
+        expect(Dependabot::Bundler::FileFetcher).not_to receive(:new)
+        expect(api_client).to receive(:record_update_job_error).with(
+          hash_including(
+            error_type: "file_fetcher_error",
+            error_details: hash_including(
+              Dependabot::ErrorAttributes::MESSAGE => "Local repository checkout not found at #{repo_contents_path}"
+            )
+          )
+        )
+        expect(api_client).to receive(:mark_job_as_processed)
+
+        expect { perform_job }.to output(/Local repository checkout not found/).to_stdout_from_any_process
+      end
+
+      shared_examples "a missing repository contents path" do
+        it "reports that the repository contents path is not set" do
+          expect(Dependabot::Bundler::FileFetcher).not_to receive(:new)
+          expect(api_client).to receive(:record_update_job_error).with(
+            hash_including(
+              error_type: "file_fetcher_error",
+              error_details: hash_including(
+                Dependabot::ErrorAttributes::MESSAGE => "DEPENDABOT_REPO_CONTENTS_PATH is not set"
+              )
+            )
+          )
+          expect(api_client).to receive(:mark_job_as_processed)
+
+          expect { perform_job }.to output(/DEPENDABOT_REPO_CONTENTS_PATH is not set/).to_stdout_from_any_process
+        end
+      end
+
+      context "when the repository contents path is nil" do
+        let(:repo_contents_path) { nil }
+
+        it_behaves_like "a missing repository contents path"
+      end
+
+      context "when the repository contents path is empty" do
+        let(:repo_contents_path) { "" }
+
+        it_behaves_like "a missing repository contents path"
+      end
+    end
+
     context "when empty directories are specified" do
       before do
         allow(Dependabot::Environment).to receive(:repo_contents_path).and_return(Dir.mktmpdir)