diff --git a/updater/bin/fetch_files.rb b/updater/bin/fetch_files.rb new file mode 100644 index 00000000000..a40be3254d1 --- /dev/null +++ b/updater/bin/fetch_files.rb @@ -0,0 +1,38 @@ +# typed: strict +# frozen_string_literal: true + +$LOAD_PATH.unshift(__dir__ + "/../lib") + +$stdout.sync = true + +require "dependabot/api_client" +require "dependabot/environment" +require "dependabot/service" +require "dependabot/setup" +require "dependabot/file_fetcher_command" +require "debug" if ENV["DEBUG"] + +class UpdaterKilledError < StandardError; end + +trap("TERM") do + puts "Received SIGTERM" + error = UpdaterKilledError.new("Updater process killed with SIGTERM") + tags = { "gh.dependabot_api.update_job.id": ENV.fetch("DEPENDABOT_JOB_ID", nil) } + + api_client = + Dependabot::ApiClient.new( + Dependabot::Environment.api_url, + Dependabot::Environment.job_id, + Dependabot::Environment.job_token + ) + Dependabot::Service.new(client: api_client).capture_exception(error: error, tags: tags) + exit +end + +begin + # Clones into DEPENDABOT_REPO_CONTENTS_PATH. When that path is a shared volume the + # update container reuses the clone instead of making its own. + Dependabot::FileFetcherCommand.new(record_ecosystem_versions: false).run +rescue Dependabot::RunFailure + exit 1 +end diff --git a/updater/bin/run b/updater/bin/run index 7a0117a84f0..8987bc23704 100755 --- a/updater/bin/run +++ b/updater/bin/run @@ -3,14 +3,24 @@ set -e command="$1" if [ -z "$command" ]; then - echo "usage: run [update_files|update_graph]" + echo "usage: run [fetch_files|update_files|update_graph]" exit 1 fi -# ignore fetch_files command for backward compatibility -if [ "$command" = "fetch_files" ]; then - echo "fetch_files command is no longer used directly" +case "$command" in + fetch_files|update_files|update_graph) ;; + *) + echo "usage: run [fetch_files|update_files|update_graph]" + exit 1 + ;; +esac + +if [ "$command" = "fetch_files" ] && [ "${DEPENDABOT_ENABLE_FETCH_FILES_COMMAND:-}" != "true" ]; then + echo "fetch_files command is disabled" exit 0 fi -bundle exec ruby "bin/${command}.rb" +# exec so the Ruby process' exit code becomes the container's exit code: callers +# chain `run fetch_files && run update_files` and must be able to tell a fetch +# failure from a success. +exec bundle exec ruby "bin/${command}.rb" diff --git a/updater/lib/dependabot/file_fetcher_command.rb b/updater/lib/dependabot/file_fetcher_command.rb index 68081ecf67f..a80c128a11f 100644 --- a/updater/lib/dependabot/file_fetcher_command.rb +++ b/updater/lib/dependabot/file_fetcher_command.rb @@ -8,6 +8,7 @@ require "dependabot/opentelemetry" require "dependabot/updater" require "dependabot/file_fetcher_command_connectivity" +require "dependabot/file_fetcher_command_local_checkout" require "octokit" require "sorbet-runtime" @@ -15,6 +16,12 @@ module Dependabot class FileFetcherCommand < BaseCommand extend T::Sig include FileFetcherCommandConnectivity + include FileFetcherCommandLocalCheckout + + sig { params(record_ecosystem_versions: T::Boolean).void } + def initialize(record_ecosystem_versions: true) + @record_ecosystem_versions = record_ecosystem_versions + end # BaseCommand does not implement this method, so we should expose # the instance variable for error handling to avoid raising a @@ -38,8 +45,7 @@ def perform_job # rubocop:disable Metrics/AbcSize begin connectivity_check if ENV["ENABLE_CONNECTIVITY_CHECK"] == "1" normalize_single_directory - validate_target_branch - dependabot_ref_namespace_available? + validate_repository clone_repo_contents @base_commit_sha = file_fetcher.commit raise "base commit SHA not found" unless @base_commit_sha @@ -262,7 +268,7 @@ def dependency_files sig { returns(T::Boolean) } def should_record_ecosystem_versions? # We don't set this flag in GHES because there's no point in recording versions since we can't access that data. - Experiments.enabled?(:record_ecosystem_versions) + @record_ecosystem_versions && Experiments.enabled?(:record_ecosystem_versions) end sig { params(file_fetcher: Dependabot::FileFetchers::Base).void } @@ -293,7 +299,7 @@ def with_retries(max_retries: 2, &_block) end end - sig { void } + sig { override.void } def dependabot_ref_namespace_available? dependabot_branch = "dependabot" begin @@ -312,7 +318,7 @@ def dependabot_ref_namespace_available? end end - sig { void } + sig { override.void } def validate_target_branch return unless job.source.branch @@ -337,12 +343,13 @@ def validate_target_branch sig { void } def clone_repo_contents + return if local_checkout_only? return unless job.clone? file_fetcher.clone_repo_contents end - sig { returns(T::Boolean) } + sig { override.returns(T::Boolean) } def already_cloned? return false unless Environment.repo_contents_path diff --git a/updater/lib/dependabot/file_fetcher_command_local_checkout.rb b/updater/lib/dependabot/file_fetcher_command_local_checkout.rb new file mode 100644 index 00000000000..083edfa4917 --- /dev/null +++ b/updater/lib/dependabot/file_fetcher_command_local_checkout.rb @@ -0,0 +1,49 @@ +# typed: strong +# frozen_string_literal: true + +require "sorbet-runtime" + +module Dependabot + module FileFetcherCommandLocalCheckout + extend T::Sig + extend T::Helpers + + abstract! + + private + + sig { abstract.void } + def validate_target_branch; end + + sig { abstract.void } + def dependabot_ref_namespace_available?; end + + sig { abstract.returns(T::Boolean) } + def already_cloned?; end + + sig { void } + def validate_repository + validate_local_checkout + return if local_checkout_only? + + validate_target_branch + dependabot_ref_namespace_available? + end + + sig { void } + def validate_local_checkout + return unless local_checkout_only? + + repo_contents_path = Environment.repo_contents_path + Kernel.raise "DEPENDABOT_REPO_CONTENTS_PATH is not set" if repo_contents_path.to_s.empty? + return if already_cloned? + + Kernel.raise "Local repository checkout not found at #{repo_contents_path}" + end + + sig { returns(T::Boolean) } + def local_checkout_only? + ENV["DEPENDABOT_LOCAL_CHECKOUT_ONLY"] == "true" + end + end +end diff --git a/updater/spec/bin/run_script_spec.rb b/updater/spec/bin/run_script_spec.rb new file mode 100644 index 00000000000..4d73144ec0a --- /dev/null +++ b/updater/spec/bin/run_script_spec.rb @@ -0,0 +1,54 @@ +# typed: false +# frozen_string_literal: true + +require "open3" +require "spec_helper" +require "tmpdir" + +class RunScript + def initialize(script_path:, updater_root:, environment:) + @script_path = script_path + @updater_root = updater_root + @environment = environment + end + + def call(command, environment = {}) + Open3.capture3(@environment.merge(environment), @script_path, command, chdir: @updater_root) + end +end + +RSpec.describe RunScript do + let(:updater_root) { File.expand_path("../..", __dir__) } + let(:script_path) { File.join(updater_root, "bin/run") } + let(:run_script) { described_class.new(script_path:, updater_root:, environment:) } + let(:environment) { { "PATH" => "#{bin_directory}:#{ENV.fetch('PATH')}" } } + let(:bin_directory) { Dir.mktmpdir } + + before do + bundle_path = File.join(bin_directory, "bundle") + File.write(bundle_path, "#!/bin/sh\nprintf 'bundle %s\\n' \"$*\"\n") + FileUtils.chmod(0o755, bundle_path) + end + + after { FileUtils.rm_rf(bin_directory) } + + it "keeps fetch_files as a no-op by default" do + stdout, stderr, status = run_script.call("fetch_files") + + expect(status).to be_success + expect(stderr).to be_empty + expect(stdout).to include("fetch_files command is disabled") + expect(stdout).not_to include("bundle exec") + end + + it "runs fetch_files when the feature flag is enabled" do + stdout, stderr, status = run_script.call( + "fetch_files", + "DEPENDABOT_ENABLE_FETCH_FILES_COMMAND" => "true" + ) + + expect(status).to be_success + expect(stderr).to be_empty + expect(stdout).to eq("bundle exec ruby bin/fetch_files.rb\n") + end +end diff --git a/updater/spec/dependabot/file_fetcher_command_spec.rb b/updater/spec/dependabot/file_fetcher_command_spec.rb index eb0c784138e..771cdad73f1 100644 --- a/updater/spec/dependabot/file_fetcher_command_spec.rb +++ b/updater/spec/dependabot/file_fetcher_command_spec.rb @@ -12,10 +12,11 @@ require "dependabot/bundler" RSpec.describe Dependabot::FileFetcherCommand do - subject(:job) { described_class.new } + subject(:job) { described_class.new(record_ecosystem_versions:) } let(:api_client) { double(Dependabot::ApiClient) } let(:job_id) { "123123" } + let(:record_ecosystem_versions) { true } before do allow(Dependabot::ApiClient).to receive(:new).and_return(api_client) @@ -62,6 +63,76 @@ expect(dependency_file.content_encoding).to eq("utf-8") end + context "when ecosystem version recording is disabled" do + let(:record_ecosystem_versions) { false } + + it "fetches files without recording ecosystem versions", + vcr: { cassette_name: "Dependabot_FileFetcherCommand/_perform_job/fetches_the_files" } do + expect(api_client).not_to receive(:record_ecosystem_versions) + + perform_job + + expect(job.files.dependency_files).not_to be_empty + end + end + + context "when only a local checkout may be used" do + let(:repo_contents_path) { Dir.mktmpdir } + + before do + allow(Dependabot::Environment).to receive_messages( + repo_contents_path: repo_contents_path + ) + stub_const("ENV", ENV.to_h.merge("DEPENDABOT_LOCAL_CHECKOUT_ONLY" => "true")) + end + + after { FileUtils.rm_rf(repo_contents_path) unless repo_contents_path.to_s.empty? } + + it "reports an error instead of fetching the repository when the checkout is missing" do + expect(Dependabot::Bundler::FileFetcher).not_to receive(:new) + expect(api_client).to receive(:record_update_job_error).with( + hash_including( + error_type: "file_fetcher_error", + error_details: hash_including( + Dependabot::ErrorAttributes::MESSAGE => "Local repository checkout not found at #{repo_contents_path}" + ) + ) + ) + expect(api_client).to receive(:mark_job_as_processed) + + expect { perform_job }.to output(/Local repository checkout not found/).to_stdout_from_any_process + end + + shared_examples "a missing repository contents path" do + it "reports that the repository contents path is not set" do + expect(Dependabot::Bundler::FileFetcher).not_to receive(:new) + expect(api_client).to receive(:record_update_job_error).with( + hash_including( + error_type: "file_fetcher_error", + error_details: hash_including( + Dependabot::ErrorAttributes::MESSAGE => "DEPENDABOT_REPO_CONTENTS_PATH is not set" + ) + ) + ) + expect(api_client).to receive(:mark_job_as_processed) + + expect { perform_job }.to output(/DEPENDABOT_REPO_CONTENTS_PATH is not set/).to_stdout_from_any_process + end + end + + context "when the repository contents path is nil" do + let(:repo_contents_path) { nil } + + it_behaves_like "a missing repository contents path" + end + + context "when the repository contents path is empty" do + let(:repo_contents_path) { "" } + + it_behaves_like "a missing repository contents path" + end + end + context "when empty directories are specified" do before do allow(Dependabot::Environment).to receive(:repo_contents_path).and_return(Dir.mktmpdir)