Support S3-compatible Pulp storage

README.md

@@ -73,6 +73,26 @@ PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings | grep WORKER_
 
 This is useful for module parameter which configure Pulpcore settings but have an `undef` default, such as `$worker_ttl`.  When the param value is `undef`, the setting is omitted from `settings.py` and therefore Pulpcore's default is used.
 
+### S3-compatible object storage
+
+Set `storage_backend` to `s3` and pass the options supported by
+[`django-storages`](https://django-storages.readthedocs.io/en/stable/backends/amazon-S3.html):
+
+```puppet
+class { 'pulpcore':
+  storage_backend => 's3',
+  storage_options => {
+    'bucket_name' => 'pulp',
+    'endpoint_url' => 'https://object.example.test',
+    'addressing_style' => 'path',
+  },
+}
+```
+
+The module installs the `pulpcore-storage-s3` package capability, omits
+`MEDIA_ROOT` so Django's default empty value applies, and preserves local Django
+static-file storage. Filesystem content storage remains the default.
+
 ## Service setup
 
 The module deploys a few systemd services:

manifests/config.pp

@@ -48,20 +48,47 @@
     order   => '02',
   }
 
+  if $pulpcore::storage_backend == 's3' {
+    $storages = {
+      'default'     => {
+        'BACKEND' => 'storages.backends.s3.S3Storage',
+        'OPTIONS' => $pulpcore::storage_options,
+      },
+      'staticfiles' => {
+        'BACKEND' => 'django.contrib.staticfiles.storage.StaticFilesStorage',
+      },
+    }
+
+    concat::fragment { 'storage':
+      target  => 'pulpcore settings',
+      content => "STORAGES = ${stdlib::to_python($storages)}\n",
+      order   => '03',
+    }
+  }
+
   file { $pulpcore::user_home:
     ensure => directory,
     owner  => $pulpcore::user,
     group  => $pulpcore::group,
     mode   => '0775',
   }
 
-  file { [$pulpcore::cache_dir, $pulpcore::media_root]:
+  file { $pulpcore::cache_dir:
     ensure => directory,
     owner  => $pulpcore::user,
     group  => $pulpcore::group,
     mode   => '0750',
   }
 
+  if $pulpcore::storage_backend == 'file' {
+    file { $pulpcore::media_root:
+      ensure => directory,
+      owner  => $pulpcore::user,
+      group  => $pulpcore::group,
+      mode   => '0750',
+    }
+  }
+
   file { unique($pulpcore::allowed_import_path + $pulpcore::allowed_export_path):
     ensure => directory,
     owner  => $pulpcore::user,

manifests/init.pp

@@ -214,6 +214,13 @@
 #   What percentage of available-workers will pulpcore use for import tasks at a time.
 #   By default, pulpcore will use all available workers.
 #
+# @param storage_backend
+#   Storage backend used for Pulp content. The default keeps local filesystem
+#   storage. Set to s3 to use S3 or an S3-compatible object store.
+#
+# @param storage_options
+#   Options passed to django-storages when storage_backend is s3.
+#
 # @example Default configuration
 #   include pulpcore
 #
@@ -282,6 +289,8 @@
   Optional[Boolean] $analytics = undef,
   Optional[Boolean] $hide_guarded_distributions = undef,
   Optional[Integer[1,100]] $import_workers_percent = undef,
+  Enum['file', 's3'] $storage_backend = 'file',
+  Hash[String[1], Any] $storage_options = {},
 ) {
   $settings_file = "${config_dir}/settings.py"
   $certs_dir = "${config_dir}/certs"

manifests/install.pp

@@ -6,6 +6,12 @@
     ensure => present,
   }
 
+  if $pulpcore::storage_backend == 's3' {
+    package { 'pulpcore-storage-s3':
+      ensure => present,
+    }
+  }
+
   if $facts['os']['selinux']['enabled'] {
     package { 'pulpcore-selinux':
       ensure => present,

spec/acceptance/storage_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper_acceptance'
+
+if ENV.fetch('BEAKER_FACTER_pulpcore_version', 'nightly') == 'nightly'
+  describe 'with S3-compatible storage' do
+    it_behaves_like 'an idempotent resource' do
+      let(:manifest) do
+        <<~PUPPET
+          class { 'pulpcore':
+            storage_backend => 's3',
+            storage_options => {
+              'access_key'       => 'test',
+              'secret_key'       => 'test',
+              'bucket_name'      => 'pulp',
+              'endpoint_url'     => 'https://object.example.test',
+              'addressing_style' => 'path',
+            },
+          }
+        PUPPET
+      end
+    end
+
+    describe command('rpm -q --whatprovides pulpcore-storage-s3') do
+      its(:exit_status) { is_expected.to eq 0 }
+    end
+
+    describe file('/etc/pulp/settings.py') do
+      it { is_expected.to be_file }
+      its(:content) { is_expected.not_to match(/^MEDIA_ROOT =/) }
+      its(:content) { is_expected.to include('"BACKEND": "storages.backends.s3.S3Storage"') }
+      its(:content) { is_expected.to include('"endpoint_url": "https://object.example.test"') }
+      its(:content) { is_expected.to include('"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"') }
+    end
+
+    describe command('PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager check') do
+      its(:exit_status) { is_expected.to eq 0 }
+    end
+  end
+end

spec/classes/pulpcore_spec.rb

@@ -12,6 +12,7 @@
           is_expected.to contain_class('pulpcore::install')
           is_expected.to contain_package('pulpcore')
           is_expected.to contain_package('pulpcore-selinux')
+          is_expected.not_to contain_package('pulpcore-storage-s3')
           is_expected.to contain_user('pulp').with_gid('pulp').with_home('/var/lib/pulp')
           is_expected.to contain_group('pulp')
         end
@@ -49,6 +50,7 @@
                 },
             }
           LOGGING
+          is_expected.not_to contain_concat__fragment('storage')
           is_expected.to contain_file('/etc/pulp')
           is_expected.to contain_file('/etc/pulp/certs/database_fields.symmetric.key')
           is_expected.to contain_file('/var/lib/pulp')
@@ -469,6 +471,38 @@
         end
       end
 
+      context 'with S3-compatible storage' do
+        let :params do
+          {
+            storage_backend: 's3',
+            storage_options: {
+              'bucket_name' => 'pulp',
+              'endpoint_url' => 'https://object.example.test',
+              'addressing_style' => 'path',
+            },
+          }
+        end
+
+        it { is_expected.to compile.with_all_deps }
+
+        it 'installs the storage dependencies' do
+          is_expected.to contain_package('pulpcore-storage-s3')
+        end
+
+        it 'configures S3 without changing static file storage' do
+          is_expected.to contain_concat__fragment('base')
+            .without_content(%r{^MEDIA_ROOT =})
+          is_expected.to contain_concat__fragment('storage').with_content(<<~STORAGE)
+            STORAGES = {"default": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": {"bucket_name": "pulp", "endpoint_url": "https://object.example.test", "addressing_style": "path"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}
+          STORAGE
+        end
+
+        it 'does not manage a local media directory' do
+          is_expected.not_to contain_file('/var/lib/pulp/media')
+          is_expected.to contain_file('/var/lib/pulp/tmp')
+        end
+      end
+
       context 'can disable services' do
         let :params do
           {

templates/settings.py.erb

@@ -43,7 +43,9 @@ REDIS_URL = "<%= @redis_url %>"
 WORKER_TTL = <%= scope['pulpcore::worker_ttl'] %>
 <% end -%>
 
+<% if scope['pulpcore::storage_backend'] == 'file' -%>
 MEDIA_ROOT = "<%= scope['pulpcore::media_root'] %>"
+<% end -%>
 STATIC_ROOT = "<%= scope['pulpcore::static_root'] %>"
 STATIC_URL = "<%= scope['pulpcore::static_url'] %>"
 FILE_UPLOAD_TEMP_DIR = "<%= scope['pulpcore::cache_dir'] %>"