Skip to content

Commit 7fc84d5

Browse files
zvonandianton-ru
zvonand
authored and
ianton-ru
committed
Merge pull request #1455 from Altinity/frontport/antalya-26.1/data_lake_namespace
26.1 Antalya port - DataLakeCatalog namespace filter
1 parent 3c28439 commit 7fc84d5

16 files changed

Lines changed: 468 additions & 35 deletions

File tree

‎docs/en/engines/database-engines/datalake.md‎

Lines changed: 42 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -46,21 +46,22 @@ catalog_type,
4646

4747
The following settings are supported:
4848

49-
| Setting | Description |
50-
|-------------------------|-----------------------------------------------------------------------------------------|
51-
| `catalog_type` | Type of catalog: `glue`, `unity` (Delta), `rest` (Iceberg), `hive`, `onelake` (Iceberg) |
52-
| `warehouse` | The warehouse/database name to use in the catalog. |
53-
| `catalog_credential` | Authentication credential for the catalog (e.g., API key or token) |
54-
| `auth_header` | Custom HTTP header for authentication with the catalog service |
55-
| `auth_scope` | OAuth2 scope for authentication (if using OAuth) |
56-
| `storage_endpoint` | Endpoint URL for the underlying storage |
57-
| `oauth_server_uri` | URI of the OAuth2 authorization server for authentication |
49+
| Setting | Description |
50+
|-------------------------|-----------------------------------------------------------------------------------------------|
51+
| `catalog_type` | Type of catalog: `glue`, `unity` (Delta), `rest` (Iceberg), `hive`, `onelake` (Iceberg) |
52+
| `warehouse` | The warehouse/database name to use in the catalog. |
53+
| `catalog_credential` | Authentication credential for the catalog (e.g., API key or token) |
54+
| `auth_header` | Custom HTTP header for authentication with the catalog service |
55+
| `auth_scope` | OAuth2 scope for authentication (if using OAuth) |
56+
| `storage_endpoint` | Endpoint URL for the underlying storage |
57+
| `oauth_server_uri` | URI of the OAuth2 authorization server for authentication |
5858
| `vended_credentials` | Boolean indicating whether to use vended credentials from the catalog (supports AWS S3 and Azure ADLS Gen2) |
59-
| `aws_access_key_id` | AWS access key ID for S3/Glue access (if not using vended credentials) |
60-
| `aws_secret_access_key` | AWS secret access key for S3/Glue access (if not using vended credentials) |
61-
| `region` | AWS region for the service (e.g., `us-east-1`) |
62-
| `dlf_access_key_id` | Access key ID for DLF access |
63-
| `dlf_access_key_secret` | Access key Secret for DLF access |
59+
| `aws_access_key_id` | AWS access key ID for S3/Glue access (if not using vended credentials) |
60+
| `aws_secret_access_key` | AWS secret access key for S3/Glue access (if not using vended credentials) |
61+
| `region` | AWS region for the service (e.g., `us-east-1`) |
62+
| `dlf_access_key_id` | Access key ID for DLF access |
63+
| `dlf_access_key_secret` | Access key Secret for DLF access |
64+
| `namespaces` | Comma-separated list of namespaces, implemented for catalog types: `rest`, `glue` and `unity` |
6465

6566
## Examples {#examples}
6667

@@ -83,4 +84,30 @@ SETTINGS
8384
onelake_client_secret = client_secret;
8485
SHOW TABLES IN databse_name;
8586
SELECT count() from database_name.table_name;
86-
```
87+
```
88+
89+
## Namespace filter {#namespace}
90+
91+
By default, ClickHouse reads tables from all namespaces available in the catalog. You can limit this behavior using the `namespaces` database setting. The value should be a comma‑separated list of namespaces that are allowed to be read.
92+
93+
Supported catalog types are `rest`, `glue` and `unity`.
94+
95+
For example, if the catalog contains three namespaces - `dev`, `stage`, and `prod` - and you want to read data only from dev and stage, set:
96+
```
97+
namespaces='dev,stage'
98+
```
99+
100+
### Nested namespaces {#namespace-nested}
101+
102+
The Iceberg (`rest`) catalog supports nested namespaces. The `namespaces` filter accepts the following patterns:
103+
104+
- `namespace` - includes tables from the specified namespace, but not from its nested namespaces.
105+
- `namespace.nested` - includes tables from the nested namespace, but not from the parent.
106+
- `namespace.*` - includes tables from all nested namespaces, but not from the parent.
107+
108+
If you need to include both a namespace and its nested namespaces, specify both explicitly. For example:
109+
```
110+
namespaces='namespace,namespace.*'
111+
```
112+
113+
The default value is '*', which means all namespaces are included.

‎src/Common/ErrorCodes.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,7 @@
650650
M(768, CANNOT_EXECUTE_PROMQL_QUERY) \
651651
M(769, NAMED_COLLECTION_IS_USED) \
652652
M(770, WASM_ERROR) \
653+
M(771, CATALOG_NAMESPACE_DISABLED) \
653654
\
654655
M(900, DISTRIBUTED_CACHE_ERROR) \
655656
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \

‎src/Databases/DataLake/DatabaseDataLake.cpp‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ namespace DatabaseDataLakeSetting
7272
extern const DatabaseDataLakeSettingsString onelake_client_secret;
7373
extern const DatabaseDataLakeSettingsString dlf_access_key_id;
7474
extern const DatabaseDataLakeSettingsString dlf_access_key_secret;
75+
extern const DatabaseDataLakeSettingsString namespaces;
7576
extern const DatabaseDataLakeSettingsString google_project_id;
7677
extern const DatabaseDataLakeSettingsString google_service_account;
7778
extern const DatabaseDataLakeSettingsString google_metadata_service;
@@ -165,8 +166,9 @@ std::shared_ptr<DataLake::ICatalog> DatabaseDataLake::getCatalog() const
165166
.aws_access_key_id = settings[DatabaseDataLakeSetting::aws_access_key_id].value,
166167
.aws_secret_access_key = settings[DatabaseDataLakeSetting::aws_secret_access_key].value,
167168
.region = settings[DatabaseDataLakeSetting::region].value,
169+
.namespaces = settings[DatabaseDataLakeSetting::namespaces].value,
168170
.aws_role_arn = settings[DatabaseDataLakeSetting::aws_role_arn].value,
169-
.aws_role_session_name = settings[DatabaseDataLakeSetting::aws_role_session_name].value,
171+
.aws_role_session_name = settings[DatabaseDataLakeSetting::aws_role_session_name].value
170172
};
171173

172174
switch (settings[DatabaseDataLakeSetting::catalog_type].value)
@@ -181,6 +183,7 @@ std::shared_ptr<DataLake::ICatalog> DatabaseDataLake::getCatalog() const
181183
settings[DatabaseDataLakeSetting::auth_header],
182184
settings[DatabaseDataLakeSetting::oauth_server_uri].value,
183185
settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value,
186+
settings[DatabaseDataLakeSetting::namespaces].value,
184187
Context::getGlobalContextInstance());
185188
break;
186189
}
@@ -195,6 +198,7 @@ std::shared_ptr<DataLake::ICatalog> DatabaseDataLake::getCatalog() const
195198
settings[DatabaseDataLakeSetting::auth_scope].value,
196199
settings[DatabaseDataLakeSetting::oauth_server_uri].value,
197200
settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value,
201+
settings[DatabaseDataLakeSetting::namespaces].value,
198202
Context::getGlobalContextInstance());
199203
break;
200204
}
@@ -225,6 +229,7 @@ std::shared_ptr<DataLake::ICatalog> DatabaseDataLake::getCatalog() const
225229
google_adc_client_secret,
226230
google_adc_refresh_token,
227231
google_adc_quota_project_id,
232+
settings[DatabaseDataLakeSetting::namespaces].value,
228233
Context::getGlobalContextInstance());
229234
break;
230235
}
@@ -234,6 +239,7 @@ std::shared_ptr<DataLake::ICatalog> DatabaseDataLake::getCatalog() const
234239
settings[DatabaseDataLakeSetting::warehouse].value,
235240
url,
236241
settings[DatabaseDataLakeSetting::catalog_credential].value,
242+
settings[DatabaseDataLakeSetting::namespaces].value,
237243
Context::getGlobalContextInstance());
238244
break;
239245
}
@@ -330,24 +336,24 @@ StorageObjectStorageConfigurationPtr DatabaseDataLake::getConfiguration(
330336
#if USE_AWS_S3
331337
case DB::DatabaseDataLakeStorageType::S3:
332338
{
333-
return std::make_shared<StorageS3IcebergConfiguration>(storage_settings);
339+
return std::make_shared<StorageS3IcebergConfiguration>(storage_settings, settings[DatabaseDataLakeSetting::namespaces].value);
334340
}
335341
#endif
336342
#if USE_AZURE_BLOB_STORAGE
337343
case DB::DatabaseDataLakeStorageType::Azure:
338344
{
339-
return std::make_shared<StorageAzureIcebergConfiguration>(storage_settings);
345+
return std::make_shared<StorageAzureIcebergConfiguration>(storage_settings, settings[DatabaseDataLakeSetting::namespaces].value);
340346
}
341347
#endif
342348
#if USE_HDFS
343349
case DB::DatabaseDataLakeStorageType::HDFS:
344350
{
345-
return std::make_shared<StorageHDFSIcebergConfiguration>(storage_settings);
351+
return std::make_shared<StorageHDFSIcebergConfiguration>(storage_settings, settings[DatabaseDataLakeSetting::namespaces].value);
346352
}
347353
#endif
348354
case DB::DatabaseDataLakeStorageType::Local:
349355
{
350-
return std::make_shared<StorageLocalIcebergConfiguration>(storage_settings);
356+
return std::make_shared<StorageLocalIcebergConfiguration>(storage_settings, settings[DatabaseDataLakeSetting::namespaces].value);
351357
}
352358
/// Fake storage in case when catalog store not only
353359
/// primary-type tables (DeltaLake or Iceberg), but for
@@ -359,7 +365,7 @@ StorageObjectStorageConfigurationPtr DatabaseDataLake::getConfiguration(
359365
/// dependencies and the most lightweight
360366
case DB::DatabaseDataLakeStorageType::Other:
361367
{
362-
return std::make_shared<StorageLocalIcebergConfiguration>(storage_settings);
368+
return std::make_shared<StorageLocalIcebergConfiguration>(storage_settings, settings[DatabaseDataLakeSetting::namespaces].value);
363369
}
364370
#if !USE_AWS_S3 || !USE_AZURE_BLOB_STORAGE || !USE_HDFS
365371
default:
@@ -376,7 +382,7 @@ StorageObjectStorageConfigurationPtr DatabaseDataLake::getConfiguration(
376382
#if USE_AWS_S3
377383
case DB::DatabaseDataLakeStorageType::S3:
378384
{
379-
return std::make_shared<StorageS3DeltaLakeConfiguration>(storage_settings);
385+
return std::make_shared<StorageS3DeltaLakeConfiguration>(storage_settings, settings[DatabaseDataLakeSetting::namespaces].value);
380386
}
381387
#endif
382388
#if USE_AZURE_BLOB_STORAGE
@@ -387,7 +393,7 @@ StorageObjectStorageConfigurationPtr DatabaseDataLake::getConfiguration(
387393
#endif
388394
case DB::DatabaseDataLakeStorageType::Local:
389395
{
390-
return std::make_shared<StorageLocalDeltaLakeConfiguration>(storage_settings);
396+
return std::make_shared<StorageLocalDeltaLakeConfiguration>(storage_settings, settings[DatabaseDataLakeSetting::namespaces].value);
391397
}
392398
/// Fake storage in case when catalog store not only
393399
/// primary-type tables (DeltaLake or Iceberg), but for
@@ -399,7 +405,7 @@ StorageObjectStorageConfigurationPtr DatabaseDataLake::getConfiguration(
399405
/// dependencies and the most lightweight
400406
case DB::DatabaseDataLakeStorageType::Other:
401407
{
402-
return std::make_shared<StorageLocalDeltaLakeConfiguration>(storage_settings);
408+
return std::make_shared<StorageLocalDeltaLakeConfiguration>(storage_settings, settings[DatabaseDataLakeSetting::namespaces].value);
403409
}
404410
default:
405411
throw Exception(ErrorCodes::BAD_ARGUMENTS,
@@ -414,12 +420,12 @@ StorageObjectStorageConfigurationPtr DatabaseDataLake::getConfiguration(
414420
#if USE_AWS_S3
415421
case DB::DatabaseDataLakeStorageType::S3:
416422
{
417-
return std::make_shared<StorageS3IcebergConfiguration>(storage_settings);
423+
return std::make_shared<StorageS3IcebergConfiguration>(storage_settings, settings[DatabaseDataLakeSetting::namespaces].value);
418424
}
419425
#endif
420426
case DB::DatabaseDataLakeStorageType::Other:
421427
{
422-
return std::make_shared<StorageLocalIcebergConfiguration>(storage_settings);
428+
return std::make_shared<StorageLocalIcebergConfiguration>(storage_settings, settings[DatabaseDataLakeSetting::namespaces].value);
423429
}
424430
default:
425431
throw Exception(ErrorCodes::BAD_ARGUMENTS,

‎src/Databases/DataLake/DatabaseDataLakeSettings.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ namespace ErrorCodes
4444
DECLARE(String, google_adc_credentials_file, "", "Deprecated setting, will throw an exception if used", 0) \
4545
DECLARE(String, dlf_access_key_id, "", "Access id of DLF token for Paimon REST Catalog", 0) \
4646
DECLARE(String, dlf_access_key_secret, "", "Access secret of DLF token for Paimon REST Catalog", 0) \
47+
DECLARE(String, namespaces, "*", "Comma-separated list of allowed namespaces", 0) \
4748

4849
#define LIST_OF_DATABASE_ICEBERG_SETTINGS(M, ALIAS) \
4950
DATABASE_ICEBERG_RELATED_SETTINGS(M, ALIAS) \

‎src/Databases/DataLake/GlueCatalog.cpp‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ namespace DB::ErrorCodes
5656
{
5757
extern const int BAD_ARGUMENTS;
5858
extern const int DATALAKE_DATABASE_ERROR;
59+
extern const int CATALOG_NAMESPACE_DISABLED;
5960
}
6061

6162
namespace DB::Setting
@@ -163,9 +164,9 @@ GlueCatalog::GlueCatalog(
163164
LOG_TRACE(log, "Creating AWS glue client with credentials empty {}, region '{}', endpoint '{}'", credentials.IsEmpty(), region, endpoint);
164165
}
165166

167+
boost::split(allowed_namespaces, settings.namespaces, boost::is_any_of(", "), boost::token_compress_on);
166168
credentials_provider = DB::S3::getCredentialsProvider(poco_config, credentials, creds_config);
167169
glue_client = std::make_unique<Aws::Glue::GlueClient>(credentials_provider, endpoint_provider, client_configuration);
168-
169170
}
170171

171172
GlueCatalog::~GlueCatalog() = default;
@@ -191,8 +192,9 @@ DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & pre
191192
for (const auto & db : dbs)
192193
{
193194
const auto & db_name = db.GetName();
194-
if (!db_name.starts_with(prefix))
195+
if (!isNamespaceAllowed(db_name) || !db_name.starts_with(prefix))
195196
continue;
197+
196198
result.push_back(db_name);
197199
if (limit != 0 && result.size() >= limit)
198200
break;
@@ -272,6 +274,9 @@ DB::Names GlueCatalog::getTables() const
272274

273275
bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name) const
274276
{
277+
if (!isNamespaceAllowed(database_name))
278+
throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name);
279+
275280
Aws::Glue::Model::GetTableRequest request;
276281
request.SetDatabaseName(database_name);
277282
request.SetName(table_name);
@@ -286,6 +291,9 @@ bool GlueCatalog::tryGetTableMetadata(
286291
DB::ContextPtr /* context_ */,
287292
TableMetadata & result) const
288293
{
294+
if (!isNamespaceAllowed(database_name))
295+
throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name);
296+
289297
Aws::Glue::Model::GetTableRequest request;
290298
request.SetDatabaseName(database_name);
291299
request.SetName(table_name);
@@ -510,7 +518,7 @@ GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTable
510518

511519
auto storage_settings = std::make_shared<DB::DataLakeStorageSettings>();
512520
storage_settings->loadFromSettingsChanges(settings.allChanged());
513-
auto configuration = std::make_shared<DB::StorageS3IcebergConfiguration>(storage_settings);
521+
auto configuration = std::make_shared<DB::StorageS3IcebergConfiguration>(storage_settings, settings.namespaces);
514522
configuration->initialize(args, getContext(), false);
515523

516524
auto object_storage = configuration->createObjectStorage(getContext(), true, {});
@@ -578,6 +586,11 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name) cons
578586

579587
void GlueCatalog::createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*metadata_content*/) const
580588
{
589+
if (!isNamespaceAllowed(namespace_name))
590+
throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED,
591+
"Failed to create table {}, namespace {} is filtered by `namespaces` database parameter",
592+
table_name, namespace_name);
593+
581594
createNamespaceIfNotExists(namespace_name);
582595

583596
Aws::Glue::Model::CreateTableRequest request;
@@ -650,6 +663,11 @@ bool GlueCatalog::updateMetadata(const String & namespace_name, const String & t
650663

651664
void GlueCatalog::dropTable(const String & namespace_name, const String & table_name) const
652665
{
666+
if (!isNamespaceAllowed(namespace_name))
667+
throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED,
668+
"Failed to drop table {}, namespace {} is filtered by `namespaces` database parameter",
669+
table_name, namespace_name);
670+
653671
Aws::Glue::Model::DeleteTableRequest request;
654672
request.SetDatabaseName(namespace_name);
655673
request.SetName(table_name);
@@ -663,6 +681,11 @@ void GlueCatalog::dropTable(const String & namespace_name, const String & table_
663681
response.GetError().GetMessage());
664682
}
665683

684+
bool GlueCatalog::isNamespaceAllowed(const std::string & namespace_) const
685+
{
686+
return allowed_namespaces.contains("*") || allowed_namespaces.contains(namespace_);
687+
}
688+
666689
}
667690

668691
#endif

‎src/Databases/DataLake/GlueCatalog.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ class GlueCatalog final : public ICatalog, private DB::WithContext
8080
std::string region;
8181
CatalogSettings settings;
8282
DB::ASTPtr table_engine_definition;
83+
std::unordered_set<std::string> allowed_namespaces;
84+
85+
bool isNamespaceAllowed(const std::string & namespace_) const;
8386

8487
DataLake::ICatalog::Namespaces getDatabases(const std::string & prefix, size_t limit = 0) const;
8588
DB::Names getTablesForDatabase(const std::string & db_name, size_t limit = 0) const;

‎src/Databases/DataLake/ICatalog.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ struct CatalogSettings
130130
String aws_access_key_id;
131131
String aws_secret_access_key;
132132
String region;
133+
String namespaces;
133134
String aws_role_arn;
134135
String aws_role_session_name;
135136

0 commit comments

Comments
 (0)