From 80b156538f2097bcd1c107f224f82e1a1e9e48f4 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 3 Oct 2026 23:43:11 -0400 Subject: [PATCH 01/13] feat(gateways): add Kafka DescribeConfigs and AlterConfigs Kafka admin clients need to read and set topic retention. DescribeConfigs reports retention.ms from Iggy message expiry and a read-only cleanup.policy. AlterConfigs persists only retention.ms, capped so segment expiry cannot wrap. --- gateways/kafka/docs/CONFIGS.md | 67 +++ .../src/protocol/handlers/alter_configs.rs | 359 ++++++++++++ .../src/protocol/handlers/describe_configs.rs | 439 +++++++++++++++ .../src/protocol/handlers/topic_config.rs | 348 ++++++++++++ gateways/kafka/tests/configs_handler_tests.rs | 113 ++++ .../kafka/tests/configs_java_client_tests.rs | 265 +++++++++ .../kafka/tests/configs_real_bridge_tests.rs | 521 ++++++++++++++++++ gateways/kafka/tests/java/ConfigsProbe.java | 129 +++++ 8 files changed, 2241 insertions(+) create mode 100644 gateways/kafka/docs/CONFIGS.md create mode 100644 gateways/kafka/src/protocol/handlers/alter_configs.rs create mode 100644 gateways/kafka/src/protocol/handlers/describe_configs.rs create mode 100644 gateways/kafka/src/protocol/handlers/topic_config.rs create mode 100644 gateways/kafka/tests/configs_handler_tests.rs create mode 100644 gateways/kafka/tests/configs_java_client_tests.rs create mode 100644 gateways/kafka/tests/configs_real_bridge_tests.rs create mode 100644 gateways/kafka/tests/java/ConfigsProbe.java diff --git a/gateways/kafka/docs/CONFIGS.md b/gateways/kafka/docs/CONFIGS.md new file mode 100644 index 0000000000..b734ca0a16 --- /dev/null +++ b/gateways/kafka/docs/CONFIGS.md @@ -0,0 +1,67 @@ +# Topic configuration + +`DescribeConfigs` (32) and `AlterConfigs` (33) apply to Kafka topic resources only +(resource type 2). Only the deprecated `Admin.alterConfigs` (API 33) writes +`retention.ms`. `IncrementalAlterConfigs` (44) stays unimplemented: it is not +advertised and closes the connection like every other unlisted key. + +With the bridge off, both APIs answer `NOT_CONTROLLER` (41) for each resource and +do not read or write Iggy. + +## What a describe returns + +A null or empty `configuration_keys` list returns both keys below. A non-empty +list returns only the requested names, in request order. A name this gateway does +not know is still returned, and that resource's error code is `INVALID_CONFIG` +(40). + +| Key | Value | Read only | Source | +| --- | ----- | --------- | ------ | +| `retention.ms` | Sealed-segment lifetime from Iggy `message_expiry`, or `-1` when those segments never expire | no | `DEFAULT_CONFIG` (5) for the unset never-expire default; `DYNAMIC_TOPIC_CONFIG` (1) when a client set it or a stored duration cannot be shown as milliseconds | +| `cleanup.policy` | `delete` | yes | `DEFAULT_CONFIG` (5) | + +`is_sensitive` is false for both. `include_synonyms` is honored and the synonym +list is empty either way. From v3, `include_documentation` is honored: true +attaches one sentence per known key, false sends no documentation. + +Iggy stores `message_expiry` as `IggyExpiry`. A duration is `IggyDuration`, and +that type counts microseconds. Retention applies to sealed segments only. Iggy +does not time-roll, so the active segment does not expire. Describe divides by +1000 only when the stored count is a positive whole number of milliseconds. +Anything else, including 0 microseconds, is `INVALID_CONFIG` (40) with a null +value and source `DYNAMIC_TOPIC_CONFIG` (1). That expiry was stored; it is not +the never-expire default. The gateway does not round. + +## What an alter stores + +Only `retention.ms` is written, through `update_topic`'s `message_expiry`. + +- `-1` stores `IggyExpiry::NeverExpire`. The server treats `0` + (`IggyExpiry::ServerDefault`) as "leave the current expiry", so `-1` is not + sent as 0. A following describe reports `-1` with source + `DYNAMIC_TOPIC_CONFIG`. +- A positive millisecond count is multiplied by 1000 and stored as + `IggyExpiry::ExpireDuration`. The text must be the canonical decimal of that + integer (`1`, not `0001` or `+1`). The duration must be at most `u32::MAX` + seconds, the same limit `IggyExpiry::from_str` enforces. A longer value is + `INVALID_CONFIG` (40) and is not stored. A following describe reports the same + decimal with source `DYNAMIC_TOPIC_CONFIG`. +- `0` and every other value are `INVALID_CONFIG` (40). + +`cleanup.policy` and any other key are `INVALID_CONFIG` (40). They are not +stored. Kafka has no `NOT_CONFIGURABLE` code. One bad key fails that resource +and writes nothing for it. Other resources in the same request are still +processed. + +`validate_only` runs the same checks, including the topic lookup, and does not +call `update_topic`. + +## Other resource errors + +| Condition | Code | Notes | +| --------- | ---- | ----- | +| Resource type other than topic | `INVALID_REQUEST` (42) | Message is `only topic resources are supported`. Other resources in the batch still run | +| Topic name fails the same rules as CreateTopics | `INVALID_TOPIC_EXCEPTION` (17) | The message is the validation reason and does not repeat the topic name | +| Topic does not exist | `UNKNOWN_TOPIC_OR_PARTITION` (3) | Checked before config keys, so a missing topic is not `INVALID_CONFIG` | +| More than 100 distinct topic names | `POLICY_VIOLATION` (44) | Every resource in the request. Duplicate names count once and are not an error by themselves | +| Non-empty tagged fields | `INVALID_REQUEST` (42) | Request-level tags fail every resource. Per-resource or per-entry tags fail that resource | diff --git a/gateways/kafka/src/protocol/handlers/alter_configs.rs b/gateways/kafka/src/protocol/handlers/alter_configs.rs new file mode 100644 index 0000000000..26154fb757 --- /dev/null +++ b/gateways/kafka/src/protocol/handlers/alter_configs.rs @@ -0,0 +1,359 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `AlterConfigs` (API key 33). + +use std::collections::HashMap; + +use bytes::Bytes; +use iggy::prelude::IggyError; +use kafka_protocol::messages::AlterConfigsRequest; +use kafka_protocol::messages::alter_configs_request::AlterConfigsResource; +use kafka_protocol::messages::alter_configs_response::{ + AlterConfigsResourceResponse, AlterConfigsResponse, +}; +use kafka_protocol::protocol::StrBytes; +use tokio::time::Instant; + +use crate::bridge::topic_map::validate_kafka_topic_name; +use crate::bridge::{BridgeError, IggyBridge}; +use crate::protocol::api::{ + API_KEY_ALTER_CONFIGS, ApiVersionRange, ERROR_INVALID_CONFIG, ERROR_INVALID_REQUEST, + ERROR_INVALID_TOPIC_EXCEPTION, ERROR_NONE, ERROR_NOT_CONTROLLER, ERROR_POLICY_VIOLATION, + ERROR_REQUEST_TIMED_OUT, ERROR_UNKNOWN_TOPIC_OR_PARTITION, GatewayState, HandleOutcome, + REQUEST_DEADLINE, +}; +use crate::protocol::bounds_guard::validate_alter_configs_shape; +use crate::protocol::handlers::topic_config::{ + ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, UNKNOWN_TAGGED_FIELDS, exceeds_topic_cap, + plan_retention_update, topic_cap_message, +}; +use crate::protocol::handlers::{ + decode_guarded, encode_message, is_supported_version, respond_or_close, + unsupported_version_response, +}; + +pub const RANGE: ApiVersionRange = ApiVersionRange { + api_key: API_KEY_ALTER_CONFIGS, + min_version: 0, + max_version: 2, +}; + +pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> HandleOutcome { + if !is_supported_version(API_KEY_ALTER_CONFIGS, api_version) { + return unsupported_version_response(API_KEY_ALTER_CONFIGS, api_version, |version| { + encode_error_response(version, ERROR_INVALID_REQUEST) + }); + } + + let req = match decode_guarded::(api_version, body, |version, body| { + validate_alter_configs_shape(version, body, state.max_frame_size) + }) { + Ok(req) => req, + Err(error) => { + tracing::debug!(%error, "Failed to decode AlterConfigs request"); + return respond_or_close( + encode_error_response(api_version, ERROR_INVALID_REQUEST), + "AlterConfigs", + ); + } + }; + + if !req.unknown_tagged_fields.is_empty() { + return respond_or_close( + encode_message( + &response(reject_tagged_resources(&req.resources)), + api_version, + 256, + ), + "AlterConfigs", + ); + } + + let Some(bridge) = &state.bridge else { + let responses = req + .resources + .iter() + .map(|resource| resource_error(resource, ERROR_NOT_CONTROLLER, None)) + .collect(); + return respond_or_close( + encode_message(&response(responses), api_version, 256), + "AlterConfigs", + ); + }; + + let deadline = Instant::now() + REQUEST_DEADLINE; + let responses = alter_all(bridge, &req, deadline).await; + respond_or_close( + encode_message(&response(responses), api_version, 256), + "AlterConfigs", + ) +} + +/// # Errors +/// +/// Returns an error when `kafka_protocol` cannot encode the response at `version`. +pub fn encode_error_response(version: i16, error_code: i16) -> crate::error::Result { + let responses = vec![ + AlterConfigsResourceResponse::default() + .with_error_code(error_code) + .with_error_message(None) + .with_resource_type(RESOURCE_TYPE_TOPIC) + .with_resource_name(StrBytes::from_static_str("")), + ]; + encode_message(&response(responses), version, 128) +} + +fn response(responses: Vec) -> AlterConfigsResponse { + AlterConfigsResponse::default() + .with_throttle_time_ms(0) + .with_responses(responses) +} + +async fn alter_all( + bridge: &IggyBridge, + req: &AlterConfigsRequest, + deadline: Instant, +) -> Vec { + let topic_names = req + .resources + .iter() + .filter(|resource| resource.resource_type == RESOURCE_TYPE_TOPIC) + .map(|resource| resource.resource_name.as_str()); + if exceeds_topic_cap(topic_names) { + let message = StrBytes::from(topic_cap_message("AlterConfigs")); + tracing::warn!("AlterConfigs request addresses too many distinct topics; rejecting"); + return req + .resources + .iter() + .map(|resource| resource_error(resource, ERROR_POLICY_VIOLATION, Some(message.clone()))) + .collect(); + } + + let mut seen: HashMap = HashMap::new(); + let mut responses = Vec::with_capacity(req.resources.len()); + let mut deadline_exceeded = false; + for resource in &req.resources { + if resource.resource_type == RESOURCE_TYPE_TOPIC + && let Some(prior) = seen.get(resource.resource_name.as_str()) + { + responses.push(prior.clone()); + continue; + } + if deadline_exceeded || Instant::now() >= deadline { + if !deadline_exceeded { + deadline_exceeded = true; + tracing::warn!( + "AlterConfigs deadline passed; answering remaining resources retriable \ + instead of starting new Iggy calls" + ); + } + let timed_out = resource_error(resource, ERROR_REQUEST_TIMED_OUT, None); + remember_topic(&mut seen, resource, &timed_out); + responses.push(timed_out); + continue; + } + + let result = alter_one(bridge, resource, req.validate_only, deadline).await; + if Instant::now() >= deadline { + deadline_exceeded = true; + } + remember_topic(&mut seen, resource, &result); + responses.push(result); + } + responses +} + +fn remember_topic( + seen: &mut HashMap, + resource: &AlterConfigsResource, + result: &AlterConfigsResourceResponse, +) { + if resource.resource_type == RESOURCE_TYPE_TOPIC { + seen.insert(resource.resource_name.as_str().to_string(), result.clone()); + } +} + +async fn alter_one( + bridge: &IggyBridge, + resource: &AlterConfigsResource, + validate_only: bool, + deadline: Instant, +) -> AlterConfigsResourceResponse { + if !resource.unknown_tagged_fields.is_empty() + || resource + .configs + .iter() + .any(|config| !config.unknown_tagged_fields.is_empty()) + { + return resource_error( + resource, + ERROR_INVALID_REQUEST, + Some(static_text(UNKNOWN_TAGGED_FIELDS)), + ); + } + if resource.resource_type != RESOURCE_TYPE_TOPIC { + return resource_error( + resource, + ERROR_INVALID_REQUEST, + Some(static_text(ONLY_TOPIC_RESOURCES)), + ); + } + let name = resource.resource_name.as_str(); + if let Err(error) = validate_kafka_topic_name("kafka_topic", name) { + return resource_error( + resource, + ERROR_INVALID_TOPIC_EXCEPTION, + Some(StrBytes::from(name_reason(&error))), + ); + } + + // Existence is checked before config keys. A missing topic is + // UNKNOWN_TOPIC_OR_PARTITION even when a key would also be invalid. + // A deadline that has already passed must not start the lockstep Iggy call. + if Instant::now() >= deadline { + tracing::warn!( + resource = name, + "AlterConfigs deadline passed before the topic lookup" + ); + return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None); + } + match bridge.get_kafka_topic(name).await { + Ok(Some(_topic)) => {} + Ok(None) => { + return resource_error(resource, ERROR_UNKNOWN_TOPIC_OR_PARTITION, None); + } + Err(error) => { + let (code, message) = bridge_failure(&error); + return resource_error(resource, code, message); + } + } + + let expiry = match plan_retention_update(resource.configs.iter().map(|config| { + ( + config.name.as_str(), + config.value.as_ref().map(StrBytes::as_str), + ) + })) { + Ok(expiry) => expiry, + Err(fault) => { + return resource_error( + resource, + ERROR_INVALID_CONFIG, + Some(StrBytes::from(fault.message())), + ); + } + }; + + // `validate_only` already performed the existence read and the same key checks. + // It does not call `update_topic`. + let Some(expiry) = expiry else { + return resource_error(resource, ERROR_NONE, None); + }; + if validate_only { + return resource_error(resource, ERROR_NONE, None); + } + + // Do not start the write once the deadline has passed, and do not wrap it in + // `timeout_at`. Dropping that future does not abort the SDK task, so + // `UpdateTopic` can commit after this handler has already answered + // REQUEST_TIMED_OUT. A short retention then deletes sealed segments. + if Instant::now() >= deadline { + tracing::warn!( + resource = name, + "AlterConfigs deadline passed before update_topic; not starting the write" + ); + return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None); + } + match bridge.update_kafka_topic_message_expiry(name, expiry).await { + Ok(()) | Err(BridgeError::Iggy(IggyError::RequestAlreadyApplied)) => { + resource_error(resource, ERROR_NONE, None) + } + Err(error) => { + let (code, message) = bridge_failure(&error); + resource_error(resource, code, message) + } + } +} + +fn reject_tagged_resources( + resources: &[AlterConfigsResource], +) -> Vec { + if resources.is_empty() { + return vec![ + AlterConfigsResourceResponse::default() + .with_error_code(ERROR_INVALID_REQUEST) + .with_error_message(Some(static_text(UNKNOWN_TAGGED_FIELDS))) + .with_resource_type(0) + .with_resource_name(StrBytes::from_static_str("")), + ]; + } + resources + .iter() + .map(|resource| { + resource_error( + resource, + ERROR_INVALID_REQUEST, + Some(static_text(UNKNOWN_TAGGED_FIELDS)), + ) + }) + .collect() +} + +fn resource_error( + resource: &AlterConfigsResource, + error_code: i16, + error_message: Option, +) -> AlterConfigsResourceResponse { + AlterConfigsResourceResponse::default() + .with_error_code(error_code) + .with_error_message(error_message) + .with_resource_type(resource.resource_type) + .with_resource_name(resource.resource_name.clone()) +} + +fn name_reason(error: &BridgeError) -> String { + match error { + BridgeError::InvalidKafkaTopicName { reason, .. } => reason.clone(), + _ => "invalid topic name".to_string(), + } +} + +fn bridge_failure(error: &BridgeError) -> (i16, Option) { + let code = error.to_kafka_error_code(); + match error { + BridgeError::InvalidKafkaTopicName { reason, .. } => { + tracing::debug!(reason, "AlterConfigs rejected an invalid topic name"); + (code, Some(StrBytes::from(reason.clone()))) + } + BridgeError::Timeout | BridgeError::SendLost(_) => { + tracing::warn!(%error, "AlterConfigs exceeded the bridge deadline"); + (code, None) + } + other => { + tracing::error!(%other, "AlterConfigs failed while updating topic configuration"); + ( + code, + Some(static_text("internal error updating topic configuration")), + ) + } + } +} + +const fn static_text(message: &'static str) -> StrBytes { + StrBytes::from_static_str(message) +} diff --git a/gateways/kafka/src/protocol/handlers/describe_configs.rs b/gateways/kafka/src/protocol/handlers/describe_configs.rs new file mode 100644 index 0000000000..530b305a41 --- /dev/null +++ b/gateways/kafka/src/protocol/handlers/describe_configs.rs @@ -0,0 +1,439 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `DescribeConfigs` (API key 32). + +use std::collections::HashMap; + +use bytes::Bytes; +use iggy::prelude::TopicDetails; +use kafka_protocol::messages::DescribeConfigsRequest; +use kafka_protocol::messages::describe_configs_request::DescribeConfigsResource; +use kafka_protocol::messages::describe_configs_response::{ + DescribeConfigsResourceResult, DescribeConfigsResponse, DescribeConfigsResult, + DescribeConfigsSynonym, +}; +use kafka_protocol::protocol::StrBytes; +use tokio::time::Instant; + +use crate::bridge::topic_map::validate_kafka_topic_name; +use crate::bridge::{BridgeError, IggyBridge}; +use crate::protocol::api::{ + API_KEY_DESCRIBE_CONFIGS, ApiVersionRange, ERROR_INVALID_CONFIG, ERROR_INVALID_REQUEST, + ERROR_INVALID_TOPIC_EXCEPTION, ERROR_NONE, ERROR_NOT_CONTROLLER, ERROR_POLICY_VIOLATION, + ERROR_REQUEST_TIMED_OUT, ERROR_UNKNOWN_TOPIC_OR_PARTITION, GatewayState, HandleOutcome, + REQUEST_DEADLINE, +}; +use crate::protocol::bounds_guard::validate_describe_configs_shape; +use crate::protocol::handlers::topic_config::{ + CLEANUP_DOC, CLEANUP_POLICY, CLEANUP_POLICY_VALUE, CONFIG_SOURCE_DEFAULT, CONFIG_SOURCE_TOPIC, + CONFIG_TYPE_LONG, CONFIG_TYPE_STRING, ListedKey, ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, + RETENTION_DOC, RETENTION_MS, UNKNOWN_TAGGED_FIELDS, exceeds_topic_cap, listed_keys, + message_expiry_is_explicit, retention_ms, topic_cap_message, +}; +use crate::protocol::handlers::{ + decode_guarded, encode_message, is_supported_version, respond_or_close, + unsupported_version_response, +}; + +pub const RANGE: ApiVersionRange = ApiVersionRange { + api_key: API_KEY_DESCRIBE_CONFIGS, + min_version: 1, + max_version: 4, +}; + +pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> HandleOutcome { + if !is_supported_version(API_KEY_DESCRIBE_CONFIGS, api_version) { + return unsupported_version_response(API_KEY_DESCRIBE_CONFIGS, api_version, |version| { + encode_error_response(version, ERROR_INVALID_REQUEST) + }); + } + + let req = match decode_guarded::(api_version, body, |version, body| { + validate_describe_configs_shape(version, body, state.max_frame_size) + }) { + Ok(req) => req, + Err(error) => { + tracing::debug!(%error, "Failed to decode DescribeConfigs request"); + return respond_or_close( + encode_error_response(api_version, ERROR_INVALID_REQUEST), + "DescribeConfigs", + ); + } + }; + + if !req.unknown_tagged_fields.is_empty() { + return respond_or_close( + encode_message( + &response(reject_tagged_resources(&req.resources)), + api_version, + 256, + ), + "DescribeConfigs", + ); + } + + let Some(bridge) = &state.bridge else { + let results = req + .resources + .iter() + .map(|resource| resource_error(resource, ERROR_NOT_CONTROLLER, None, Vec::new())) + .collect(); + return respond_or_close( + encode_message(&response(results), api_version, 256), + "DescribeConfigs", + ); + }; + + let deadline = Instant::now() + REQUEST_DEADLINE; + let results = describe_all(bridge, &req, deadline).await; + respond_or_close( + encode_message(&response(results), api_version, 256), + "DescribeConfigs", + ) +} + +/// # Errors +/// +/// Returns an error when `kafka_protocol` cannot encode the response at `version`. +pub fn encode_error_response(version: i16, error_code: i16) -> crate::error::Result { + let results = vec![ + DescribeConfigsResult::default() + .with_error_code(error_code) + .with_error_message(None) + .with_resource_type(RESOURCE_TYPE_TOPIC) + .with_resource_name(StrBytes::from_static_str("")) + .with_configs(Vec::new()), + ]; + encode_message(&response(results), version, 128) +} + +fn response(results: Vec) -> DescribeConfigsResponse { + DescribeConfigsResponse::default() + .with_throttle_time_ms(0) + .with_results(results) +} + +async fn describe_all( + bridge: &IggyBridge, + req: &DescribeConfigsRequest, + deadline: Instant, +) -> Vec { + let topic_names = req + .resources + .iter() + .filter(|resource| resource.resource_type == RESOURCE_TYPE_TOPIC) + .map(|resource| resource.resource_name.as_str()); + if exceeds_topic_cap(topic_names) { + let message = StrBytes::from(topic_cap_message("DescribeConfigs")); + tracing::warn!("DescribeConfigs request addresses too many distinct topics; rejecting"); + return req + .resources + .iter() + .map(|resource| { + resource_error( + resource, + ERROR_POLICY_VIOLATION, + Some(message.clone()), + Vec::new(), + ) + }) + .collect(); + } + + // `get_kafka_topics` lists partition counts, not `message_expiry`, so each distinct + // topic is one `get_kafka_topic`. Duplicates reuse the first result and do not call again. + let mut seen: HashMap = HashMap::new(); + let mut results = Vec::with_capacity(req.resources.len()); + let mut deadline_exceeded = false; + for resource in &req.resources { + if resource.resource_type == RESOURCE_TYPE_TOPIC + && let Some(prior) = seen.get(resource.resource_name.as_str()) + { + results.push(prior.clone()); + continue; + } + if deadline_exceeded || Instant::now() >= deadline { + if !deadline_exceeded { + deadline_exceeded = true; + tracing::warn!( + "DescribeConfigs deadline passed; answering remaining resources retriable \ + instead of starting new Iggy calls" + ); + } + let timed_out = resource_error(resource, ERROR_REQUEST_TIMED_OUT, None, Vec::new()); + remember_topic(&mut seen, resource, &timed_out); + results.push(timed_out); + continue; + } + + let result = describe_one( + bridge, + resource, + req.include_synonyms, + req.include_documentation, + deadline, + ) + .await; + if Instant::now() >= deadline { + deadline_exceeded = true; + } + remember_topic(&mut seen, resource, &result); + results.push(result); + } + results +} + +fn remember_topic( + seen: &mut HashMap, + resource: &DescribeConfigsResource, + result: &DescribeConfigsResult, +) { + if resource.resource_type == RESOURCE_TYPE_TOPIC { + seen.insert(resource.resource_name.as_str().to_string(), result.clone()); + } +} + +async fn describe_one( + bridge: &IggyBridge, + resource: &DescribeConfigsResource, + include_synonyms: bool, + include_documentation: bool, + deadline: Instant, +) -> DescribeConfigsResult { + if !resource.unknown_tagged_fields.is_empty() { + return resource_error( + resource, + ERROR_INVALID_REQUEST, + Some(static_text(UNKNOWN_TAGGED_FIELDS)), + Vec::new(), + ); + } + if resource.resource_type != RESOURCE_TYPE_TOPIC { + return resource_error( + resource, + ERROR_INVALID_REQUEST, + Some(static_text(ONLY_TOPIC_RESOURCES)), + Vec::new(), + ); + } + let name = resource.resource_name.as_str(); + if let Err(error) = validate_kafka_topic_name("kafka_topic", name) { + return resource_error( + resource, + ERROR_INVALID_TOPIC_EXCEPTION, + Some(StrBytes::from(name_reason(&error))), + Vec::new(), + ); + } + + if Instant::now() >= deadline { + tracing::warn!( + resource = name, + "DescribeConfigs deadline passed before the topic lookup" + ); + return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None, Vec::new()); + } + let topic = match bridge.get_kafka_topic(name).await { + Ok(Some(topic)) => topic, + Ok(None) => { + return resource_error(resource, ERROR_UNKNOWN_TOPIC_OR_PARTITION, None, Vec::new()); + } + Err(error) => { + let (code, message) = bridge_failure(&error, "reading"); + return resource_error(resource, code, message, Vec::new()); + } + }; + + let (configs, unknown, unrepresentable) = described_entries( + &topic, + resource.configuration_keys.as_deref(), + include_synonyms, + include_documentation, + ); + + if unknown || unrepresentable { + let message = if unrepresentable { + "stored message expiry is not a whole number of milliseconds" + } else { + "unknown config key" + }; + return resource_error( + resource, + ERROR_INVALID_CONFIG, + Some(static_text(message)), + configs, + ); + } + resource_error(resource, ERROR_NONE, None, configs) +} + +#[must_use] +fn described_entries( + topic: &TopicDetails, + configuration_keys: Option<&[StrBytes]>, + include_synonyms: bool, + include_documentation: bool, +) -> (Vec, bool, bool) { + let explicit = message_expiry_is_explicit(&topic.options); + let retention = retention_ms(topic.message_expiry, explicit); + let keys = listed_keys(configuration_keys); + let mut unknown = false; + let mut unrepresentable = false; + let mut configs = Vec::with_capacity(keys.len()); + for key in keys { + match key { + ListedKey::Retention => { + let (value, source) = retention.as_ref().map_or_else( + |()| { + // A stored duration that cannot be shown as retention.ms is still the + // topic's expiry, not the never-expire default. + unrepresentable = true; + (None, CONFIG_SOURCE_TOPIC) + }, + |retention| (Some(retention.value.as_str()), retention.source), + ); + configs.push(config_entry( + RETENTION_MS, + value, + false, + source, + CONFIG_TYPE_LONG, + include_synonyms, + include_documentation.then_some(RETENTION_DOC), + )); + } + ListedKey::Cleanup => configs.push(config_entry( + CLEANUP_POLICY, + Some(CLEANUP_POLICY_VALUE), + true, + CONFIG_SOURCE_DEFAULT, + CONFIG_TYPE_STRING, + include_synonyms, + include_documentation.then_some(CLEANUP_DOC), + )), + ListedKey::Unknown(name) => { + unknown = true; + configs.push(config_entry( + &name, + None, + false, + CONFIG_SOURCE_DEFAULT, + 0, + include_synonyms, + None, + )); + } + } + } + (configs, unknown, unrepresentable) +} + +fn config_entry( + name: &str, + value: Option<&str>, + read_only: bool, + source: i8, + config_type: i8, + include_synonyms: bool, + documentation: Option<&str>, +) -> DescribeConfigsResourceResult { + DescribeConfigsResourceResult::default() + .with_name(StrBytes::from(name.to_string())) + .with_value(value.map(|value| StrBytes::from(value.to_string()))) + .with_read_only(read_only) + .with_config_source(source) + .with_is_sensitive(false) + .with_synonyms(synonyms(include_synonyms)) + .with_config_type(config_type) + .with_documentation(documentation.map(|text| StrBytes::from(text.to_string()))) +} + +/// Neither known key has a synonym. `include_synonyms` asks for the list and receives none. +const fn synonyms(_include_synonyms: bool) -> Vec { + Vec::new() +} + +fn reject_tagged_resources(resources: &[DescribeConfigsResource]) -> Vec { + if resources.is_empty() { + return vec![ + DescribeConfigsResult::default() + .with_error_code(ERROR_INVALID_REQUEST) + .with_error_message(Some(static_text(UNKNOWN_TAGGED_FIELDS))) + .with_resource_type(0) + .with_resource_name(StrBytes::from_static_str("")) + .with_configs(Vec::new()), + ]; + } + resources + .iter() + .map(|resource| { + resource_error( + resource, + ERROR_INVALID_REQUEST, + Some(static_text(UNKNOWN_TAGGED_FIELDS)), + Vec::new(), + ) + }) + .collect() +} + +fn resource_error( + resource: &DescribeConfigsResource, + error_code: i16, + error_message: Option, + configs: Vec, +) -> DescribeConfigsResult { + DescribeConfigsResult::default() + .with_error_code(error_code) + .with_error_message(error_message) + .with_resource_type(resource.resource_type) + .with_resource_name(resource.resource_name.clone()) + .with_configs(configs) +} + +fn name_reason(error: &BridgeError) -> String { + match error { + BridgeError::InvalidKafkaTopicName { reason, .. } => reason.clone(), + _ => "invalid topic name".to_string(), + } +} + +fn bridge_failure(error: &BridgeError, action: &str) -> (i16, Option) { + let code = error.to_kafka_error_code(); + match error { + BridgeError::InvalidKafkaTopicName { reason, .. } => { + tracing::debug!(reason, "DescribeConfigs rejected an invalid topic name"); + (code, Some(StrBytes::from(reason.clone()))) + } + BridgeError::Timeout | BridgeError::SendLost(_) => { + tracing::warn!(%error, "DescribeConfigs {action} exceeded the bridge deadline"); + (code, None) + } + other => { + tracing::error!(%other, "DescribeConfigs failed while {action} topic configuration"); + ( + code, + Some(static_text("internal error reading topic configuration")), + ) + } + } +} + +const fn static_text(message: &'static str) -> StrBytes { + StrBytes::from_static_str(message) +} diff --git a/gateways/kafka/src/protocol/handlers/topic_config.rs b/gateways/kafka/src/protocol/handlers/topic_config.rs new file mode 100644 index 0000000000..8760922820 --- /dev/null +++ b/gateways/kafka/src/protocol/handlers/topic_config.rs @@ -0,0 +1,348 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Topic configuration shared by `DescribeConfigs` and `AlterConfigs`. +//! +//! Iggy stores `message_expiry` as [`IggyExpiry`]. A duration is microseconds +//! (`IggyDuration::from(u64)` / `as_micros`). `u64::MAX` is [`IggyExpiry::NeverExpire`] +//! and `0` is [`IggyExpiry::ServerDefault`], so neither is a usable millisecond expiry. +//! Kafka `retention.ms` is a decimal millisecond count, or `-1` when sealed segments +//! never expire. A positive count is capped at `u32::MAX` seconds, matching +//! [`IggyExpiry::from_str`]. + +use std::collections::HashSet; +use std::str::FromStr; + +use iggy::prelude::{HeaderKey, IggyDuration, IggyExpiry, ResourceOptions, topic_option_keys}; +use kafka_protocol::protocol::StrBytes; + +/// Kafka `ConfigResource.Type.TOPIC`. +pub const RESOURCE_TYPE_TOPIC: i8 = 2; + +/// Kafka `ConfigSource.DYNAMIC_TOPIC_CONFIG`: the value was stored on the topic. +pub const CONFIG_SOURCE_TOPIC: i8 = 1; + +/// Kafka `ConfigSource.DEFAULT_CONFIG`: the value is the gateway default. +pub const CONFIG_SOURCE_DEFAULT: i8 = 5; + +/// Kafka `ConfigType.STRING`. +pub const CONFIG_TYPE_STRING: i8 = 2; + +/// Kafka `ConfigType.LONG`. +pub const CONFIG_TYPE_LONG: i8 = 5; + +pub const RETENTION_MS: &str = "retention.ms"; +pub const CLEANUP_POLICY: &str = "cleanup.policy"; +pub const CLEANUP_POLICY_VALUE: &str = "delete"; + +pub const ONLY_TOPIC_RESOURCES: &str = "only topic resources are supported"; +pub const UNKNOWN_TAGGED_FIELDS: &str = "unknown tagged fields are not supported"; + +pub const RETENTION_DOC: &str = "How long a sealed segment is kept, in milliseconds. The active segment does not expire. -1 means sealed segments never expire."; +pub const CLEANUP_DOC: &str = "Iggy deletes expired messages and does not compact a topic."; + +/// Distinct Kafka topic names one `DescribeConfigs` or `AlterConfigs` request may send to Iggy. +/// +/// Same ceiling as `CreateTopics`: each distinct name is a lockstep call on the shared client. +/// Duplicate names in one request count once. A larger batch is `POLICY_VIOLATION` on every +/// resource. The decode-time element ceiling is a separate, larger limit and is not this cap. +pub const MAX_CONFIG_TOPICS: usize = 100; + +/// One config name a describe request asked for, in request order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ListedKey { + Retention, + Cleanup, + Unknown(String), +} + +/// Why one resource's alter cannot be stored. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ConfigFault { + UnknownKey(String), + Repeated(String), + ReadOnly(&'static str), + InvalidRetention, +} + +impl ConfigFault { + #[must_use] + pub fn message(&self) -> String { + match self { + Self::UnknownKey(name) => format!("unknown config key '{name}'"), + Self::Repeated(name) => format!("config key '{name}' is repeated"), + Self::ReadOnly(message) => (*message).to_string(), + Self::InvalidRetention => format!( + "retention.ms must be -1 or a positive number of milliseconds at most {} seconds", + u32::MAX + ), + } + } +} + +/// `retention.ms` as Kafka should report it, plus the config source byte. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RetentionMs { + pub value: String, + pub source: i8, +} + +/// Whether the topic's `message_expiry` option was sent by a client. +/// +/// Admission stores the never-expire default with `explicit == false`. An alter +/// of `retention.ms` stores the key with `explicit == true`, including `-1`. +#[must_use] +pub fn message_expiry_is_explicit(options: &ResourceOptions) -> bool { + let Ok(key) = HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY) else { + return false; + }; + options.get(&key).is_some_and(|entry| entry.explicit) +} + +/// Map an Iggy expiry onto Kafka `retention.ms`. +/// +/// A derived never-expire value is [`CONFIG_SOURCE_DEFAULT`]. An explicit one, +/// and any stored duration, is [`CONFIG_SOURCE_TOPIC`]. A duration that is not a +/// positive whole number of milliseconds is refused: reporting a rounded count +/// would describe a different value than Iggy stored. +/// +/// # Errors +/// +/// Returns `Err` when `expiry` is a duration of `0` microseconds or a duration +/// that is not a whole number of milliseconds. +pub fn retention_ms(expiry: IggyExpiry, explicit: bool) -> Result { + match expiry { + IggyExpiry::NeverExpire | IggyExpiry::ServerDefault => Ok(RetentionMs { + value: "-1".to_string(), + source: if explicit { + CONFIG_SOURCE_TOPIC + } else { + CONFIG_SOURCE_DEFAULT + }, + }), + IggyExpiry::ExpireDuration(duration) => { + let micros = duration.as_micros(); + if micros == 0 || !micros.is_multiple_of(1_000) { + return Err(()); + } + let millis = i64::try_from(micros / 1_000).map_err(|_| ())?; + if millis <= 0 { + return Err(()); + } + Ok(RetentionMs { + value: millis.to_string(), + source: CONFIG_SOURCE_TOPIC, + }) + } + } +} + +/// Parse a Kafka `retention.ms` value into the expiry `update_topic` stores. +/// +/// `-1` is [`IggyExpiry::NeverExpire`] (`u64::MAX` on the wire), not +/// [`IggyExpiry::ServerDefault`] (`0`), because the server ignores the `0` +/// sentinel and leaves the previous expiry in place. The text must be the +/// canonical decimal of the parsed integer, so a later describe reports the +/// same characters the alter accepted. +/// +/// # Errors +/// +/// Returns [`ConfigFault::InvalidRetention`] for any value other than `-1` or a +/// positive millisecond count of at most `u32::MAX` seconds. +pub fn parse_retention_ms(value: &str) -> Result { + let Ok(millis) = value.parse::() else { + return Err(ConfigFault::InvalidRetention); + }; + if millis.to_string() != value { + return Err(ConfigFault::InvalidRetention); + } + if millis == -1 { + return Ok(IggyExpiry::NeverExpire); + } + if millis <= 0 { + return Err(ConfigFault::InvalidRetention); + } + let millis = u64::try_from(millis).map_err(|_| ConfigFault::InvalidRetention)?; + // `IggyExpiry::from_str` rejects `as_secs() > u32::MAX`. A longer duration makes + // `Segment::is_expired` wrap `max_timestamp + micros` (and panic in debug), so a + // sealed segment looks expired and is deleted. + if millis / 1_000 > u64::from(u32::MAX) { + return Err(ConfigFault::InvalidRetention); + } + let Some(micros) = millis.checked_mul(1_000) else { + return Err(ConfigFault::InvalidRetention); + }; + Ok(IggyExpiry::ExpireDuration(IggyDuration::from(micros))) +} + +/// Whether `names` contains more than [`MAX_CONFIG_TOPICS`] distinct strings. +#[must_use] +pub fn exceeds_topic_cap<'a>(names: impl IntoIterator) -> bool { + names.into_iter().collect::>().len() > MAX_CONFIG_TOPICS +} + +/// Client-facing text for a request over [`MAX_CONFIG_TOPICS`]. +#[must_use] +pub fn topic_cap_message(api_name: &str) -> String { + format!( + "this gateway addresses at most {MAX_CONFIG_TOPICS} distinct topics per {api_name} request" + ) +} + +/// Keys to return for one describe resource. +/// +/// `None` and an empty list both mean both known keys. Any other name is +/// [`ListedKey::Unknown`] and stays in the list. +#[must_use] +pub fn listed_keys(configuration_keys: Option<&[StrBytes]>) -> Vec { + let Some(keys) = configuration_keys.filter(|keys| !keys.is_empty()) else { + return vec![ListedKey::Retention, ListedKey::Cleanup]; + }; + keys.iter() + .map(|key| match key.as_str() { + RETENTION_MS => ListedKey::Retention, + CLEANUP_POLICY => ListedKey::Cleanup, + other => ListedKey::Unknown(other.to_string()), + }) + .collect() +} + +/// The expiry to store, or `None` when the resource names no `retention.ms`. +/// +/// One invalid or repeated key fails the whole resource. Nothing is selected +/// for storage in that case. +/// +/// # Errors +/// +/// Returns the first [`ConfigFault`] in request order. +pub fn plan_retention_update<'a>( + configs: impl IntoIterator)>, +) -> Result, ConfigFault> { + let mut seen = HashSet::new(); + let mut expiry = None; + for (name, value) in configs { + if !seen.insert(name) { + return Err(ConfigFault::Repeated(name.to_string())); + } + match name { + RETENTION_MS => { + let Some(value) = value else { + return Err(ConfigFault::InvalidRetention); + }; + expiry = Some(parse_retention_ms(value)?); + } + CLEANUP_POLICY => { + return Err(ConfigFault::ReadOnly("cleanup.policy is read-only")); + } + other => return Err(ConfigFault::UnknownKey(other.to_string())), + } + } + Ok(expiry) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retention_ms_round_trips_a_positive_value_and_never_expire() { + let expiry = parse_retention_ms("1500").expect("1500 ms"); + let described = retention_ms(expiry, true).expect("representable"); + assert_eq!(described.value, "1500"); + assert_eq!(described.source, CONFIG_SOURCE_TOPIC); + assert_eq!( + u64::from(expiry), + 1_500_000, + "IggyDuration stores microseconds" + ); + + let never = parse_retention_ms("-1").expect("never expire"); + assert_eq!(never, IggyExpiry::NeverExpire); + let described = retention_ms(never, true).expect("never expire"); + assert_eq!(described.value, "-1"); + assert_eq!(described.source, CONFIG_SOURCE_TOPIC); + let unset = retention_ms(IggyExpiry::NeverExpire, false).expect("default"); + assert_eq!(unset.value, "-1"); + assert_eq!(unset.source, CONFIG_SOURCE_DEFAULT); + } + + #[test] + fn retention_ms_rejects_zero_noncanonical_and_unrepresentable_values() { + assert!(parse_retention_ms("0").is_err()); + assert!(parse_retention_ms("-2").is_err()); + assert!(parse_retention_ms("0001").is_err()); + assert!(parse_retention_ms("+1").is_err()); + assert!(parse_retention_ms("1.5").is_err()); + assert!(parse_retention_ms("").is_err()); + assert!(parse_retention_ms(&i64::MAX.to_string()).is_err()); + let max_secs_ms = u64::from(u32::MAX) * 1_000; + let at_cap = parse_retention_ms(&max_secs_ms.to_string()).expect("u32::MAX seconds"); + assert_eq!(u64::from(at_cap), max_secs_ms * 1_000); + let with_remainder = max_secs_ms + 999; + assert!( + parse_retention_ms(&with_remainder.to_string()).is_ok(), + "a remainder under one second stays within IggyExpiry::from_str" + ); + assert!(parse_retention_ms(&(max_secs_ms + 1_000).to_string()).is_err()); + assert!(parse_retention_ms(&(u64::MAX / 1_000).to_string()).is_err()); + + let sub_ms = IggyExpiry::ExpireDuration(IggyDuration::from(1_500_u64)); + assert!(retention_ms(sub_ms, true).is_err()); + } + + #[test] + fn listed_keys_defaults_to_both_and_keeps_an_unknown_name() { + assert_eq!( + listed_keys(None), + vec![ListedKey::Retention, ListedKey::Cleanup] + ); + assert_eq!( + listed_keys(Some(&[])), + vec![ListedKey::Retention, ListedKey::Cleanup] + ); + let asked = [ + StrBytes::from_static_str(CLEANUP_POLICY), + StrBytes::from_static_str("no.such"), + ]; + assert_eq!( + listed_keys(Some(&asked)), + vec![ + ListedKey::Cleanup, + ListedKey::Unknown("no.such".to_string()) + ] + ); + } + + #[test] + fn one_bad_key_rejects_the_whole_alter_plan() { + let err = plan_retention_update([(RETENTION_MS, Some("5000")), ("no.such", Some("1"))]) + .expect_err("unknown key"); + assert_eq!(err, ConfigFault::UnknownKey("no.such".to_string())); + + let err = plan_retention_update([(CLEANUP_POLICY, Some("delete"))]).expect_err("read only"); + assert!(matches!(err, ConfigFault::ReadOnly(_))); + + assert_eq!( + plan_retention_update([(RETENTION_MS, Some("-1"))]).expect("clear"), + Some(IggyExpiry::NeverExpire) + ); + assert_eq!( + plan_retention_update([] as [(&str, Option<&str>); 0]).expect("empty"), + None + ); + } +} diff --git a/gateways/kafka/tests/configs_handler_tests.rs b/gateways/kafka/tests/configs_handler_tests.rs new file mode 100644 index 0000000000..ede44539b5 --- /dev/null +++ b/gateways/kafka/tests/configs_handler_tests.rs @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `DescribeConfigs` and `AlterConfigs` with the bridge off. + +use bytes::{Bytes, BytesMut}; +use kafka_protocol::messages::alter_configs_request::{ + AlterConfigsRequest, AlterConfigsResource, AlterableConfig, +}; +use kafka_protocol::messages::alter_configs_response::AlterConfigsResponse; +use kafka_protocol::messages::describe_configs_request::{ + DescribeConfigsRequest, DescribeConfigsResource, +}; +use kafka_protocol::messages::describe_configs_response::DescribeConfigsResponse; +use kafka_protocol::protocol::{Decodable, Encodable, StrBytes}; +use tokio_util::sync::CancellationToken; + +use iggy_gateway_kafka::group::{GroupCoordinator, GroupCoordinatorConfig}; +use iggy_gateway_kafka::protocol::api::{BrokerAdvertise, ERROR_NOT_CONTROLLER, GatewayState}; +use iggy_gateway_kafka::protocol::handlers::{alter_configs, describe_configs}; + +const DESCRIBE_VERSION: i16 = 4; +const ALTER_VERSION: i16 = 2; + +fn state_without_bridge() -> GatewayState { + GatewayState::new( + BrokerAdvertise::default(), + None, + 8 * 1024 * 1024, + false, + 0, + GroupCoordinator::new(GroupCoordinatorConfig::default(), CancellationToken::new()), + ) +} + +fn encode(message: &M, version: i16) -> Bytes { + let mut buf = BytesMut::new(); + message.encode(&mut buf, version).expect("encode request"); + buf.freeze() +} + +fn decode(body: Bytes, version: i16) -> M { + let mut buf = body; + M::decode(&mut buf, version).expect("decode response") +} + +#[tokio::test] +async fn describe_configs_without_a_bridge_returns_not_controller() { + let request = DescribeConfigsRequest::default() + .with_resources(vec![ + DescribeConfigsResource::default() + .with_resource_type(2) + .with_resource_name(StrBytes::from_static_str("orders")) + .with_configuration_keys(None), + ]) + .with_include_synonyms(false) + .with_include_documentation(false); + let outcome = describe_configs::handle( + &state_without_bridge(), + DESCRIBE_VERSION, + encode(&request, DESCRIBE_VERSION), + ) + .await; + let response: DescribeConfigsResponse = decode( + outcome.expect_response("DescribeConfigs answers"), + DESCRIBE_VERSION, + ); + assert_eq!(response.results.len(), 1); + assert_eq!(response.results[0].error_code, ERROR_NOT_CONTROLLER); + assert!(response.results[0].configs.is_empty()); +} + +#[tokio::test] +async fn alter_configs_without_a_bridge_returns_not_controller() { + let request = AlterConfigsRequest::default() + .with_resources(vec![ + AlterConfigsResource::default() + .with_resource_type(2) + .with_resource_name(StrBytes::from_static_str("orders")) + .with_configs(vec![ + AlterableConfig::default() + .with_name(StrBytes::from_static_str("retention.ms")) + .with_value(Some(StrBytes::from_static_str("1000"))), + ]), + ]) + .with_validate_only(false); + let outcome = alter_configs::handle( + &state_without_bridge(), + ALTER_VERSION, + encode(&request, ALTER_VERSION), + ) + .await; + let response: AlterConfigsResponse = decode( + outcome.expect_response("AlterConfigs answers"), + ALTER_VERSION, + ); + assert_eq!(response.responses.len(), 1); + assert_eq!(response.responses[0].error_code, ERROR_NOT_CONTROLLER); +} diff --git a/gateways/kafka/tests/configs_java_client_tests.rs b/gateways/kafka/tests/configs_java_client_tests.rs new file mode 100644 index 0000000000..4dc5905089 --- /dev/null +++ b/gateways/kafka/tests/configs_java_client_tests.rs @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka 3.9 `AdminClient` describe/alter against the gateway, on the host JDK. +//! +//! The docker `--network host` suite cannot reach a loopback gateway from macOS. +//! This test publishes nothing into a container: it binds `127.0.0.1` and runs +//! `java` on the host. Jars come from Maven Central when they are not cached. + +use std::io::Read; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::sync::broadcast; + +use iggy_gateway_kafka::bridge::IggyBridge; +use iggy_gateway_kafka::server::bind_listener; +use iggy_gateway_kafka::{GatewayConfig, KafkaGateway}; + +#[path = "common/iggy_server.rs"] +mod iggy_server; + +use iggy_server::TestServer; + +const TOPIC: &str = "orders"; + +/// Bounds a child the way `kafka_client_e2e_tests.rs` bounds a client run, without GNU `timeout` +/// (this suite runs on the host JDK, including macOS). A hung `java` or `curl` used to block the +/// process until the runner itself was killed. +const CHILD_LIMIT: Duration = Duration::from_secs(120); + +fn output_bounded(command: &mut Command, what: &str) -> Output { + let mut child = command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap_or_else(|error| panic!("spawn {what}: {error}")); + let mut stdout = child.stdout.take().expect("stdout"); + let mut stderr = child.stderr.take().expect("stderr"); + let stdout_thread = std::thread::spawn(move || { + let mut buf = Vec::new(); + stdout.read_to_end(&mut buf).map(|_read| buf) + }); + let stderr_thread = std::thread::spawn(move || { + let mut buf = Vec::new(); + stderr.read_to_end(&mut buf).map(|_read| buf) + }); + let deadline = Instant::now() + CHILD_LIMIT; + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) if Instant::now() >= deadline => { + let _ = child.kill(); + let _ = child.wait(); + panic!("{what} exceeded {CHILD_LIMIT:?}"); + } + Ok(None) => std::thread::sleep(Duration::from_millis(50)), + Err(error) => panic!("poll {what}: {error}"), + } + }; + let stdout = stdout_thread + .join() + .unwrap_or_else(|_| panic!("{what} stdout thread")) + .unwrap_or_else(|error| panic!("read {what} stdout: {error}")); + let stderr = stderr_thread + .join() + .unwrap_or_else(|_| panic!("{what} stderr thread")) + .unwrap_or_else(|error| panic!("read {what} stderr: {error}")); + Output { + status, + stdout, + stderr, + } +} + +const JARS: &[(&str, &str)] = &[ + ( + "kafka-clients-3.9.0.jar", + "https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/3.9.0/kafka-clients-3.9.0.jar", + ), + ( + "slf4j-api-1.7.36.jar", + "https://repo1.maven.org/maven2/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar", + ), + ( + "slf4j-simple-1.7.36.jar", + "https://repo1.maven.org/maven2/org/slf4j/slf4j-simple/1.7.36/slf4j-simple-1.7.36.jar", + ), + ( + "lz4-java-1.8.0.jar", + "https://repo1.maven.org/maven2/org/lz4/lz4-java/1.8.0/lz4-java-1.8.0.jar", + ), + ( + "snappy-java-1.1.10.5.jar", + "https://repo1.maven.org/maven2/org/xerial/snappy/snappy-java/1.1.10.5/snappy-java-1.1.10.5.jar", + ), + ( + "zstd-jni-1.5.6-4.jar", + "https://repo1.maven.org/maven2/com/github/luben/zstd-jni/1.5.6-4/zstd-jni-1.5.6-4.jar", + ), +]; + +fn jar_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/kafka-3.9-admin-jars") +} + +fn ensure_jars(dir: &Path) { + std::fs::create_dir_all(dir).expect("create jar cache"); + for (name, url) in JARS { + let path = dir.join(name); + if path.exists() { + continue; + } + let output = output_bounded( + Command::new("curl") + .args([ + "--fail", + "--silent", + "--show-error", + "--location", + "--max-time", + "60", + "-o", + ]) + .arg(&path) + .arg(url), + name, + ); + if !output.status.success() { + let _ = std::fs::remove_file(&path); + } + assert!( + output.status.success(), + "download {url} failed: {}{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + } +} + +fn classpath(jars: &Path, classes: &Path) -> String { + let mut parts = vec![classes.display().to_string()]; + for (name, _) in JARS { + parts.push(jars.join(name).display().to_string()); + } + parts.join(":") +} + +async fn spawn_gateway(server: &TestServer) -> SocketAddr { + let bridge = IggyBridge::connect(server.test_config()) + .await + .expect("bridge connects"); + bridge + .ensure_stream_and_topic(TOPIC, 1) + .await + .expect("create topic"); + let listener = bind_listener("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("local addr"); + let gateway = KafkaGateway::new(GatewayConfig { + bind_addr: addr.to_string(), + ..GatewayConfig::default() + }) + .with_bridge(Some(Arc::new(bridge))); + let (shutdown, receiver) = broadcast::channel::<()>(1); + tokio::spawn(async move { + let _ = gateway.run(listener, receiver).await; + }); + std::mem::forget(shutdown); + addr +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn kafka_39_admin_client_describes_and_alters_topic_configs() { + let java = output_bounded(Command::new("java").arg("-version"), "java -version"); + assert!( + java.status.success(), + "java -version failed: {}", + String::from_utf8_lossy(&java.stderr) + ); + + let jars = jar_dir(); + ensure_jars(&jars); + let classes = tempfile::tempdir().expect("class dir"); + let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/java/ConfigsProbe.java"); + let javac = output_bounded( + Command::new("javac") + .arg("--release") + .arg("17") + .arg("-cp") + .arg(classpath(&jars, classes.path())) + .arg("-d") + .arg(classes.path()) + .arg(&source), + "javac", + ); + assert!( + javac.status.success(), + "javac failed\n{}", + String::from_utf8_lossy(&javac.stderr) + ); + + let data_dir = tempfile::tempdir().expect("tempdir"); + let server = TestServer::spawn(data_dir.path()).await; + let addr = spawn_gateway(&server).await; + + let output = output_bounded( + Command::new("java") + .arg("-cp") + .arg(classpath(&jars, classes.path())) + .arg("ConfigsProbe") + .arg(addr.to_string()) + .arg(TOPIC), + "java ConfigsProbe", + ); + let text = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + println!("{text}"); + assert!( + output.status.success(), + "ConfigsProbe failed ({:?})\n{text}", + output.status.code() + ); + assert!(text.contains("CONFIGS_PROBE_OK"), "{text}"); + assert!( + text.contains("describe_batch_50_elapsed_ms="), + "missing timing line\n{text}" + ); + assert!( + text.contains("retention.ms=-1 source=DEFAULT_CONFIG"), + "{text}" + ); + assert!( + text.contains("cleanup.policy=delete read_only=true source=DEFAULT_CONFIG"), + "{text}" + ); + assert!( + text.contains("unknown_key_rejected=InvalidConfigurationException"), + "{text}" + ); + assert!(text.contains("validate_only_retention.ms=-1"), "{text}"); + assert!( + text.contains("altered_retention.ms=8000 source=DYNAMIC_TOPIC_CONFIG"), + "{text}" + ); +} diff --git a/gateways/kafka/tests/configs_real_bridge_tests.rs b/gateways/kafka/tests/configs_real_bridge_tests.rs new file mode 100644 index 0000000000..933a85a9f3 --- /dev/null +++ b/gateways/kafka/tests/configs_real_bridge_tests.rs @@ -0,0 +1,521 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `DescribeConfigs` and `AlterConfigs` against a real `iggy-server`. + +use std::sync::Arc; + +use bytes::{Bytes, BytesMut}; +use kafka_protocol::messages::alter_configs_request::{ + AlterConfigsRequest, AlterConfigsResource, AlterableConfig, +}; +use kafka_protocol::messages::alter_configs_response::AlterConfigsResponse; +use kafka_protocol::messages::describe_configs_request::{ + DescribeConfigsRequest, DescribeConfigsResource, +}; +use kafka_protocol::messages::describe_configs_response::{ + DescribeConfigsResourceResult, DescribeConfigsResponse, +}; +use kafka_protocol::protocol::{Decodable, Encodable, StrBytes}; +use serial_test::serial; +use tokio_util::sync::CancellationToken; + +use iggy_gateway_kafka::bridge::IggyBridge; +use iggy_gateway_kafka::group::{GroupCoordinator, GroupCoordinatorConfig}; +use iggy_gateway_kafka::protocol::api::{ + BrokerAdvertise, ERROR_INVALID_CONFIG, ERROR_INVALID_REQUEST, ERROR_INVALID_TOPIC_EXCEPTION, + ERROR_NONE, ERROR_UNKNOWN_TOPIC_OR_PARTITION, GatewayState, +}; +use iggy_gateway_kafka::protocol::handlers::{alter_configs, describe_configs}; + +#[path = "common/iggy_server.rs"] +mod iggy_server; + +use iggy_server::TestServer; + +const DESCRIBE_V4: i16 = 4; +const DESCRIBE_V1: i16 = 1; +const ALTER_V2: i16 = 2; +const ALTER_V0: i16 = 0; +const TOPIC: i8 = 2; +const SOURCE_TOPIC: i8 = 1; +const SOURCE_DEFAULT: i8 = 5; + +const fn text(value: &'static str) -> StrBytes { + StrBytes::from_static_str(value) +} + +fn encode(message: &M, version: i16) -> Bytes { + let mut buf = BytesMut::new(); + message.encode(&mut buf, version).expect("encode request"); + buf.freeze() +} + +fn decode(body: Bytes, version: i16) -> M { + let mut buf = body; + M::decode(&mut buf, version).expect("decode response") +} + +async fn connected_state(server: &TestServer) -> GatewayState { + let bridge = IggyBridge::connect(server.test_config()) + .await + .expect("bridge connects"); + GatewayState::new( + BrokerAdvertise::default(), + Some(Arc::new(bridge)), + 8 * 1024 * 1024, + false, + 0, + GroupCoordinator::new(GroupCoordinatorConfig::default(), CancellationToken::new()), + ) +} + +async fn create_topic(state: &GatewayState, name: &str) { + let bridge = state.bridge.as_ref().expect("bridge"); + bridge + .ensure_stream_and_topic(name, 1) + .await + .expect("create topic"); +} + +fn describe_resource(name: &str, keys: Option>) -> DescribeConfigsResource { + DescribeConfigsResource::default() + .with_resource_type(TOPIC) + .with_resource_name(StrBytes::from(name.to_string())) + .with_configuration_keys(keys.map(|keys| keys.into_iter().map(text).collect())) +} + +async fn describe( + state: &GatewayState, + version: i16, + resources: Vec, + include_synonyms: bool, + include_documentation: bool, +) -> DescribeConfigsResponse { + let request = DescribeConfigsRequest::default() + .with_resources(resources) + .with_include_synonyms(include_synonyms) + .with_include_documentation(include_documentation); + let outcome = describe_configs::handle(state, version, encode(&request, version)).await; + decode(outcome.expect_response("DescribeConfigs answers"), version) +} + +fn alter_resource( + name: &str, + resource_type: i8, + configs: &[(&'static str, &'static str)], +) -> AlterConfigsResource { + AlterConfigsResource::default() + .with_resource_type(resource_type) + .with_resource_name(StrBytes::from(name.to_string())) + .with_configs( + configs + .iter() + .map(|(key, value)| { + AlterableConfig::default() + .with_name(text(key)) + .with_value(Some(text(value))) + }) + .collect(), + ) +} + +async fn alter( + state: &GatewayState, + version: i16, + resources: Vec, + validate_only: bool, +) -> AlterConfigsResponse { + let request = AlterConfigsRequest::default() + .with_resources(resources) + .with_validate_only(validate_only); + let outcome = alter_configs::handle(state, version, encode(&request, version)).await; + decode(outcome.expect_response("AlterConfigs answers"), version) +} + +fn entry<'a>( + configs: &'a [DescribeConfigsResourceResult], + name: &str, +) -> &'a DescribeConfigsResourceResult { + configs + .iter() + .find(|config| config.name.as_str() == name) + .unwrap_or_else(|| panic!("missing config {name}")) +} + +#[tokio::test] +#[serial] +async fn describe_configs_returns_retention_and_cleanup_and_alter_persists_retention() { + let data_dir = tempfile::tempdir().expect("tempdir"); + let server = TestServer::spawn(data_dir.path()).await; + let state = connected_state(&server).await; + create_topic(&state, "orders").await; + + let described = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("orders", None)], + true, + true, + ) + .await; + let result = &described.results[0]; + assert_eq!(result.error_code, ERROR_NONE); + assert_eq!(result.configs.len(), 2); + let retention = entry(&result.configs, "retention.ms"); + assert_eq!(retention.value.as_ref().map(StrBytes::as_str), Some("-1")); + assert!(!retention.read_only); + assert!(!retention.is_sensitive); + assert_eq!(retention.config_source, SOURCE_DEFAULT); + assert!(retention.synonyms.is_empty()); + assert!(retention.documentation.is_some()); + let cleanup = entry(&result.configs, "cleanup.policy"); + assert_eq!(cleanup.value.as_ref().map(StrBytes::as_str), Some("delete")); + assert!(cleanup.read_only); + assert_eq!(cleanup.config_source, SOURCE_DEFAULT); + assert!(cleanup.synonyms.is_empty()); + assert!(cleanup.documentation.is_some()); + + let without_docs = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("orders", None)], + false, + false, + ) + .await; + let bare = &without_docs.results[0].configs; + assert!(entry(bare, "retention.ms").documentation.is_none()); + assert!(entry(bare, "retention.ms").synonyms.is_empty()); + + let filtered = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("orders", Some(vec!["cleanup.policy"]))], + false, + true, + ) + .await; + assert_eq!(filtered.results[0].error_code, ERROR_NONE); + assert_eq!(filtered.results[0].configs.len(), 1); + assert_eq!( + filtered.results[0].configs[0].name.as_str(), + "cleanup.policy" + ); + + let v1 = describe( + &state, + DESCRIBE_V1, + vec![describe_resource("orders", None)], + false, + false, + ) + .await; + assert_eq!(v1.results[0].error_code, ERROR_NONE); + assert_eq!(v1.results[0].configs.len(), 2); + + let validated = alter( + &state, + ALTER_V2, + vec![alter_resource("orders", TOPIC, &[("retention.ms", "8000")])], + true, + ) + .await; + assert_eq!(validated.responses[0].error_code, ERROR_NONE); + let still_default = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("orders", Some(vec!["retention.ms"]))], + false, + false, + ) + .await; + let unchanged = &still_default.results[0].configs[0]; + assert_eq!(unchanged.value.as_ref().map(StrBytes::as_str), Some("-1")); + assert_eq!(unchanged.config_source, SOURCE_DEFAULT); + + let written = alter( + &state, + ALTER_V2, + vec![alter_resource("orders", TOPIC, &[("retention.ms", "8000")])], + false, + ) + .await; + assert_eq!(written.responses[0].error_code, ERROR_NONE); + let after = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("orders", Some(vec!["retention.ms"]))], + false, + false, + ) + .await; + let changed = &after.results[0].configs[0]; + assert_eq!(changed.value.as_ref().map(StrBytes::as_str), Some("8000")); + assert_eq!(changed.config_source, SOURCE_TOPIC); + + let cleared = alter( + &state, + ALTER_V0, + vec![alter_resource("orders", TOPIC, &[("retention.ms", "-1")])], + false, + ) + .await; + assert_eq!(cleared.responses[0].error_code, ERROR_NONE); + let never = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("orders", Some(vec!["retention.ms"]))], + false, + false, + ) + .await; + let cleared_entry = &never.results[0].configs[0]; + assert_eq!( + cleared_entry.value.as_ref().map(StrBytes::as_str), + Some("-1") + ); + assert_eq!(cleared_entry.config_source, SOURCE_TOPIC); +} + +#[tokio::test] +#[serial] +async fn describe_and_alter_configs_reject_unknown_nontopic_missing_and_invalid_names() { + let data_dir = tempfile::tempdir().expect("tempdir"); + let server = TestServer::spawn(data_dir.path()).await; + let state = connected_state(&server).await; + create_topic(&state, "orders").await; + + let unknown = describe( + &state, + DESCRIBE_V4, + vec![describe_resource( + "orders", + Some(vec!["retention.ms", "no.such"]), + )], + false, + false, + ) + .await; + assert_eq!(unknown.results[0].error_code, ERROR_INVALID_CONFIG); + assert_eq!(unknown.results[0].configs.len(), 2); + assert_eq!(unknown.results[0].configs[1].name.as_str(), "no.such"); + let message = unknown.results[0] + .error_message + .as_ref() + .map_or("", StrBytes::as_str); + assert!(!message.contains("no.such")); + + let broker = describe( + &state, + DESCRIBE_V4, + vec![ + DescribeConfigsResource::default() + .with_resource_type(4) + .with_resource_name(StrBytes::from_static_str("1")) + .with_configuration_keys(None), + describe_resource("orders", Some(vec!["cleanup.policy"])), + ], + false, + false, + ) + .await; + assert_eq!(broker.results[0].error_code, ERROR_INVALID_REQUEST); + assert!(broker.results[0].configs.is_empty()); + assert_eq!( + broker.results[0] + .error_message + .as_ref() + .map(StrBytes::as_str), + Some("only topic resources are supported") + ); + assert_eq!(broker.results[1].error_code, ERROR_NONE); + + let missing = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("missing-topic", None)], + false, + false, + ) + .await; + assert_eq!( + missing.results[0].error_code, + ERROR_UNKNOWN_TOPIC_OR_PARTITION + ); + assert!(missing.results[0].configs.is_empty()); + + let invalid = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("has a space", None)], + false, + false, + ) + .await; + assert_eq!(invalid.results[0].error_code, ERROR_INVALID_TOPIC_EXCEPTION); + let reason = invalid.results[0] + .error_message + .as_ref() + .map_or("", StrBytes::as_str); + assert!(!reason.contains("has a space")); + + let rejected = alter( + &state, + ALTER_V2, + vec![alter_resource("orders", TOPIC, &[("no.such", "1")])], + false, + ) + .await; + assert_eq!(rejected.responses[0].error_code, ERROR_INVALID_CONFIG); + let still = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("orders", Some(vec!["retention.ms"]))], + false, + false, + ) + .await; + assert_eq!( + still.results[0].configs[0] + .value + .as_ref() + .map(StrBytes::as_str), + Some("-1") + ); + assert_eq!(still.results[0].configs[0].config_source, SOURCE_DEFAULT); + + let readonly = alter( + &state, + ALTER_V2, + vec![alter_resource( + "orders", + TOPIC, + &[("cleanup.policy", "compact")], + )], + false, + ) + .await; + assert_eq!(readonly.responses[0].error_code, ERROR_INVALID_CONFIG); + + let missing_alter = alter( + &state, + ALTER_V2, + vec![alter_resource( + "missing-topic", + TOPIC, + &[("retention.ms", "1000")], + )], + false, + ) + .await; + assert_eq!( + missing_alter.responses[0].error_code, + ERROR_UNKNOWN_TOPIC_OR_PARTITION + ); + + let bad_name = alter( + &state, + ALTER_V2, + vec![alter_resource( + "has a space", + TOPIC, + &[("retention.ms", "1000")], + )], + false, + ) + .await; + assert_eq!( + bad_name.responses[0].error_code, + ERROR_INVALID_TOPIC_EXCEPTION + ); + let alter_reason = bad_name.responses[0] + .error_message + .as_ref() + .map_or("", StrBytes::as_str); + assert!(!alter_reason.contains("has a space")); + + let nontopic = alter( + &state, + ALTER_V2, + vec![alter_resource("1", 4, &[("retention.ms", "1000")])], + false, + ) + .await; + assert_eq!(nontopic.responses[0].error_code, ERROR_INVALID_REQUEST); + assert_eq!( + nontopic.responses[0] + .error_message + .as_ref() + .map(StrBytes::as_str), + Some("only topic resources are supported") + ); +} + +#[tokio::test] +#[serial] +async fn alter_configs_one_bad_key_skips_only_that_resource() { + let data_dir = tempfile::tempdir().expect("tempdir"); + let server = TestServer::spawn(data_dir.path()).await; + let state = connected_state(&server).await; + create_topic(&state, "kept").await; + create_topic(&state, "changed").await; + + let altered = alter( + &state, + ALTER_V2, + vec![ + alter_resource("kept", TOPIC, &[("retention.ms", "4000"), ("no.such", "1")]), + alter_resource("changed", TOPIC, &[("retention.ms", "5000")]), + ], + false, + ) + .await; + assert_eq!(altered.responses[0].error_code, ERROR_INVALID_CONFIG); + assert_eq!(altered.responses[1].error_code, ERROR_NONE); + + let described = describe( + &state, + DESCRIBE_V4, + vec![ + describe_resource("kept", Some(vec!["retention.ms"])), + describe_resource("changed", Some(vec!["retention.ms"])), + ], + false, + false, + ) + .await; + assert_eq!( + described.results[0].configs[0] + .value + .as_ref() + .map(StrBytes::as_str), + Some("-1") + ); + assert_eq!( + described.results[0].configs[0].config_source, + SOURCE_DEFAULT + ); + assert_eq!( + described.results[1].configs[0] + .value + .as_ref() + .map(StrBytes::as_str), + Some("5000") + ); + assert_eq!(described.results[1].configs[0].config_source, SOURCE_TOPIC); +} diff --git a/gateways/kafka/tests/java/ConfigsProbe.java b/gateways/kafka/tests/java/ConfigsProbe.java new file mode 100644 index 0000000000..6c757f4044 --- /dev/null +++ b/gateways/kafka/tests/java/ConfigsProbe.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.AlterConfigsOptions; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.errors.InvalidConfigurationException; + +/** + * Kafka 3.9 AdminClient probe for DescribeConfigs and the deprecated AlterConfigs API. + * Prints elapsed time for 50 describes and does not fail on a latency threshold. + */ +public final class ConfigsProbe { + private ConfigsProbe() {} + + @SuppressWarnings("deprecation") + public static void main(String[] args) throws Exception { + if (args.length != 2) { + throw new IllegalArgumentException("usage: ConfigsProbe "); + } + String bootstrap = args[0]; + String topic = args[1]; + Properties props = new Properties(); + props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap); + props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "20000"); + props.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, "20000"); + props.put(AdminClientConfig.CLIENT_ID_CONFIG, "iggy-configs-probe"); + + ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, topic); + try (AdminClient admin = AdminClient.create(props)) { + Config before = describe(admin, resource); + ConfigEntry retention = before.get("retention.ms"); + ConfigEntry cleanup = before.get("cleanup.policy"); + require(retention != null && "-1".equals(retention.value()), "initial retention.ms"); + require( + retention.source() == ConfigEntry.ConfigSource.DEFAULT_CONFIG, + "initial retention source"); + require(!retention.isReadOnly(), "retention.ms is writable"); + require(cleanup != null && "delete".equals(cleanup.value()), "cleanup.policy"); + require(cleanup.isReadOnly(), "cleanup.policy is read only"); + require( + cleanup.source() == ConfigEntry.ConfigSource.DEFAULT_CONFIG, + "cleanup source"); + System.out.println( + "retention.ms=" + retention.value() + " source=" + retention.source()); + System.out.println( + "cleanup.policy=" + + cleanup.value() + + " read_only=" + + cleanup.isReadOnly() + + " source=" + + cleanup.source()); + + try { + admin.alterConfigs(Map.of(resource, config("no.such", "1"))).all().get(); + throw new AssertionError("unknown key was accepted"); + } catch (ExecutionException exception) { + if (!(exception.getCause() instanceof InvalidConfigurationException)) { + throw exception; + } + System.out.println("unknown_key_rejected=" + exception.getCause().getClass().getSimpleName()); + } + + Config stillDefault = describe(admin, resource); + require("-1".equals(stillDefault.get("retention.ms").value()), "unknown key wrote retention"); + + admin.alterConfigs(Map.of(resource, config("retention.ms", "8000")), new AlterConfigsOptions().validateOnly(true)) + .all() + .get(); + Config validated = describe(admin, resource); + require("-1".equals(validated.get("retention.ms").value()), "validate_only wrote retention"); + System.out.println("validate_only_retention.ms=" + validated.get("retention.ms").value()); + + admin.alterConfigs(Map.of(resource, config("retention.ms", "8000"))).all().get(); + Config altered = describe(admin, resource); + ConfigEntry changed = altered.get("retention.ms"); + require("8000".equals(changed.value()), "altered retention.ms"); + require( + changed.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG, + "altered retention source"); + System.out.println("altered_retention.ms=" + changed.value() + " source=" + changed.source()); + + long started = System.nanoTime(); + for (int i = 0; i < 50; i++) { + describe(admin, resource); + } + long elapsedMs = (System.nanoTime() - started) / 1_000_000L; + System.out.println("describe_batch_50_elapsed_ms=" + elapsedMs); + System.out.println("CONFIGS_PROBE_OK"); + } + } + + private static Config describe(AdminClient admin, ConfigResource resource) throws Exception { + return admin.describeConfigs(List.of(resource)).all().get().get(resource); + } + + private static Config config(String name, String value) { + return new Config(List.of(new ConfigEntry(name, value))); + } + + private static void require(boolean condition, String message) { + if (!condition) { + throw new AssertionError(message); + } + } +} From 5b7ad2f51fc54d80fe2bb3967744b94ae16f74ff Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 3 Oct 2026 23:51:00 -0400 Subject: [PATCH 02/13] feat(gateways): add Kafka DescribeConfigs and AlterConfigs Kafka admin clients need to read and set topic retention. DescribeConfigs reports retention.ms from Iggy message expiry and a read-only cleanup.policy.AlterConfigs persists only retention.ms, capped so segment expiry cannot wrap. --- gateways/kafka/README.md | 28 +++-- gateways/kafka/docs/MANUAL_TESTING.md | 8 +- gateways/kafka/docs/SCOPE.md | 16 ++- .../kafka/src/bridge/iggy_bridge/topics.rs | 34 +++++- gateways/kafka/src/bridge/mod.rs | 2 +- gateways/kafka/src/protocol/api.rs | 29 +++-- gateways/kafka/src/protocol/bounds_guard.rs | 108 +++++++++++++++++- .../src/protocol/handlers/create_topics.rs | 3 +- gateways/kafka/src/protocol/handlers/mod.rs | 16 ++- gateways/kafka/tests/common/scope.rs | 2 + gateways/kafka/tests/common/wire.rs | 35 ++++++ .../kafka/tests/golden_wire_fixtures_tests.rs | 16 ++- .../kafka/tests/version_firewall_tests.rs | 76 ++++++++++-- gateways/kafka/tools/kafka-tool/README.md | 4 +- gateways/kafka/tools/kafka-tool/src/main.rs | 10 ++ .../kafka/tools/kafka-tool/src/response.rs | 38 +++++- 16 files changed, 364 insertions(+), 61 deletions(-) diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index e8841b8d68..77b97c5c6d 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -1,11 +1,12 @@ # Kafka gateway (`iggy-gateway-kafka`) -Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421): a TCP listener on the Kafka wire port that decodes requests and validates scoped API keys and versions. With a bridge, Produce, Fetch, ListOffsets, Metadata and CreateTopics use Iggy. InitProducerId and consumer group coordination work with or without one. +Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421): a TCP listener on the Kafka wire port that decodes requests and validates scoped API keys and versions. With a bridge, Produce, Fetch, ListOffsets, Metadata, CreateTopics, DescribeConfigs and AlterConfigs use Iggy. InitProducerId and consumer group coordination work with or without one. -> **Stub warning:** with `IGGY_KAFKA_BRIDGE_ENABLED=true`, Produce, Fetch, ListOffsets, Metadata and -> CreateTopics use Iggy. With the bridge off (the default), those five are stubs: Produce, Fetch and -> ListOffsets answer retriable `NOT_LEADER_OR_FOLLOWER` (6), CreateTopics answers `NOT_CONTROLLER` -> (41), and Metadata reports every requested topic unknown. **CreateTopics runs as the bridge's own +> **Stub warning:** with `IGGY_KAFKA_BRIDGE_ENABLED=true`, Produce, Fetch, ListOffsets, Metadata, +> CreateTopics, DescribeConfigs and AlterConfigs use Iggy. With the bridge off (the default), those +> seven are stubs: Produce, Fetch and ListOffsets answer retriable `NOT_LEADER_OR_FOLLOWER` (6), +> CreateTopics, DescribeConfigs and AlterConfigs answer `NOT_CONTROLLER` (41), and Metadata reports +> every requested topic unknown. **CreateTopics runs as the bridge's own > Iggy user**: with the bridge on and `IGGY_KAFKA_SASL_ENABLED` off (the default), any client that > can reach this port can create topics (up to 1000 partitions each). Every client reads and writes > as the bridge's Iggy user. See [docs/SCOPE.md](docs/SCOPE.md). @@ -185,6 +186,7 @@ See [docs/SCOPE.md](docs/SCOPE.md) for [#3421](https://github.com/apache/iggy/is - [docs/CONSUMER_GROUPS.md](docs/CONSUMER_GROUPS.md) — group membership, rebalances, and why one gateway per bootstrap endpoint - [docs/AUTHENTICATION.md](docs/AUTHENTICATION.md) — how a Kafka client authenticates, and why PLAIN only - [docs/ACL_MAPPING.md](docs/ACL_MAPPING.md) — how Iggy permissions are described as Kafka ACLs +- [docs/CONFIGS.md](docs/CONFIGS.md) — DescribeConfigs and AlterConfigs for topic `retention.ms` ### Delivery guarantees @@ -277,8 +279,9 @@ Full reasoning, including what was rejected and why, is in streams/topics, provisions them on demand, looks up high watermarks (one or many partitions of a topic per call) for `ListOffsets`, and probes and polls partitions for Fetch. Produce ([#3535](https://github.com/apache/iggy/issues/3535)), Fetch ([#3536](https://github.com/apache/iggy/issues/3536)), ListOffsets -([#3537](https://github.com/apache/iggy/issues/3537)), Metadata ([#3534](https://github.com/apache/iggy/issues/3534)) and CreateTopics -([#3538](https://github.com/apache/iggy/issues/3538)) call it. +([#3537](https://github.com/apache/iggy/issues/3537)), Metadata ([#3534](https://github.com/apache/iggy/issues/3534)), CreateTopics +([#3538](https://github.com/apache/iggy/issues/3538)), DescribeConfigs and AlterConfigs call it. +Topic configuration behavior is in [docs/CONFIGS.md](docs/CONFIGS.md). Tested by `bridge`'s own unit tests, `tests/bridge_iggy_integration_tests.rs` and the `tests/*_real_bridge_tests.rs` suites. All but the unit tests start a real `iggy-server`. @@ -457,9 +460,9 @@ call created it." A *different* `partition_count` against an already-existing to two concurrent callers requesting different counts for the same topic must not both see success. Topics created this way have **no message expiry** - Iggy's own server default, not Kafka's 7-day -default. Nothing is bounding retention until it's configured explicitly (Iggy's own topic options, -outside this bridge today); repointing a Kafka app that assumes bounded retention onto this bridge -will accumulate data indefinitely unless you set that up yourself. +default. `AlterConfigs` can set that expiry afterwards through `retention.ms` +([docs/CONFIGS.md](docs/CONFIGS.md)). Until a client does, messages never expire, so a Kafka app +that assumes the 7-day default will accumulate data on this bridge. They also use Iggy's default **durability**, `Durability::Replicated` - quorum commit without an additional stable-storage barrier, with the disk write itself threshold-gated (flushed at 1024 @@ -469,8 +472,9 @@ cut before that threshold is reached - worth knowing rather than discovering lat ### Concurrency ceiling -- One `IggyClient` serves Produce, Metadata and CreateTopics for every Kafka connection, one Iggy - request at a time. Fetch polls use 4 more, one per read slot. Topic probes use 1 more. +- One `IggyClient` serves Produce, Metadata, CreateTopics, DescribeConfigs and AlterConfigs for + every Kafka connection, one Iggy request at a time. Fetch polls use 4 more, one per read slot. + Topic probes use 1 more. - `IGGY_KAFKA_MAX_CONNECTIONS` does not change that. A Produce client pool is a TODO in [docs/SCOPE.md](docs/SCOPE.md). - Order: set `max.in.flight.requests.per.connection=1`, or `retries=0`. Otherwise a retried batch diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md index 3df52b8321..e1b4aa356e 100644 --- a/gateways/kafka/docs/MANUAL_TESTING.md +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -222,13 +222,15 @@ For each API key, test **min−1**, **min**, **max**, **max+1** using `kafka-mes | 13 | LeaveGroup | 0 | 5 | −1, 0, 5, 6 | | 14 | SyncGroup | 0 | 5 | −1, 0, 5, 6 | | 22 | InitProducerId | 0 | 5 | −1, 0, 5, 6 | +| 32 | DescribeConfigs | 1 | 4 | 0, 1, 4, 5 | +| 33 | AlterConfigs | 0 | 2 | −1, 0, 2, 3 | | ID | Test | Expected for in-range | Expected for out-of-range | | ---- | ------ | ---------------------- | --------------------------- | -| B1 | ApiVersions negotiation | `error_code=0`; body lists 12 API keys with correct min/max | KIP-511 exception: still answers, `error_code=35` (UNSUPPORTED_VERSION), v0 response header regardless of the request's own encoding | +| B1 | ApiVersions negotiation | `error_code=0`; body lists 14 API keys with correct min/max | KIP-511 exception: still answers, `error_code=35` (UNSUPPORTED_VERSION), v0 response header regardless of the request's own encoding | | B2 | Metadata out-of-range | N/A | **Connection closes**, no response sent - Metadata has no top-level error field to carry a version-correct error in | | B3 | Produce/Fetch/ListOffsets/CreateTopics/InitProducerId out-of-range | N/A | **Connection closes** for both above-max and below-min - `kafka_protocol`'s schema floor for each of these five messages equals `SUPPORTED_RANGES`' own min, so there is no encodable error response below min either (see `SCOPE.md`'s Governance model) | -| B4 | ApiVersions lists only scoped keys | Decode response | Contains keys 0,1,2,3,10,11,12,13,14,18,19,22 only — no OffsetCommit/OffsetFetch, and no transaction keys (24, 25, 26, 28) | +| B4 | ApiVersions lists only scoped keys | Decode response | Contains keys 0,1,2,3,10,11,12,13,14,18,19,22,32,33 only — no OffsetCommit/OffsetFetch, and no transaction keys (24, 25, 26, 28) | An out-of-range version only ever produces `error_code=35` on ApiVersions (B1); every other API key's out-of-range case closes the connection - see B2/B3. InitProducerId and Produce also send @@ -437,7 +439,7 @@ Gateway commit: ___________ kcat version (if used): ___________ [ ] A1–A10 Smoke tests -[ ] B1–B4 Version firewall (all 7 keys × 4 boundary versions) +[ ] B1–B4 Version firewall (all 14 keys × 4 boundary versions) [ ] C1–C4 Unsupported API keys [ ] D1–D10 Flexible vs legacy encoding [ ] E1–E4 Metadata stub semantics diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index b2c7360400..16c96352be 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -2,9 +2,9 @@ ## Issue #3421 — in scope (this iteration) -A TCP listener on the Kafka wire port. It decodes requests, validates scoped API keys, versions and wire formats, and answers them. With a bridge, Produce, Fetch, ListOffsets, Metadata and CreateTopics use Iggy. +A TCP listener on the Kafka wire port. It decodes requests, validates scoped API keys, versions and wire formats, and answers them. With a bridge, Produce, Fetch, ListOffsets, Metadata, CreateTopics, DescribeConfigs and AlterConfigs use Iggy. -**Stub semantics (important):** without a bridge, every API answers with a stub. Produce discards the payload and answers with retriable `NOT_LEADER_OR_FOLLOWER` (6). Fetch and ListOffsets answer 6 too. CreateTopics validates the request but answers `NOT_CONTROLLER` (41), so clients do not believe topics were created. Do not read `ec=0` from a stub as durable storage. Produce is the one API that stores records once you configure a bridge ([#3535](https://github.com/apache/iggy/issues/3535)). +**Stub semantics (important):** without a bridge, every API answers with a stub. Produce discards the payload and answers with retriable `NOT_LEADER_OR_FOLLOWER` (6). Fetch and ListOffsets answer 6 too. CreateTopics, DescribeConfigs and AlterConfigs answer `NOT_CONTROLLER` (41), so clients do not believe topics were created or configs were read or changed. Do not read `ec=0` from a stub as durable storage. Produce is the one API that stores records once you configure a bridge ([#3535](https://github.com/apache/iggy/issues/3535)). | Deliverable | Status | Location | | ------------- | -------- | ---------- | @@ -29,7 +29,7 @@ Expand `SUPPORTED_RANGES` only after a key/version pair is manually tested. ApiV **Every unsupported-version case closes the connection, for every listed key** - not just above the encoder max. `kafka_protocol`'s schema floor for each supported message happens to equal `SUPPORTED_RANGES`' own min today (Produce 3, Fetch 4, ListOffsets 1, Metadata 0, ApiVersions 0, -CreateTopics 2, InitProducerId 0, and 0 for the four consumer-group keys), so there is no version below an API's +CreateTopics 2, InitProducerId 0, DescribeConfigs 1, AlterConfigs 0, and 0 for the four consumer-group keys), so there is no version below an API's min that the crate can actually encode a response for either - `unsupported_version_response` still tries, but the encode attempt fails and the connection closes rather than sending a malformed body. @@ -55,6 +55,8 @@ it knows the server supports flexible encoding. | 13 | LeaveGroup | 0 | 5 | 0, 1, 2, 3, 4, 5 | Removes members, per-member errors from v3; flexible encoding at v4+ | | 14 | SyncGroup | 0 | 5 | 0, 1, 2, 3, 4, 5 | Relays the leader's assignment blobs; flexible encoding at v4+ | | 22 | InitProducerId | 0 | 5 | 0, 1, 2, 3, 4, 5 | Allocate a producer id (epoch 0); a `transactional_id` gets `UNSUPPORTED_VERSION` (35); flexible encoding at v2+ | +| 32 | DescribeConfigs | 1 | 4 | 1, 2, 3, 4 | Topic resources only. Returns `retention.ms` and `cleanup.policy`. Flexible encoding at v4. See [`CONFIGS.md`](CONFIGS.md) | +| 33 | AlterConfigs | 0 | 2 | 0, 1, 2 | Persists `retention.ms` only. Flexible encoding at v2. See [`CONFIGS.md`](CONFIGS.md) | A request is accepted when `min_version ≤ api_version ≤ max_version` for that API key. Any other version for a listed key closes the connection (ApiVersions excepted - see Governance model above). @@ -78,6 +80,8 @@ Use this table when configuring clients or generating wire fixtures with `kafka- | 18 | ApiVersions | 0–3 | v3 | | 19 | CreateTopics | 2–5 | v5 | | 22 | InitProducerId | 0–5 | v2 | +| 32 | DescribeConfigs | 1–4 | v4 | +| 33 | AlterConfigs | 0–2 | v2 | --- @@ -94,7 +98,7 @@ All API keys not listed above close the connection (see Governance model above) | 29 | DescribeAcls | Implemented behind `IGGY_KAFKA_SASL_ENABLED`, advertised only while it is on ([`ACL_MAPPING.md`](ACL_MAPPING.md)) | | 36 | SaslAuthenticate | Implemented behind `IGGY_KAFKA_SASL_ENABLED`, advertised only while it is on ([`AUTHENTICATION.md`](AUTHENTICATION.md)) | | 24, 25, 26, 28 | AddPartitionsToTxn, AddOffsetsToTxn, EndTxn, TxnOffsetCommit | Transactions - not supported, see below | -| 20, 23, 27, 30–35, 37+ | DeleteTopics, `OffsetForLeaderEpoch`, `WriteTxnMarkers`, `CreateAcls`/`DeleteAcls`, etc. | Later issues | +| 20, 23, 27, 30, 31, 34, 35, 37+ | DeleteTopics, `OffsetForLeaderEpoch`, `WriteTxnMarkers`, `CreateAcls`/`DeleteAcls`, IncrementalAlterConfigs (44), etc. | Later issues. DescribeConfigs (32) and AlterConfigs (33) are supported | | 21 | DeleteRecords | Not advertised on purpose, see below | | 68 | ConsumerGroupHeartbeat | KIP-848 protocol, opt-in via `group.protocol=consumer`; the 4.0 default is still `classic` | @@ -157,8 +161,8 @@ nor moved by retention. That is the same core change the ListOffsets `EARLIEST` | Layer | #3421 | Description | | ------- | ------- | ------------- | | **1 — Wire framing** | In scope | `server.rs` — custom, zero-copy frame I/O; `header.rs` delegates version selection to `kafka_protocol::messages::ApiKey` | -| **2 — Request/response codecs** | Partial | Decode/encode via the `kafka_protocol` crate (broker feature only) for 12 keys; `bounds_guard.rs` pre-validates against unbounded allocation before handing a frame to the crate; stub responses except InitProducerId and the five consumer-group keys, and Produce, Fetch, ListOffsets, Metadata and CreateTopics with a bridge | -| **3 — Iggy bridge** | Produce, Fetch, ListOffsets, Metadata and CreateTopics wired | `bridge/` module (connection, topic mapping, provisioning, high watermark, `topic_target` + `send_records`, `probe` + `poll`). Produce ([#3535](https://github.com/apache/iggy/issues/3535)), Fetch ([#3536](https://github.com/apache/iggy/issues/3536)), ListOffsets ([#3537](https://github.com/apache/iggy/issues/3537)), Metadata ([#3534](https://github.com/apache/iggy/issues/3534)) and CreateTopics ([#3538](https://github.com/apache/iggy/issues/3538)) call it | +| **2 — Request/response codecs** | Partial | Decode/encode via the `kafka_protocol` crate (broker feature only) for 14 keys; `bounds_guard.rs` pre-validates against unbounded allocation before handing a frame to the crate; stub responses except InitProducerId and the five consumer-group keys, and Produce, Fetch, ListOffsets, Metadata, CreateTopics, DescribeConfigs and AlterConfigs with a bridge | +| **3 — Iggy bridge** | Produce, Fetch, ListOffsets, Metadata, CreateTopics, DescribeConfigs and AlterConfigs wired | `bridge/` module (connection, topic mapping, provisioning, high watermark, `topic_target` + `send_records`, `probe` + `poll`). Produce ([#3535](https://github.com/apache/iggy/issues/3535)), Fetch ([#3536](https://github.com/apache/iggy/issues/3536)), ListOffsets ([#3537](https://github.com/apache/iggy/issues/3537)), Metadata ([#3534](https://github.com/apache/iggy/issues/3534)), CreateTopics ([#3538](https://github.com/apache/iggy/issues/3538)), DescribeConfigs and AlterConfigs call it | --- diff --git a/gateways/kafka/src/bridge/iggy_bridge/topics.rs b/gateways/kafka/src/bridge/iggy_bridge/topics.rs index bf6855ecbf..83c4d7c041 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/topics.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/topics.rs @@ -20,7 +20,8 @@ use std::collections::{HashMap, HashSet}; use iggy::prelude::{ - Identifier, IggyError, StreamClient, TopicClient, TopicCreateOptions, TopicDetails, + Identifier, IggyError, IggyExpiry, StreamClient, TopicClient, TopicCreateOptions, TopicDetails, + TopicUpdateOptions, }; use kafka_protocol::protocol::StrBytes; use tracing::{debug, info}; @@ -283,6 +284,37 @@ impl IggyBridge { with_request_timeout(self.client.get_topic(&stream_id, &topic_id)).await } + /// Sets `message_expiry` on the Iggy topic backing `kafka_topic`. + /// + /// Does not create the topic and does not rename it: `name` is the mapped Iggy + /// topic name, which `update_topic` also takes as the topic's new name. + /// + /// # Errors + /// + /// Returns [`BridgeError::InvalidKafkaTopicName`] if `kafka_topic` fails Kafka's own + /// topic-naming rules. Returns [`BridgeError::Timeout`] if a call takes longer than + /// `REQUEST_TIMEOUT`. Returns [`BridgeError::Iggy`] for connectivity, auth, or a + /// missing topic. + pub async fn update_kafka_topic_message_expiry( + &self, + kafka_topic: &str, + message_expiry: IggyExpiry, + ) -> Result<(), BridgeError> { + validate_kafka_topic_name("kafka_topic", kafka_topic)?; + let (stream_name, topic_name) = self.config.topic_mapping.resolve(kafka_topic); + let stream_id = Identifier::named(stream_name).map_err(BridgeError::Iggy)?; + let topic_id = Identifier::named(topic_name).map_err(BridgeError::Iggy)?; + let options = TopicUpdateOptions { + message_expiry: Some(message_expiry), + ..TopicUpdateOptions::default() + }; + with_request_timeout( + self.client + .update_topic(&stream_id, &topic_id, topic_name, &options), + ) + .await + } + /// Resolves many Kafka-visible names at once, one entry per input in the same order. /// /// Batches by the Iggy stream each name resolves to, rather than paying one round trip per diff --git a/gateways/kafka/src/bridge/mod.rs b/gateways/kafka/src/bridge/mod.rs index 1c8836ac18..abdec3f71a 100644 --- a/gateways/kafka/src/bridge/mod.rs +++ b/gateways/kafka/src/bridge/mod.rs @@ -19,7 +19,7 @@ //! //! Maps Kafka topics to Iggy streams/topics, exposes create-if-missing provisioning and //! high-watermark lookups, and translates Iggy errors to Kafka wire error codes. `ListOffsets`, -//! Produce, Fetch, Metadata and `CreateTopics` call it. +//! Produce, Fetch, Metadata, `CreateTopics`, `DescribeConfigs` and `AlterConfigs` call it. pub mod config; pub mod error; diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index c2478ecbf6..850a15928d 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -38,9 +38,9 @@ use crate::protocol::bounds_guard::{ }; use crate::protocol::handlers::init_producer_id::ProducerIdAllocator; use crate::protocol::handlers::{ - api_versions, create_topics, decode_guarded, dispatch, fetch, find_coordinator, heartbeat, - init_producer_id, join_group, leave_group, list_offsets, metadata, produce, respond_or_close, - sync_group, + alter_configs, api_versions, create_topics, decode_guarded, describe_configs, dispatch, fetch, + find_coordinator, heartbeat, init_producer_id, join_group, leave_group, list_offsets, metadata, + produce, respond_or_close, sync_group, }; use crate::protocol::probe_board::ProbeBoard; use crate::protocol::sasl::{ @@ -59,6 +59,8 @@ pub const API_KEY_SYNC_GROUP: i16 = 14; pub const API_KEY_SASL_HANDSHAKE: i16 = 17; pub const API_KEY_API_VERSIONS: i16 = 18; pub const API_KEY_CREATE_TOPICS: i16 = 19; +pub const API_KEY_DESCRIBE_CONFIGS: i16 = 32; +pub const API_KEY_ALTER_CONFIGS: i16 = 33; pub const API_KEY_INIT_PRODUCER_ID: i16 = 22; pub const API_KEY_DESCRIBE_ACLS: i16 = 29; pub const API_KEY_SASL_AUTHENTICATE: i16 = 36; @@ -136,14 +138,16 @@ pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; /// map's keys the same way regardless of what a client's replica list under each key says (this /// bridge doesn't model replicas at all, so only the key set is checked). pub const ERROR_INVALID_REPLICA_ASSIGNMENT: i16 = 39; -/// `CreateTopics` stub: do not claim topics were created (no controller / no Iggy bridge). +/// No controller and no Iggy bridge. `CreateTopics`, `DescribeConfigs`, and `AlterConfigs` +/// answer this so a client does not treat a stub as a completed create, read, or alter. pub const ERROR_NOT_CONTROLLER: i16 = 41; pub const ERROR_INVALID_REQUEST: i16 = 42; -/// `CreateTopics`: a requested topic carried one or more per-topic Kafka configs. +/// A config this gateway will not apply. /// -/// None of `retention.ms`, `cleanup.policy`, etc. maps onto an Iggy topic option this bridge -/// applies, so every non-empty `configs` list is rejected outright rather than silently dropping -/// a subset an operator might believe took effect. +/// `CreateTopics` rejects any per-topic config list and points the client at `AlterConfigs`. +/// `AlterConfigs` stores `retention.ms` and rejects every other key, and a value that cannot +/// be stored. `DescribeConfigs` sends this when a requested key is unknown or a stored expiry +/// cannot be shown as `retention.ms`. pub const ERROR_INVALID_CONFIG: i16 = 40; /// `ListOffsets`' code for a timestamp lookup the broker cannot perform. /// @@ -153,7 +157,8 @@ pub const ERROR_INVALID_CONFIG: i16 = 40; /// [`ERROR_UNKNOWN_SERVER_ERROR`] until its own `default.api.timeout.ms`. pub const ERROR_UNSUPPORTED_FOR_MESSAGE_FORMAT: i16 = ResponseError::UnsupportedForMessageFormat.code(); -/// `CreateTopics`: request addressed more distinct topics than this bridge admits in one call. +/// `CreateTopics`, `DescribeConfigs`, and `AlterConfigs`: the request addressed more distinct +/// topics than this bridge admits in one call. /// /// A server-imposed limit, not a malformed request - `INVALID_REQUEST` would blame the client for /// a request Kafka itself would accept. @@ -272,6 +277,8 @@ static SUPPORTED_RANGES: &[ApiVersionRange] = &[ metadata::RANGE, api_versions::RANGE, create_topics::RANGE, + describe_configs::RANGE, + alter_configs::RANGE, init_producer_id::RANGE, find_coordinator::RANGE, join_group::RANGE, @@ -664,6 +671,10 @@ pub fn encode_error_for_key( API_KEY_FETCH => fetch::encode_error_response(api_version, error_code), API_KEY_LIST_OFFSETS => list_offsets::encode_error_response(api_version, error_code), API_KEY_CREATE_TOPICS => create_topics::encode_error_response(api_version, error_code), + API_KEY_DESCRIBE_CONFIGS => { + describe_configs::encode_error_response(api_version, error_code) + } + API_KEY_ALTER_CONFIGS => alter_configs::encode_error_response(api_version, error_code), // Carries the live SASL setting, not a hardcoded `false`: answering an illegal-state // ApiVersions with a SASL-less key set contradicts the advertisement sent one frame // earlier on the same connection. diff --git a/gateways/kafka/src/protocol/bounds_guard.rs b/gateways/kafka/src/protocol/bounds_guard.rs index ae265b8ed9..5fd688d221 100644 --- a/gateways/kafka/src/protocol/bounds_guard.rs +++ b/gateways/kafka/src/protocol/bounds_guard.rs @@ -26,8 +26,8 @@ //! topics count requests ~481 GB; a 4-byte Metadata v0 frame requests ~143 GiB. No SASL/TLS gates //! any of `SUPPORTED_RANGES`, so this is reachable by anyone who can `connect()`. //! -//! This module walks the same field shape `kafka_protocol`'s real decode walks for each of the -//! eleven accepted message types, but only to validate every length-prefixed field (array count, +//! This module walks the same field shape `kafka_protocol`'s real decode walks for each accepted +//! message type, but only to validate every length-prefixed field (array count, //! string length, bytes length, tagged-field size) against what could still fit in the bytes //! remaining in the frame - it never materializes a value or allocates a collection. Call the //! matching `validate_*_shape` function before handing the body to `kafka_protocol`. @@ -1052,6 +1052,110 @@ pub fn validate_sync_group_shape(version: i16, body: &Bytes, max_frame_size: usi /// reaches `parse_plain`, and what a future mechanism would carry into a credential exchange. const MAX_SASL_AUTH_BYTES: usize = 4096; +/// Mirrors the field order `DescribeConfigsRequest::decode` walks. +/// +/// # Errors +/// +/// Returns an error when a declared array or string length cannot fit in the bytes remaining +/// in the frame, or the body is truncated or malformed in a way that cannot be walked. +pub fn validate_describe_configs_shape( + version: i16, + body: &Bytes, + max_frame_size: usize, +) -> Result<()> { + let mut c = ShapeCursor::new(body.clone(), max_frame_size); + let flexible = version >= 4; + + let resources_count = if flexible { + c.compact_array_count()? + } else { + c.legacy_array_count()? + }; + for _ in 0..resources_count { + let _resource_type = c.read_i8()?; + if flexible { + c.compact_string(false)?; + } else { + c.legacy_string(false)?; + } + let keys_count = if flexible { + c.compact_array_count_nullable()? + } else { + c.legacy_array_count_nullable()? + }; + for _ in 0..keys_count { + if flexible { + c.compact_string(false)?; + } else { + c.legacy_string(false)?; + } + } + if flexible { + c.tagged_fields()?; + } + } + let _include_synonyms = c.read_bool()?; + if version >= 3 { + let _include_documentation = c.read_bool()?; + } + if flexible { + c.tagged_fields()?; + } + Ok(()) +} + +/// Mirrors the field order `AlterConfigsRequest::decode` walks. +/// +/// # Errors +/// +/// Returns an error when a declared array or string length cannot fit in the bytes remaining +/// in the frame, or the body is truncated or malformed in a way that cannot be walked. +pub fn validate_alter_configs_shape( + version: i16, + body: &Bytes, + max_frame_size: usize, +) -> Result<()> { + let mut c = ShapeCursor::new(body.clone(), max_frame_size); + let flexible = version >= 2; + + let resources_count = if flexible { + c.compact_array_count()? + } else { + c.legacy_array_count()? + }; + for _ in 0..resources_count { + let _resource_type = c.read_i8()?; + if flexible { + c.compact_string(false)?; + } else { + c.legacy_string(false)?; + } + let configs_count = if flexible { + c.compact_array_count()? + } else { + c.legacy_array_count()? + }; + for _ in 0..configs_count { + if flexible { + c.compact_string(false)?; + c.compact_string(true)?; + c.tagged_fields()?; + } else { + c.legacy_string(false)?; + c.legacy_string(true)?; + } + } + if flexible { + c.tagged_fields()?; + } + } + let _validate_only = c.read_bool()?; + if flexible { + c.tagged_fields()?; + } + Ok(()) +} + /// `DescribeAcls` carries a fixed-shape filter: four enums and three nullable strings. /// /// No arrays and nothing echoed into the response, so there is no amplification to project. The diff --git a/gateways/kafka/src/protocol/handlers/create_topics.rs b/gateways/kafka/src/protocol/handlers/create_topics.rs index ee79f93519..97e4903a85 100644 --- a/gateways/kafka/src/protocol/handlers/create_topics.rs +++ b/gateways/kafka/src/protocol/handlers/create_topics.rs @@ -333,7 +333,8 @@ fn local_shape_error( return Err(( ERROR_INVALID_CONFIG, Some(StrBytes::from( - "per-topic configs are not supported by this bridge".to_string(), + "CreateTopics does not apply per-topic configs. Set retention.ms with AlterConfigs" + .to_string(), )), )); } diff --git a/gateways/kafka/src/protocol/handlers/mod.rs b/gateways/kafka/src/protocol/handlers/mod.rs index 4ecfbe9fe3..229d29b802 100644 --- a/gateways/kafka/src/protocol/handlers/mod.rs +++ b/gateways/kafka/src/protocol/handlers/mod.rs @@ -24,8 +24,10 @@ //! `kafka_protocol` owns wire encoding. What lives here is policy: which placeholder values and //! error codes a request gets back, and when the connection closes instead. +pub mod alter_configs; pub mod api_versions; pub mod create_topics; +pub mod describe_configs; pub mod fetch; pub mod find_coordinator; pub mod heartbeat; @@ -36,6 +38,7 @@ pub mod list_offsets; pub mod metadata; pub mod produce; pub mod sync_group; +pub(crate) mod topic_config; use bytes::{Buf, Bytes, BytesMut}; use kafka_protocol::messages::TransactionalId; @@ -44,11 +47,12 @@ use tokio::runtime::{Handle, RuntimeFlavor}; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::api::{ - API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_FIND_COORDINATOR, - API_KEY_HEARTBEAT, API_KEY_INIT_PRODUCER_ID, API_KEY_JOIN_GROUP, API_KEY_LEAVE_GROUP, - API_KEY_LIST_OFFSETS, API_KEY_METADATA, API_KEY_PRODUCE, API_KEY_SYNC_GROUP, ConnectionState, - ERROR_INVALID_REQUEST, ERROR_UNSUPPORTED_VERSION, GatewayState, HandleOutcome, - is_supported_version, supported_max_version, + API_KEY_ALTER_CONFIGS, API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_DESCRIBE_CONFIGS, + API_KEY_FETCH, API_KEY_FIND_COORDINATOR, API_KEY_HEARTBEAT, API_KEY_INIT_PRODUCER_ID, + API_KEY_JOIN_GROUP, API_KEY_LEAVE_GROUP, API_KEY_LIST_OFFSETS, API_KEY_METADATA, + API_KEY_PRODUCE, API_KEY_SYNC_GROUP, ConnectionState, ERROR_INVALID_REQUEST, + ERROR_UNSUPPORTED_VERSION, GatewayState, HandleOutcome, is_supported_version, + supported_max_version, }; /// Record encodes and decodes of this many bytes or more run off the async worker. @@ -86,6 +90,8 @@ pub async fn dispatch( API_KEY_METADATA => metadata::handle(state, api_version, body).await, API_KEY_API_VERSIONS => api_versions::handle(state, api_version, body).await, API_KEY_CREATE_TOPICS => create_topics::handle(state, api_version, body).await, + API_KEY_DESCRIBE_CONFIGS => describe_configs::handle(state, api_version, body).await, + API_KEY_ALTER_CONFIGS => alter_configs::handle(state, api_version, body).await, API_KEY_FIND_COORDINATOR => find_coordinator::handle(state, api_version, body).await, API_KEY_JOIN_GROUP => join_group::handle(state, api_version, body).await, API_KEY_HEARTBEAT => heartbeat::handle(state, api_version, body).await, diff --git a/gateways/kafka/tests/common/scope.rs b/gateways/kafka/tests/common/scope.rs index 2926b0751e..e0be8d519b 100644 --- a/gateways/kafka/tests/common/scope.rs +++ b/gateways/kafka/tests/common/scope.rs @@ -38,6 +38,8 @@ pub const SCOPED_API_KEYS: &[(i16, &str, i16, i16)] = &[ (18, "ApiVersions", 0, 3), (19, "CreateTopics", 2, 5), (22, "InitProducerId", 0, 5), + (32, "DescribeConfigs", 1, 4), + (33, "AlterConfigs", 0, 2), ]; pub fn default_broker() -> BrokerAdvertise { diff --git a/gateways/kafka/tests/common/wire.rs b/gateways/kafka/tests/common/wire.rs index 89db5f219e..1605f18856 100644 --- a/gateways/kafka/tests/common/wire.rs +++ b/gateways/kafka/tests/common/wire.rs @@ -215,6 +215,41 @@ pub fn build_produce_flexible_empty_request(acks: i16) -> Bytes { enc.freeze() } +/// `DescribeConfigs` with no resources. Flexible from v4. `include_documentation` exists from v3. +pub fn build_describe_configs_empty_request(version: i16) -> Bytes { + let flexible = version >= 4; + let mut enc = Encoder::with_capacity(16); + if flexible { + enc.write_varint(1); + } else { + enc.write_i32(0); + } + enc.write_bool(false); // include_synonyms + if version >= 3 { + enc.write_bool(false); // include_documentation + } + if flexible { + enc.write_empty_tagged_fields(); + } + enc.freeze() +} + +/// `AlterConfigs` with no resources. Flexible from v2. +pub fn build_alter_configs_empty_request(version: i16) -> Bytes { + let flexible = version >= 2; + let mut enc = Encoder::with_capacity(16); + if flexible { + enc.write_varint(1); + } else { + enc.write_i32(0); + } + enc.write_bool(false); // validate_only + if flexible { + enc.write_empty_tagged_fields(); + } + enc.freeze() +} + /// `InitProducerId` request for any supported version (v0-v5), flexible from v2. pub fn build_init_producer_id_request(version: i16, transactional_id: Option<&str>) -> Bytes { let flexible = version >= 2; diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index e99b775847..199d40ef7f 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -41,11 +41,11 @@ async fn golden_apiversions_v3_flexible_response_fixture() { .await .expect_response("test request has acks != 0 and expects a response"); - // error_code=0, api_count=12 (compact array: N+1=13) + // error_code=0, api_count=14 (compact array: N+1=15) // each entry followed by an empty tagged-fields byte; throttle_ms=0; top-level tagged fields - let expected: [u8; 92] = [ + let expected: [u8; 106] = [ 0x00, 0x00, // error_code - 0x0D, // compact array count (12+1) + 0x0F, // compact array count (14+1) 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, // key 0: Produce 0-9 (advertised) 0x00, 0x01, 0x00, 0x04, 0x00, 0x0C, 0x00, // key 1: Fetch 4-12 0x00, 0x02, 0x00, 0x01, 0x00, 0x06, 0x00, // key 2: ListOffsets 1-6 @@ -58,6 +58,8 @@ async fn golden_apiversions_v3_flexible_response_fixture() { 0x00, 0x12, 0x00, 0x00, 0x00, 0x03, 0x00, // key 18: ApiVersions 0-3 0x00, 0x13, 0x00, 0x02, 0x00, 0x05, 0x00, // key 19: CreateTopics 2-5 0x00, 0x16, 0x00, 0x00, 0x00, 0x05, 0x00, // key 22: InitProducerId 0-5 + 0x00, 0x20, 0x00, 0x01, 0x00, 0x04, 0x00, // key 32: DescribeConfigs 1-4 + 0x00, 0x21, 0x00, 0x00, 0x00, 0x02, 0x00, // key 33: AlterConfigs 0-2 0x00, 0x00, 0x00, 0x00, // throttle_ms 0x00, // top-level tagged fields ]; @@ -71,10 +73,10 @@ async fn golden_apiversions_v1_response_fixture() { .await .expect_response("test request has acks != 0 and expects a response"); - // error_code=0, api_count=12; Produce advertises min=0 per KAFKA-18659; throttle_ms=0 - let expected: [u8; 82] = [ + // error_code=0, api_count=14; Produce advertises min=0 per KAFKA-18659; throttle_ms=0 + let expected: [u8; 94] = [ 0x00, 0x00, // error_code - 0x00, 0x00, 0x00, 0x0C, // api count = 12 + 0x00, 0x00, 0x00, 0x0E, // api count = 14 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, // key 0: Produce 0-9 (advertised) 0x00, 0x01, 0x00, 0x04, 0x00, 0x0C, // key 1: Fetch 4-12 0x00, 0x02, 0x00, 0x01, 0x00, 0x06, // key 2: ListOffsets 1-6 @@ -87,6 +89,8 @@ async fn golden_apiversions_v1_response_fixture() { 0x00, 0x12, 0x00, 0x00, 0x00, 0x03, // key 18: ApiVersions 0-3 0x00, 0x13, 0x00, 0x02, 0x00, 0x05, // key 19: CreateTopics 2-5 0x00, 0x16, 0x00, 0x00, 0x00, 0x05, // key 22: InitProducerId 0-5 + 0x00, 0x20, 0x00, 0x01, 0x00, 0x04, // key 32: DescribeConfigs 1-4 + 0x00, 0x21, 0x00, 0x00, 0x00, 0x02, // key 33: AlterConfigs 0-2 0x00, 0x00, 0x00, 0x00, // throttle_ms ]; assert_eq!(actual.as_ref(), &expected); diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index 63199e2002..549e77f00e 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -37,11 +37,12 @@ use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; use iggy_gateway_kafka::protocol::api::{ - API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_FIND_COORDINATOR, - API_KEY_HEARTBEAT, API_KEY_INIT_PRODUCER_ID, API_KEY_JOIN_GROUP, API_KEY_LEAVE_GROUP, - API_KEY_LIST_OFFSETS, API_KEY_METADATA, API_KEY_PRODUCE, API_KEY_SYNC_GROUP, - ERROR_INVALID_REQUEST, ERROR_NONE, ERROR_UNSUPPORTED_VERSION, advertised_min_version, - handle_request, is_supported_version, supported_api_ranges, + API_KEY_ALTER_CONFIGS, API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_DESCRIBE_CONFIGS, + API_KEY_FETCH, API_KEY_FIND_COORDINATOR, API_KEY_HEARTBEAT, API_KEY_INIT_PRODUCER_ID, + API_KEY_JOIN_GROUP, API_KEY_LEAVE_GROUP, API_KEY_LIST_OFFSETS, API_KEY_METADATA, + API_KEY_PRODUCE, API_KEY_SYNC_GROUP, ERROR_INVALID_REQUEST, ERROR_NONE, + ERROR_UNSUPPORTED_VERSION, advertised_min_version, handle_request, is_supported_version, + supported_api_ranges, }; use codec::Decoder; @@ -54,8 +55,9 @@ use tcp::{ parse_response_payload, read_byte_with_timeout, round_trip, }; use wire::{ - JoinGroupParams, OUT_OF_SCOPE_API_KEYS, SyncGroupParams, build_api_versions_flexible_request, - build_create_topics_empty_request, build_fetch_empty_topics_request, + JoinGroupParams, OUT_OF_SCOPE_API_KEYS, SyncGroupParams, build_alter_configs_empty_request, + build_api_versions_flexible_request, build_create_topics_empty_request, + build_describe_configs_empty_request, build_fetch_empty_topics_request, build_find_coordinator_request, build_heartbeat_request, build_init_producer_id_request, build_join_group_request, build_leave_group_request, build_list_offsets_request, build_metadata_all_topics_flexible, build_metadata_all_topics_legacy, @@ -63,8 +65,8 @@ use wire::{ }; #[test] -fn supported_ranges_table_has_twelve_entries() { - assert_eq!(supported_api_ranges().len(), 12); +fn supported_ranges_table_has_fourteen_entries() { + assert_eq!(supported_api_ranges().len(), 14); } #[test] @@ -101,7 +103,7 @@ fn is_supported_version_matches_scope_table() { /// /// Relies on `SUPPORTED_RANGES` (src) and `SCOPED_API_KEYS` (test) sharing declaration order /// (Produce, Fetch, `ListOffsets`, Metadata, `ApiVersions`, `CreateTopics`, `FindCoordinator`, -/// `JoinGroup`, Heartbeat, `LeaveGroup`, `SyncGroup`) - `supported_ranges_table_has_twelve_entries` plus +/// `JoinGroup`, Heartbeat, `LeaveGroup`, `SyncGroup`) - `supported_ranges_table_has_fourteen_entries` plus /// `is_supported_version_matches_scope_table` already pin that both tables cover the same keys. #[tokio::test] async fn apiversions_advertises_exact_supported_ranges_v1() { @@ -518,6 +520,8 @@ fn request_body_for_scoped_api(api_key: i16, name: &str, version: i16) -> Bytes }, ), API_KEY_INIT_PRODUCER_ID => build_init_producer_id_request(version, None), + API_KEY_DESCRIBE_CONFIGS => build_describe_configs_empty_request(version), + API_KEY_ALTER_CONFIGS => build_alter_configs_empty_request(version), _ => Bytes::new(), } } @@ -850,3 +854,55 @@ async fn corrupt_create_topics_body_returns_invalid_request_error() { assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); } + +/// v1 is legacy. A count that stops mid-integer never reaches `kafka_protocol`. +#[tokio::test] +async fn corrupt_describe_configs_truncated_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_DESCRIBE_CONFIGS, 1, body, &default_broker()) + .await + .expect_response("DescribeConfigs v1 has an encodable error response"); + let mut d = Decoder::new(resp); + assert_eq!(d.read_i32().unwrap(), 0, "throttle"); + assert_eq!(d.read_i32().unwrap(), 1, "results len"); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + +/// A resource count past what the remaining frame can hold is rejected by the shape walk. +#[tokio::test] +async fn corrupt_describe_configs_overlong_count_returns_invalid_request_error() { + let body = Bytes::from_static(&[0x00, 0x01, 0x86, 0xA0]); + let resp = handle_request(API_KEY_DESCRIBE_CONFIGS, 1, body, &default_broker()) + .await + .expect_response("DescribeConfigs v1 has an encodable error response"); + let mut d = Decoder::new(resp); + assert_eq!(d.read_i32().unwrap(), 0, "throttle"); + assert_eq!(d.read_i32().unwrap(), 1, "results len"); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + +/// v0 is legacy. A count that stops mid-integer never reaches `kafka_protocol`. +#[tokio::test] +async fn corrupt_alter_configs_truncated_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_ALTER_CONFIGS, 0, body, &default_broker()) + .await + .expect_response("AlterConfigs v0 has an encodable error response"); + let mut d = Decoder::new(resp); + assert_eq!(d.read_i32().unwrap(), 0, "throttle"); + assert_eq!(d.read_i32().unwrap(), 1, "responses len"); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + +/// A resource count past what the remaining frame can hold is rejected by the shape walk. +#[tokio::test] +async fn corrupt_alter_configs_overlong_count_returns_invalid_request_error() { + let body = Bytes::from_static(&[0x00, 0x01, 0x86, 0xA0]); + let resp = handle_request(API_KEY_ALTER_CONFIGS, 0, body, &default_broker()) + .await + .expect_response("AlterConfigs v0 has an encodable error response"); + let mut d = Decoder::new(resp); + assert_eq!(d.read_i32().unwrap(), 0, "throttle"); + assert_eq!(d.read_i32().unwrap(), 1, "responses len"); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} diff --git a/gateways/kafka/tools/kafka-tool/README.md b/gateways/kafka/tools/kafka-tool/README.md index 22a2bdf522..2be828d8fa 100644 --- a/gateways/kafka/tools/kafka-tool/README.md +++ b/gateways/kafka/tools/kafka-tool/README.md @@ -230,8 +230,8 @@ Kafka introduced "flexible" encoding (compact ULEB128 strings/arrays) starting a ### API Key Coverage -- **Explicit builders (25 API keys):** Produce, Fetch, ListOffsets, Metadata, OffsetCommit, OffsetFetch, FindCoordinator, JoinGroup, Heartbeat, LeaveGroup, SyncGroup, DescribeGroups, ListGroups, SaslHandshake, ApiVersions, CreateTopics, DeleteTopics, DeleteRecords, InitProducerId, AddPartitionsToTxn, AddOffsetsToTxn, EndTxn, TxnOffsetCommit, DescribeConfigs, SaslAuthenticate -- **Header-framing test (39 API keys):** All remaining API keys are framed correctly with an empty payload — useful for testing that your server returns a proper error response rather than crashing +- **Explicit builders (26 API keys):** Produce, Fetch, ListOffsets, Metadata, OffsetCommit, OffsetFetch, FindCoordinator, JoinGroup, Heartbeat, LeaveGroup, SyncGroup, DescribeGroups, ListGroups, SaslHandshake, ApiVersions, CreateTopics, DeleteTopics, DeleteRecords, InitProducerId, AddPartitionsToTxn, AddOffsetsToTxn, EndTxn, TxnOffsetCommit, DescribeConfigs, AlterConfigs, SaslAuthenticate +- **Header-framing test (38 API keys):** All remaining API keys are framed correctly with an empty payload — useful for testing that your server returns a proper error response rather than crashing --- diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs index 0d70a23feb..0944e8ac68 100644 --- a/gateways/kafka/tools/kafka-tool/src/main.rs +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -20,6 +20,7 @@ use bytes::{BufMut, Bytes, BytesMut}; use clap::{Parser, Subcommand}; use iggy_gateway_kafka::protocol::api::supported_api_ranges; use kafka_protocol::messages::add_partitions_to_txn_request::*; +use kafka_protocol::messages::alter_configs_request::AlterConfigsResource; use kafka_protocol::messages::create_topics_request::*; use kafka_protocol::messages::delete_records_request::*; use kafka_protocol::messages::delete_topics_request::*; @@ -550,6 +551,15 @@ fn build_payload(api_key: i16, version: i16) -> Result { .encode(&mut buf, version) .context("DescribeConfigs")?; } + 33 => { + let resource = AlterConfigsResource::default() + .with_resource_type(2) + .with_resource_name(StrBytes::from_static_str("test-topic")); + AlterConfigsRequest::default() + .with_resources(vec![resource]) + .encode(&mut buf, version) + .context("AlterConfigs")?; + } 36 => { SaslAuthenticateRequest::default() .with_auth_bytes(Bytes::from_static(b"\x00iggy\x00secret")) diff --git a/gateways/kafka/tools/kafka-tool/src/response.rs b/gateways/kafka/tools/kafka-tool/src/response.rs index 467f225fd4..d779da525d 100644 --- a/gateways/kafka/tools/kafka-tool/src/response.rs +++ b/gateways/kafka/tools/kafka-tool/src/response.rs @@ -19,8 +19,9 @@ use bytes::Bytes; use kafka_protocol::messages::{ - ApiKey, ApiVersionsResponse, CreateTopicsResponse, FetchResponse, InitProducerIdResponse, - ListOffsetsResponse, MetadataResponse, ProduceResponse, + AlterConfigsResponse, ApiKey, ApiVersionsResponse, CreateTopicsResponse, + DescribeConfigsResponse, FetchResponse, InitProducerIdResponse, ListOffsetsResponse, + MetadataResponse, ProduceResponse, }; use kafka_protocol::protocol::Decodable; @@ -106,7 +107,7 @@ fn is_acceptable_verify_error(api_key: i16, error_code: i16) -> bool { match api_key { 0..=2 => error_code == 6, // Produce/Fetch/ListOffsets stub: NOT_LEADER_OR_FOLLOWER 3 => error_code == 3, // Metadata stub: UNKNOWN_TOPIC_OR_PARTITION - 19 => error_code == 41, // CreateTopics stub: NOT_CONTROLLER + 19 | 32 | 33 => error_code == 41, // CreateTopics / DescribeConfigs / AlterConfigs stub _ => false, } } @@ -355,6 +356,37 @@ fn decode_body( )); } } + 32 => { + let resp = DescribeConfigsResponse::decode(&mut buf, api_version)?; + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + details.push(format!("results={}", resp.results.len())); + for (i, result) in resp.results.iter().enumerate().take(4) { + codes.push(result.error_code); + details.push(format!( + "results[{i}]: type={} name={} ec={} ({}) configs={}", + result.resource_type, + result.resource_name, + result.error_code, + format_error_code(result.error_code), + result.configs.len() + )); + } + } + 33 => { + let resp = AlterConfigsResponse::decode(&mut buf, api_version)?; + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + details.push(format!("responses={}", resp.responses.len())); + for (i, response) in resp.responses.iter().enumerate().take(4) { + codes.push(response.error_code); + details.push(format!( + "responses[{i}]: type={} name={} ec={} ({})", + response.resource_type, + response.resource_name, + response.error_code, + format_error_code(response.error_code) + )); + } + } 22 => { let resp = InitProducerIdResponse::decode(&mut buf, api_version)?; codes.push(resp.error_code); From bb74467721e721a3b1ef6aa41c63be053ad47307 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 4 Oct 2026 06:48:44 -0400 Subject: [PATCH 03/13] Implementing review commands. gateways/kafka/docs/CONFIGS.md gateways/kafka/src/protocol/handlers/alter_configs.rs gateways/kafka/src/protocol/handlers/describe_configs.rs gateways/kafka/src/protocol/handlers/topic_config.rs gateways/kafka/tests/common/wire.rs gateways/kafka/tests/configs_handler_tests.rs gateways/kafka/tests/configs_java_client_tests.rs gateways/kafka/tests/configs_real_bridge_tests.rs --- gateways/kafka/docs/CONFIGS.md | 9 +- .../src/protocol/handlers/alter_configs.rs | 154 +++++-------- .../src/protocol/handlers/describe_configs.rs | 215 ++++++++---------- .../src/protocol/handlers/topic_config.rs | 96 +++++++- gateways/kafka/tests/common/wire.rs | 16 +- gateways/kafka/tests/configs_handler_tests.rs | 21 +- .../kafka/tests/configs_java_client_tests.rs | 37 ++- .../kafka/tests/configs_real_bridge_tests.rs | 141 ++++++++++-- 8 files changed, 424 insertions(+), 265 deletions(-) diff --git a/gateways/kafka/docs/CONFIGS.md b/gateways/kafka/docs/CONFIGS.md index b734ca0a16..4acf0a942e 100644 --- a/gateways/kafka/docs/CONFIGS.md +++ b/gateways/kafka/docs/CONFIGS.md @@ -34,7 +34,9 @@ the never-expire default. The gateway does not round. ## What an alter stores -Only `retention.ms` is written, through `update_topic`'s `message_expiry`. +Only `retention.ms` is written, through `update_topic`'s `message_expiry`. A resource +whose `configs` list does not name `retention.ms` is left unchanged. Omitting the key +does not clear a previously set expiry. - `-1` stores `IggyExpiry::NeverExpire`. The server treats `0` (`IggyExpiry::ServerDefault`) as "leave the current expiry", so `-1` is not @@ -63,5 +65,6 @@ call `update_topic`. | Resource type other than topic | `INVALID_REQUEST` (42) | Message is `only topic resources are supported`. Other resources in the batch still run | | Topic name fails the same rules as CreateTopics | `INVALID_TOPIC_EXCEPTION` (17) | The message is the validation reason and does not repeat the topic name | | Topic does not exist | `UNKNOWN_TOPIC_OR_PARTITION` (3) | Checked before config keys, so a missing topic is not `INVALID_CONFIG` | -| More than 100 distinct topic names | `POLICY_VIOLATION` (44) | Every resource in the request. Duplicate names count once and are not an error by themselves | -| Non-empty tagged fields | `INVALID_REQUEST` (42) | Request-level tags fail every resource. Per-resource or per-entry tags fail that resource | +| More than 100 distinct topic names | `POLICY_VIOLATION` (44) | Every resource in the request | +| A topic name repeated across resources | `INVALID_REQUEST` (42) | Every occurrence of the duplicate, not just the second one. Same choice CreateTopics makes for a repeated topic name | +| Non-empty unknown tagged fields | `INVALID_REQUEST` (42) | Request-level tags fail every resource. A resource or config entry's own tags fail that resource. Empty tagged fields are the normal flexible encoding | diff --git a/gateways/kafka/src/protocol/handlers/alter_configs.rs b/gateways/kafka/src/protocol/handlers/alter_configs.rs index 26154fb757..8da1671b3d 100644 --- a/gateways/kafka/src/protocol/handlers/alter_configs.rs +++ b/gateways/kafka/src/protocol/handlers/alter_configs.rs @@ -17,8 +17,6 @@ //! `AlterConfigs` (API key 33). -use std::collections::HashMap; - use bytes::Bytes; use iggy::prelude::IggyError; use kafka_protocol::messages::AlterConfigsRequest; @@ -39,8 +37,8 @@ use crate::protocol::api::{ }; use crate::protocol::bounds_guard::validate_alter_configs_shape; use crate::protocol::handlers::topic_config::{ - ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, UNKNOWN_TAGGED_FIELDS, exceeds_topic_cap, - plan_retention_update, topic_cap_message, + ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, bridge_failure, exceeds_topic_cap, + find_duplicate_names, name_reason, plan_retention_update, static_text, topic_cap_message, }; use crate::protocol::handlers::{ decode_guarded, encode_message, is_supported_version, respond_or_close, @@ -73,17 +71,6 @@ pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> Hand } }; - if !req.unknown_tagged_fields.is_empty() { - return respond_or_close( - encode_message( - &response(reject_tagged_resources(&req.resources)), - api_version, - 256, - ), - "AlterConfigs", - ); - } - let Some(bridge) = &state.bridge else { let responses = req .resources @@ -129,12 +116,33 @@ async fn alter_all( req: &AlterConfigsRequest, deadline: Instant, ) -> Vec { - let topic_names = req + if !req.unknown_tagged_fields.is_empty() { + let message = static_text("unknown tagged fields are not supported"); + return if req.resources.is_empty() { + vec![ + AlterConfigsResourceResponse::default() + .with_error_code(ERROR_INVALID_REQUEST) + .with_error_message(Some(message)) + .with_resource_type(RESOURCE_TYPE_TOPIC) + .with_resource_name(StrBytes::from_static_str("")), + ] + } else { + req.resources + .iter() + .map(|resource| { + resource_error(resource, ERROR_INVALID_REQUEST, Some(message.clone())) + }) + .collect() + }; + } + + let topic_names: Vec<&str> = req .resources .iter() .filter(|resource| resource.resource_type == RESOURCE_TYPE_TOPIC) - .map(|resource| resource.resource_name.as_str()); - if exceeds_topic_cap(topic_names) { + .map(|resource| resource.resource_name.as_str()) + .collect(); + if exceeds_topic_cap(topic_names.iter().copied()) { let message = StrBytes::from(topic_cap_message("AlterConfigs")); tracing::warn!("AlterConfigs request addresses too many distinct topics; rejecting"); return req @@ -144,14 +152,17 @@ async fn alter_all( .collect(); } - let mut seen: HashMap = HashMap::new(); + // Real Kafka refuses every occurrence of a duplicate resource name outright rather than + // picking a winner - the same choice CreateTopics makes for a repeated topic name. + let duplicate_names = find_duplicate_names(topic_names.iter().copied()); + let mut responses = Vec::with_capacity(req.resources.len()); let mut deadline_exceeded = false; for resource in &req.resources { if resource.resource_type == RESOURCE_TYPE_TOPIC - && let Some(prior) = seen.get(resource.resource_name.as_str()) + && duplicate_names.contains(resource.resource_name.as_str()) { - responses.push(prior.clone()); + responses.push(resource_error(resource, ERROR_INVALID_REQUEST, None)); continue; } if deadline_exceeded || Instant::now() >= deadline { @@ -162,9 +173,7 @@ async fn alter_all( instead of starting new Iggy calls" ); } - let timed_out = resource_error(resource, ERROR_REQUEST_TIMED_OUT, None); - remember_topic(&mut seen, resource, &timed_out); - responses.push(timed_out); + responses.push(resource_error(resource, ERROR_REQUEST_TIMED_OUT, None)); continue; } @@ -172,22 +181,11 @@ async fn alter_all( if Instant::now() >= deadline { deadline_exceeded = true; } - remember_topic(&mut seen, resource, &result); responses.push(result); } responses } -fn remember_topic( - seen: &mut HashMap, - resource: &AlterConfigsResource, - result: &AlterConfigsResourceResponse, -) { - if resource.resource_type == RESOURCE_TYPE_TOPIC { - seen.insert(resource.resource_name.as_str().to_string(), result.clone()); - } -} - async fn alter_one( bridge: &IggyBridge, resource: &AlterConfigsResource, @@ -203,7 +201,7 @@ async fn alter_one( return resource_error( resource, ERROR_INVALID_REQUEST, - Some(static_text(UNKNOWN_TAGGED_FIELDS)), + Some(static_text("unknown tagged fields are not supported")), ); } if resource.resource_type != RESOURCE_TYPE_TOPIC { @@ -232,15 +230,23 @@ async fn alter_one( ); return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None); } - match bridge.get_kafka_topic(name).await { - Ok(Some(_topic)) => {} - Ok(None) => { + match tokio::time::timeout_at(deadline, bridge.get_kafka_topic(name)).await { + Ok(Ok(Some(_topic))) => {} + Ok(Ok(None)) => { return resource_error(resource, ERROR_UNKNOWN_TOPIC_OR_PARTITION, None); } - Err(error) => { - let (code, message) = bridge_failure(&error); + Ok(Err(error)) => { + let (code, message) = bridge_failure(&error, "AlterConfigs", "altering"); return resource_error(resource, code, message); } + Err(_elapsed) => { + tracing::warn!( + resource = name, + "AlterConfigs: this resource's bridge work exceeded the request deadline; \ + answering retriable instead of blocking further" + ); + return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None); + } } let expiry = match plan_retention_update(resource.configs.iter().map(|config| { @@ -258,12 +264,14 @@ async fn alter_one( ); } }; - - // `validate_only` already performed the existence read and the same key checks. - // It does not call `update_topic`. + // A resource that does not name retention.ms has nothing to store. Omitting the key + // does not clear a previously set expiry. let Some(expiry) = expiry else { return resource_error(resource, ERROR_NONE, None); }; + + // `validate_only` already performed the existence read and the same key checks. + // It does not call `update_topic`. if validate_only { return resource_error(resource, ERROR_NONE, None); } @@ -284,36 +292,12 @@ async fn alter_one( resource_error(resource, ERROR_NONE, None) } Err(error) => { - let (code, message) = bridge_failure(&error); + let (code, message) = bridge_failure(&error, "AlterConfigs", "altering"); resource_error(resource, code, message) } } } -fn reject_tagged_resources( - resources: &[AlterConfigsResource], -) -> Vec { - if resources.is_empty() { - return vec![ - AlterConfigsResourceResponse::default() - .with_error_code(ERROR_INVALID_REQUEST) - .with_error_message(Some(static_text(UNKNOWN_TAGGED_FIELDS))) - .with_resource_type(0) - .with_resource_name(StrBytes::from_static_str("")), - ]; - } - resources - .iter() - .map(|resource| { - resource_error( - resource, - ERROR_INVALID_REQUEST, - Some(static_text(UNKNOWN_TAGGED_FIELDS)), - ) - }) - .collect() -} - fn resource_error( resource: &AlterConfigsResource, error_code: i16, @@ -325,35 +309,3 @@ fn resource_error( .with_resource_type(resource.resource_type) .with_resource_name(resource.resource_name.clone()) } - -fn name_reason(error: &BridgeError) -> String { - match error { - BridgeError::InvalidKafkaTopicName { reason, .. } => reason.clone(), - _ => "invalid topic name".to_string(), - } -} - -fn bridge_failure(error: &BridgeError) -> (i16, Option) { - let code = error.to_kafka_error_code(); - match error { - BridgeError::InvalidKafkaTopicName { reason, .. } => { - tracing::debug!(reason, "AlterConfigs rejected an invalid topic name"); - (code, Some(StrBytes::from(reason.clone()))) - } - BridgeError::Timeout | BridgeError::SendLost(_) => { - tracing::warn!(%error, "AlterConfigs exceeded the bridge deadline"); - (code, None) - } - other => { - tracing::error!(%other, "AlterConfigs failed while updating topic configuration"); - ( - code, - Some(static_text("internal error updating topic configuration")), - ) - } - } -} - -const fn static_text(message: &'static str) -> StrBytes { - StrBytes::from_static_str(message) -} diff --git a/gateways/kafka/src/protocol/handlers/describe_configs.rs b/gateways/kafka/src/protocol/handlers/describe_configs.rs index 530b305a41..200261443e 100644 --- a/gateways/kafka/src/protocol/handlers/describe_configs.rs +++ b/gateways/kafka/src/protocol/handlers/describe_configs.rs @@ -17,8 +17,6 @@ //! `DescribeConfigs` (API key 32). -use std::collections::HashMap; - use bytes::Bytes; use iggy::prelude::TopicDetails; use kafka_protocol::messages::DescribeConfigsRequest; @@ -30,8 +28,8 @@ use kafka_protocol::messages::describe_configs_response::{ use kafka_protocol::protocol::StrBytes; use tokio::time::Instant; +use crate::bridge::IggyBridge; use crate::bridge::topic_map::validate_kafka_topic_name; -use crate::bridge::{BridgeError, IggyBridge}; use crate::protocol::api::{ API_KEY_DESCRIBE_CONFIGS, ApiVersionRange, ERROR_INVALID_CONFIG, ERROR_INVALID_REQUEST, ERROR_INVALID_TOPIC_EXCEPTION, ERROR_NONE, ERROR_NOT_CONTROLLER, ERROR_POLICY_VIOLATION, @@ -41,9 +39,10 @@ use crate::protocol::api::{ use crate::protocol::bounds_guard::validate_describe_configs_shape; use crate::protocol::handlers::topic_config::{ CLEANUP_DOC, CLEANUP_POLICY, CLEANUP_POLICY_VALUE, CONFIG_SOURCE_DEFAULT, CONFIG_SOURCE_TOPIC, - CONFIG_TYPE_LONG, CONFIG_TYPE_STRING, ListedKey, ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, - RETENTION_DOC, RETENTION_MS, UNKNOWN_TAGGED_FIELDS, exceeds_topic_cap, listed_keys, - message_expiry_is_explicit, retention_ms, topic_cap_message, + CONFIG_TYPE_LONG, CONFIG_TYPE_STRING, CONFIG_TYPE_UNKNOWN, ListedKey, ONLY_TOPIC_RESOURCES, + RESOURCE_TYPE_TOPIC, RETENTION_DOC, RETENTION_MS, bridge_failure, exceeds_topic_cap, + find_duplicate_names, listed_keys, message_expiry_is_explicit, name_reason, retention_ms, + static_text, topic_cap_message, }; use crate::protocol::handlers::{ decode_guarded, encode_message, is_supported_version, respond_or_close, @@ -76,17 +75,6 @@ pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> Hand } }; - if !req.unknown_tagged_fields.is_empty() { - return respond_or_close( - encode_message( - &response(reject_tagged_resources(&req.resources)), - api_version, - 256, - ), - "DescribeConfigs", - ); - } - let Some(bridge) = &state.bridge else { let results = req .resources @@ -133,12 +121,39 @@ async fn describe_all( req: &DescribeConfigsRequest, deadline: Instant, ) -> Vec { - let topic_names = req + if !req.unknown_tagged_fields.is_empty() { + let message = static_text("unknown tagged fields are not supported"); + return if req.resources.is_empty() { + vec![ + DescribeConfigsResult::default() + .with_error_code(ERROR_INVALID_REQUEST) + .with_error_message(Some(message)) + .with_resource_type(RESOURCE_TYPE_TOPIC) + .with_resource_name(StrBytes::from_static_str("")) + .with_configs(Vec::new()), + ] + } else { + req.resources + .iter() + .map(|resource| { + resource_error( + resource, + ERROR_INVALID_REQUEST, + Some(message.clone()), + Vec::new(), + ) + }) + .collect() + }; + } + + let topic_names: Vec<&str> = req .resources .iter() .filter(|resource| resource.resource_type == RESOURCE_TYPE_TOPIC) - .map(|resource| resource.resource_name.as_str()); - if exceeds_topic_cap(topic_names) { + .map(|resource| resource.resource_name.as_str()) + .collect(); + if exceeds_topic_cap(topic_names.iter().copied()) { let message = StrBytes::from(topic_cap_message("DescribeConfigs")); tracing::warn!("DescribeConfigs request addresses too many distinct topics; rejecting"); return req @@ -155,16 +170,23 @@ async fn describe_all( .collect(); } - // `get_kafka_topics` lists partition counts, not `message_expiry`, so each distinct - // topic is one `get_kafka_topic`. Duplicates reuse the first result and do not call again. - let mut seen: HashMap = HashMap::new(); + // Real Kafka refuses every occurrence of a duplicate resource name outright rather than + // answering it from a cached result that may have been filtered by a different + // `configuration_keys` list - the same choice CreateTopics makes for a repeated topic name. + let duplicate_names = find_duplicate_names(topic_names.iter().copied()); + let mut results = Vec::with_capacity(req.resources.len()); let mut deadline_exceeded = false; for resource in &req.resources { if resource.resource_type == RESOURCE_TYPE_TOPIC - && let Some(prior) = seen.get(resource.resource_name.as_str()) + && duplicate_names.contains(resource.resource_name.as_str()) { - results.push(prior.clone()); + results.push(resource_error( + resource, + ERROR_INVALID_REQUEST, + None, + Vec::new(), + )); continue; } if deadline_exceeded || Instant::now() >= deadline { @@ -175,9 +197,12 @@ async fn describe_all( instead of starting new Iggy calls" ); } - let timed_out = resource_error(resource, ERROR_REQUEST_TIMED_OUT, None, Vec::new()); - remember_topic(&mut seen, resource, &timed_out); - results.push(timed_out); + results.push(resource_error( + resource, + ERROR_REQUEST_TIMED_OUT, + None, + Vec::new(), + )); continue; } @@ -192,22 +217,11 @@ async fn describe_all( if Instant::now() >= deadline { deadline_exceeded = true; } - remember_topic(&mut seen, resource, &result); results.push(result); } results } -fn remember_topic( - seen: &mut HashMap, - resource: &DescribeConfigsResource, - result: &DescribeConfigsResult, -) { - if resource.resource_type == RESOURCE_TYPE_TOPIC { - seen.insert(resource.resource_name.as_str().to_string(), result.clone()); - } -} - async fn describe_one( bridge: &IggyBridge, resource: &DescribeConfigsResource, @@ -219,7 +233,7 @@ async fn describe_one( return resource_error( resource, ERROR_INVALID_REQUEST, - Some(static_text(UNKNOWN_TAGGED_FIELDS)), + Some(static_text("unknown tagged fields are not supported")), Vec::new(), ); } @@ -248,34 +262,49 @@ async fn describe_one( ); return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None, Vec::new()); } - let topic = match bridge.get_kafka_topic(name).await { - Ok(Some(topic)) => topic, - Ok(None) => { + let topic = match tokio::time::timeout_at(deadline, bridge.get_kafka_topic(name)).await { + Ok(Ok(Some(topic))) => topic, + Ok(Ok(None)) => { return resource_error(resource, ERROR_UNKNOWN_TOPIC_OR_PARTITION, None, Vec::new()); } - Err(error) => { - let (code, message) = bridge_failure(&error, "reading"); + Ok(Err(error)) => { + let (code, message) = bridge_failure(&error, "DescribeConfigs", "reading"); return resource_error(resource, code, message, Vec::new()); } + Err(_elapsed) => { + tracing::warn!( + resource = name, + "DescribeConfigs: this resource's bridge work exceeded the request deadline; \ + answering retriable instead of blocking further" + ); + return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None, Vec::new()); + } }; - let (configs, unknown, unrepresentable) = described_entries( + let (configs, unknown_key, unrepresentable) = described_entries( &topic, resource.configuration_keys.as_deref(), include_synonyms, include_documentation, ); - if unknown || unrepresentable { - let message = if unrepresentable { - "stored message expiry is not a whole number of milliseconds" - } else { - "unknown config key" - }; + if let Some(unknown_key) = unknown_key { return resource_error( resource, ERROR_INVALID_CONFIG, - Some(static_text(message)), + Some(StrBytes::from(format!( + "unknown config key '{unknown_key}'" + ))), + configs, + ); + } + if unrepresentable { + return resource_error( + resource, + ERROR_INVALID_CONFIG, + Some(static_text( + "stored message expiry is not a whole number of milliseconds", + )), configs, ); } @@ -288,11 +317,11 @@ fn described_entries( configuration_keys: Option<&[StrBytes]>, include_synonyms: bool, include_documentation: bool, -) -> (Vec, bool, bool) { +) -> (Vec, Option, bool) { let explicit = message_expiry_is_explicit(&topic.options); let retention = retention_ms(topic.message_expiry, explicit); let keys = listed_keys(configuration_keys); - let mut unknown = false; + let mut unknown_key = None; let mut unrepresentable = false; let mut configs = Vec::with_capacity(keys.len()); for key in keys { @@ -327,20 +356,22 @@ fn described_entries( include_documentation.then_some(CLEANUP_DOC), )), ListedKey::Unknown(name) => { - unknown = true; + if unknown_key.is_none() { + unknown_key = Some(name.clone()); + } configs.push(config_entry( &name, None, false, CONFIG_SOURCE_DEFAULT, - 0, + CONFIG_TYPE_UNKNOWN, include_synonyms, None, )); } } } - (configs, unknown, unrepresentable) + (configs, unknown_key, unrepresentable) } fn config_entry( @@ -363,33 +394,17 @@ fn config_entry( .with_documentation(documentation.map(|text| StrBytes::from(text.to_string()))) } -/// Neither known key has a synonym. `include_synonyms` asks for the list and receives none. -const fn synonyms(_include_synonyms: bool) -> Vec { - Vec::new() -} - -fn reject_tagged_resources(resources: &[DescribeConfigsResource]) -> Vec { - if resources.is_empty() { - return vec![ - DescribeConfigsResult::default() - .with_error_code(ERROR_INVALID_REQUEST) - .with_error_message(Some(static_text(UNKNOWN_TAGGED_FIELDS))) - .with_resource_type(0) - .with_resource_name(StrBytes::from_static_str("")) - .with_configs(Vec::new()), - ]; +/// Neither known key has a synonym. `true` asks for the list and receives none. +#[allow( + clippy::if_same_then_else, + reason = "true asks for synonyms and there are none" +)] +fn synonyms(include_synonyms: bool) -> Vec { + if include_synonyms { + Vec::new() + } else { + Vec::new() } - resources - .iter() - .map(|resource| { - resource_error( - resource, - ERROR_INVALID_REQUEST, - Some(static_text(UNKNOWN_TAGGED_FIELDS)), - Vec::new(), - ) - }) - .collect() } fn resource_error( @@ -405,35 +420,3 @@ fn resource_error( .with_resource_name(resource.resource_name.clone()) .with_configs(configs) } - -fn name_reason(error: &BridgeError) -> String { - match error { - BridgeError::InvalidKafkaTopicName { reason, .. } => reason.clone(), - _ => "invalid topic name".to_string(), - } -} - -fn bridge_failure(error: &BridgeError, action: &str) -> (i16, Option) { - let code = error.to_kafka_error_code(); - match error { - BridgeError::InvalidKafkaTopicName { reason, .. } => { - tracing::debug!(reason, "DescribeConfigs rejected an invalid topic name"); - (code, Some(StrBytes::from(reason.clone()))) - } - BridgeError::Timeout | BridgeError::SendLost(_) => { - tracing::warn!(%error, "DescribeConfigs {action} exceeded the bridge deadline"); - (code, None) - } - other => { - tracing::error!(%other, "DescribeConfigs failed while {action} topic configuration"); - ( - code, - Some(static_text("internal error reading topic configuration")), - ) - } - } -} - -const fn static_text(message: &'static str) -> StrBytes { - StrBytes::from_static_str(message) -} diff --git a/gateways/kafka/src/protocol/handlers/topic_config.rs b/gateways/kafka/src/protocol/handlers/topic_config.rs index 8760922820..1b6ccb56c0 100644 --- a/gateways/kafka/src/protocol/handlers/topic_config.rs +++ b/gateways/kafka/src/protocol/handlers/topic_config.rs @@ -30,6 +30,8 @@ use std::str::FromStr; use iggy::prelude::{HeaderKey, IggyDuration, IggyExpiry, ResourceOptions, topic_option_keys}; use kafka_protocol::protocol::StrBytes; +use crate::bridge::BridgeError; + /// Kafka `ConfigResource.Type.TOPIC`. pub const RESOURCE_TYPE_TOPIC: i8 = 2; @@ -45,12 +47,14 @@ pub const CONFIG_TYPE_STRING: i8 = 2; /// Kafka `ConfigType.LONG`. pub const CONFIG_TYPE_LONG: i8 = 5; +/// Kafka `ConfigType.UNKNOWN`. +pub const CONFIG_TYPE_UNKNOWN: i8 = 0; + pub const RETENTION_MS: &str = "retention.ms"; pub const CLEANUP_POLICY: &str = "cleanup.policy"; pub const CLEANUP_POLICY_VALUE: &str = "delete"; pub const ONLY_TOPIC_RESOURCES: &str = "only topic resources are supported"; -pub const UNKNOWN_TAGGED_FIELDS: &str = "unknown tagged fields are not supported"; pub const RETENTION_DOC: &str = "How long a sealed segment is kept, in milliseconds. The active segment does not expire. -1 means sealed segments never expire."; pub const CLEANUP_DOC: &str = "Iggy deletes expired messages and does not compact a topic."; @@ -58,8 +62,10 @@ pub const CLEANUP_DOC: &str = "Iggy deletes expired messages and does not compac /// Distinct Kafka topic names one `DescribeConfigs` or `AlterConfigs` request may send to Iggy. /// /// Same ceiling as `CreateTopics`: each distinct name is a lockstep call on the shared client. -/// Duplicate names in one request count once. A larger batch is `POLICY_VIOLATION` on every -/// resource. The decode-time element ceiling is a separate, larger limit and is not this cap. +/// Duplicate names in one request count once for this cap, but a repeated name is itself +/// rejected outright - see [`find_duplicate_names`]. A larger batch is `POLICY_VIOLATION` on +/// every resource. The decode-time element ceiling is a separate, larger limit and is not this +/// cap. pub const MAX_CONFIG_TOPICS: usize = 100; /// One config name a describe request asked for, in request order. @@ -139,12 +145,8 @@ pub fn retention_ms(expiry: IggyExpiry, explicit: bool) -> Result Result { return Err(ConfigFault::InvalidRetention); } let millis = u64::try_from(millis).map_err(|_| ConfigFault::InvalidRetention)?; - // `IggyExpiry::from_str` rejects `as_secs() > u32::MAX`. A longer duration makes - // `Segment::is_expired` wrap `max_timestamp + micros` (and panic in debug), so a - // sealed segment looks expired and is deleted. + // `IggyExpiry::from_str` rejects `as_secs() > u32::MAX`. The typed update path + // does not, but a value the server's own parser refuses is not one this gateway + // should store. if millis / 1_000 > u64::from(u32::MAX) { return Err(ConfigFault::InvalidRetention); } @@ -203,6 +205,67 @@ pub fn topic_cap_message(api_name: &str) -> String { ) } +/// Names in `names` that occur more than once. +/// +/// Real Kafka refuses every occurrence of a duplicate resource name with `INVALID_REQUEST` +/// (42) rather than silently picking a winner - the same choice `CreateTopics`' own +/// `find_duplicate_names` makes for a repeated topic name in one batch. +#[must_use] +pub fn find_duplicate_names<'a>(names: impl IntoIterator) -> HashSet<&'a str> { + let mut seen = HashSet::new(); + let mut duplicates = HashSet::new(); + for name in names { + if !seen.insert(name) { + duplicates.insert(name); + } + } + duplicates +} + +/// The reason a topic name failed Kafka's own naming rules, or a fixed fallback. +#[must_use] +pub fn name_reason(error: &BridgeError) -> String { + match error { + BridgeError::InvalidKafkaTopicName { reason, .. } => reason.clone(), + _ => "invalid topic name".to_string(), + } +} + +#[must_use] +pub const fn static_text(message: &'static str) -> StrBytes { + StrBytes::from_static_str(message) +} + +/// Maps a bridge failure to a Kafka error code and client-facing message, logging the real +/// cause server-side. +/// +/// `handler` names the API in the log line (`"DescribeConfigs"`/`"AlterConfigs"`); `action` is +/// the present participle of what the bridge call was doing (`"reading"`/`"altering"`), reused +/// in both the timeout log and the generic internal-error message. +#[must_use] +pub fn bridge_failure(error: &BridgeError, handler: &str, action: &str) -> (i16, Option) { + let code = error.to_kafka_error_code(); + match error { + BridgeError::InvalidKafkaTopicName { reason, .. } => { + tracing::debug!(reason, "{handler} rejected an invalid topic name"); + (code, Some(StrBytes::from(reason.clone()))) + } + BridgeError::Timeout | BridgeError::SendLost(_) => { + tracing::warn!(%error, "{handler} {action} exceeded the bridge deadline"); + (code, None) + } + other => { + tracing::error!(%other, "{handler} failed while {action} topic configuration"); + ( + code, + Some(StrBytes::from(format!( + "internal error {action} topic configuration" + ))), + ) + } + } +} + /// Keys to return for one describe resource. /// /// `None` and an empty list both mean both known keys. Any other name is @@ -327,6 +390,17 @@ mod tests { ); } + #[test] + fn find_duplicate_names_finds_a_name_repeated_across_two_resources() { + let duplicates = find_duplicate_names(["orders", "payments", "orders"]); + assert_eq!(duplicates, HashSet::from(["orders"])); + } + + #[test] + fn find_duplicate_names_is_empty_when_every_name_is_unique() { + assert!(find_duplicate_names(["orders", "payments"]).is_empty()); + } + #[test] fn one_bad_key_rejects_the_whole_alter_plan() { let err = plan_retention_update([(RETENTION_MS, Some("5000")), ("no.such", Some("1"))]) diff --git a/gateways/kafka/tests/common/wire.rs b/gateways/kafka/tests/common/wire.rs index 1605f18856..b9dd3c6bbe 100644 --- a/gateways/kafka/tests/common/wire.rs +++ b/gateways/kafka/tests/common/wire.rs @@ -26,10 +26,24 @@ //! every function some *other* binary uses as unused here. #![allow(dead_code)] -use bytes::Bytes; +use bytes::{Bytes, BytesMut}; +use kafka_protocol::protocol::{Decodable, Encodable}; use super::codec::Encoder; +/// Encodes any `kafka_protocol` request or response type at `version`. +pub fn encode(message: &M, version: i16) -> Bytes { + let mut buf = BytesMut::new(); + message.encode(&mut buf, version).expect("encode request"); + buf.freeze() +} + +/// Decodes any `kafka_protocol` request or response type at `version`. +pub fn decode(body: Bytes, version: i16) -> M { + let mut buf = body; + M::decode(&mut buf, version).expect("decode response") +} + /// Consumer-group and admin keys explicitly out of scope in SCOPE.md. pub const OUT_OF_SCOPE_API_KEYS: &[(i16, &str)] = &[ (8, "OffsetCommit"), diff --git a/gateways/kafka/tests/configs_handler_tests.rs b/gateways/kafka/tests/configs_handler_tests.rs index ede44539b5..4bf8c9346b 100644 --- a/gateways/kafka/tests/configs_handler_tests.rs +++ b/gateways/kafka/tests/configs_handler_tests.rs @@ -17,7 +17,6 @@ //! `DescribeConfigs` and `AlterConfigs` with the bridge off. -use bytes::{Bytes, BytesMut}; use kafka_protocol::messages::alter_configs_request::{ AlterConfigsRequest, AlterConfigsResource, AlterableConfig, }; @@ -26,13 +25,20 @@ use kafka_protocol::messages::describe_configs_request::{ DescribeConfigsRequest, DescribeConfigsResource, }; use kafka_protocol::messages::describe_configs_response::DescribeConfigsResponse; -use kafka_protocol::protocol::{Decodable, Encodable, StrBytes}; +use kafka_protocol::protocol::StrBytes; use tokio_util::sync::CancellationToken; use iggy_gateway_kafka::group::{GroupCoordinator, GroupCoordinatorConfig}; use iggy_gateway_kafka::protocol::api::{BrokerAdvertise, ERROR_NOT_CONTROLLER, GatewayState}; use iggy_gateway_kafka::protocol::handlers::{alter_configs, describe_configs}; +#[path = "common/codec.rs"] +mod codec; +#[path = "common/wire.rs"] +mod wire; + +use wire::{decode, encode}; + const DESCRIBE_VERSION: i16 = 4; const ALTER_VERSION: i16 = 2; @@ -47,17 +53,6 @@ fn state_without_bridge() -> GatewayState { ) } -fn encode(message: &M, version: i16) -> Bytes { - let mut buf = BytesMut::new(); - message.encode(&mut buf, version).expect("encode request"); - buf.freeze() -} - -fn decode(body: Bytes, version: i16) -> M { - let mut buf = body; - M::decode(&mut buf, version).expect("decode response") -} - #[tokio::test] async fn describe_configs_without_a_bridge_returns_not_controller() { let request = DescribeConfigsRequest::default() diff --git a/gateways/kafka/tests/configs_java_client_tests.rs b/gateways/kafka/tests/configs_java_client_tests.rs index 4dc5905089..1fde0a4eca 100644 --- a/gateways/kafka/tests/configs_java_client_tests.rs +++ b/gateways/kafka/tests/configs_java_client_tests.rs @@ -46,6 +46,34 @@ const TOPIC: &str = "orders"; /// process until the runner itself was killed. const CHILD_LIMIT: Duration = Duration::from_secs(120); +/// Reports why the suite cannot run, and whether that is fatal. +/// +/// Returns `true` when the caller should skip. `JAVA_E2E_REQUIRED=1` makes it panic instead, so a +/// CI job that means to run this fails loudly rather than reporting a pass over zero assertions - +/// same convention as `kafka_client_e2e_tests.rs`'s own `skip`/`KAFKA_E2E_REQUIRED`. +fn skip(reason: &str) -> bool { + assert!( + std::env::var("JAVA_E2E_REQUIRED").as_deref() != Ok("1"), + "JAVA_E2E_REQUIRED=1 but the suite cannot run: {reason}" + ); + eprintln!("skipping Java admin-client end-to-end test: {reason}"); + true +} + +fn java_missing() -> bool { + let available = Command::new("java") + .arg("-version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()); + if available { + false + } else { + skip("java is unavailable") + } +} + fn output_bounded(command: &mut Command, what: &str) -> Output { let mut child = command .stdout(Stdio::piped()) @@ -188,12 +216,9 @@ async fn spawn_gateway(server: &TestServer) -> SocketAddr { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn kafka_39_admin_client_describes_and_alters_topic_configs() { - let java = output_bounded(Command::new("java").arg("-version"), "java -version"); - assert!( - java.status.success(), - "java -version failed: {}", - String::from_utf8_lossy(&java.stderr) - ); + if java_missing() { + return; + } let jars = jar_dir(); ensure_jars(&jars); diff --git a/gateways/kafka/tests/configs_real_bridge_tests.rs b/gateways/kafka/tests/configs_real_bridge_tests.rs index 933a85a9f3..8893c6ee7e 100644 --- a/gateways/kafka/tests/configs_real_bridge_tests.rs +++ b/gateways/kafka/tests/configs_real_bridge_tests.rs @@ -19,7 +19,6 @@ use std::sync::Arc; -use bytes::{Bytes, BytesMut}; use kafka_protocol::messages::alter_configs_request::{ AlterConfigsRequest, AlterConfigsResource, AlterableConfig, }; @@ -30,7 +29,7 @@ use kafka_protocol::messages::describe_configs_request::{ use kafka_protocol::messages::describe_configs_response::{ DescribeConfigsResourceResult, DescribeConfigsResponse, }; -use kafka_protocol::protocol::{Decodable, Encodable, StrBytes}; +use kafka_protocol::protocol::StrBytes; use serial_test::serial; use tokio_util::sync::CancellationToken; @@ -42,10 +41,15 @@ use iggy_gateway_kafka::protocol::api::{ }; use iggy_gateway_kafka::protocol::handlers::{alter_configs, describe_configs}; +#[path = "common/codec.rs"] +mod codec; #[path = "common/iggy_server.rs"] mod iggy_server; +#[path = "common/wire.rs"] +mod wire; use iggy_server::TestServer; +use wire::{decode, encode}; const DESCRIBE_V4: i16 = 4; const DESCRIBE_V1: i16 = 1; @@ -59,17 +63,6 @@ const fn text(value: &'static str) -> StrBytes { StrBytes::from_static_str(value) } -fn encode(message: &M, version: i16) -> Bytes { - let mut buf = BytesMut::new(); - message.encode(&mut buf, version).expect("encode request"); - buf.freeze() -} - -fn decode(body: Bytes, version: i16) -> M { - let mut buf = body; - M::decode(&mut buf, version).expect("decode response") -} - async fn connected_state(server: &TestServer) -> GatewayState { let bridge = IggyBridge::connect(server.test_config()) .await @@ -318,7 +311,10 @@ async fn describe_and_alter_configs_reject_unknown_nontopic_missing_and_invalid_ .error_message .as_ref() .map_or("", StrBytes::as_str); - assert!(!message.contains("no.such")); + assert!( + message.contains("no.such"), + "unknown-key message must name the key, matching AlterConfigs' own wording: {message}" + ); let broker = describe( &state, @@ -519,3 +515,120 @@ async fn alter_configs_one_bad_key_skips_only_that_resource() { ); assert_eq!(described.results[1].configs[0].config_source, SOURCE_TOPIC); } + +/// Regression test for the duplicate-resource memoization bug: a second `AlterConfigs` +/// resource entry for an already-seen topic name must not be silently answered from a cached +/// result - its own, different `retention.ms` must never be reported as applied when the write +/// for it never ran. +#[tokio::test] +#[serial] +async fn alter_configs_rejects_every_occurrence_of_a_duplicate_resource_name() { + let data_dir = tempfile::tempdir().expect("tempdir"); + let server = TestServer::spawn(data_dir.path()).await; + let state = connected_state(&server).await; + create_topic(&state, "orders").await; + + let altered = alter( + &state, + ALTER_V2, + vec![ + alter_resource("orders", TOPIC, &[("retention.ms", "4000")]), + alter_resource("orders", TOPIC, &[("retention.ms", "5000")]), + ], + false, + ) + .await; + assert_eq!(altered.responses.len(), 2); + assert_eq!(altered.responses[0].error_code, ERROR_INVALID_REQUEST); + assert_eq!(altered.responses[1].error_code, ERROR_INVALID_REQUEST); + + // Neither occurrence's value was applied - the stored retention is still the default. + let described = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("orders", Some(vec!["retention.ms"]))], + false, + false, + ) + .await; + assert_eq!( + described.results[0].configs[0] + .value + .as_ref() + .map(StrBytes::as_str), + Some("-1") + ); +} + +/// The `DescribeConfigs` sibling of the same memoization pattern: a duplicate resource name +/// with a *different* `configuration_keys` filter must not be answered from the first entry's +/// cached, differently-filtered result. +#[tokio::test] +#[serial] +async fn describe_configs_rejects_every_occurrence_of_a_duplicate_resource_name() { + let data_dir = tempfile::tempdir().expect("tempdir"); + let server = TestServer::spawn(data_dir.path()).await; + let state = connected_state(&server).await; + create_topic(&state, "orders").await; + + let described = describe( + &state, + DESCRIBE_V4, + vec![ + describe_resource("orders", Some(vec!["retention.ms"])), + describe_resource("orders", Some(vec!["cleanup.policy"])), + ], + false, + false, + ) + .await; + assert_eq!(described.results.len(), 2); + assert_eq!(described.results[0].error_code, ERROR_INVALID_REQUEST); + assert_eq!(described.results[1].error_code, ERROR_INVALID_REQUEST); +} + +/// An alter that does not name `retention.ms` does not clear a previously set expiry. +#[tokio::test] +#[serial] +async fn alter_configs_with_no_configs_leaves_retention_unchanged() { + let data_dir = tempfile::tempdir().expect("tempdir"); + let server = TestServer::spawn(data_dir.path()).await; + let state = connected_state(&server).await; + create_topic(&state, "orders").await; + + let first = alter( + &state, + ALTER_V2, + vec![alter_resource("orders", TOPIC, &[("retention.ms", "4000")])], + false, + ) + .await; + assert_eq!(first.responses[0].error_code, ERROR_NONE); + + let reset = alter( + &state, + ALTER_V2, + vec![alter_resource("orders", TOPIC, &[])], + false, + ) + .await; + assert_eq!(reset.responses[0].error_code, ERROR_NONE); + + let described = describe( + &state, + DESCRIBE_V4, + vec![describe_resource("orders", Some(vec!["retention.ms"]))], + false, + false, + ) + .await; + assert_eq!( + described.results[0].configs[0] + .value + .as_ref() + .map(StrBytes::as_str), + Some("4000"), + "an empty configs list must not clear a previously set retention.ms" + ); + assert_eq!(described.results[0].configs[0].config_source, SOURCE_TOPIC); +} From e8300074b6a16a8b016cdcaffe54cebec42428c9 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 4 Oct 2026 09:51:13 -0400 Subject: [PATCH 04/13] feat(gateways): accept retention.minutes and retention.hours Milliseconds stay the value stored in Iggy. Those names are remembered as synonyms only when an alter uses them. --- gateways/kafka/docs/CONFIGS.md | 99 +++-- gateways/kafka/src/bridge/iggy_bridge/mod.rs | 10 + gateways/kafka/src/protocol/api.rs | 26 +- .../src/protocol/handlers/alter_configs.rs | 29 +- .../src/protocol/handlers/describe_configs.rs | 315 +++++++++++++-- .../src/protocol/handlers/topic_config.rs | 359 ++++++++++++++++-- gateways/kafka/tests/configs_handler_tests.rs | 24 +- 7 files changed, 771 insertions(+), 91 deletions(-) diff --git a/gateways/kafka/docs/CONFIGS.md b/gateways/kafka/docs/CONFIGS.md index 4acf0a942e..f8a93c7e5e 100644 --- a/gateways/kafka/docs/CONFIGS.md +++ b/gateways/kafka/docs/CONFIGS.md @@ -2,8 +2,9 @@ `DescribeConfigs` (32) and `AlterConfigs` (33) apply to Kafka topic resources only (resource type 2). Only the deprecated `Admin.alterConfigs` (API 33) writes -`retention.ms`. `IncrementalAlterConfigs` (44) stays unimplemented: it is not -advertised and closes the connection like every other unlisted key. +retention. `IncrementalAlterConfigs` (44) is not in this change. It is planned +after this work merges. It is not advertised, and a client that sends 44 still +gets the connection closed. Only `AlterConfigs` (33) writes retention. With the bridge off, both APIs answer `NOT_CONTROLLER` (41) for each resource and do not read or write Iggy. @@ -20,9 +21,24 @@ not know is still returned, and that resource's error code is `INVALID_CONFIG` | `retention.ms` | Sealed-segment lifetime from Iggy `message_expiry`, or `-1` when those segments never expire | no | `DEFAULT_CONFIG` (5) for the unset never-expire default; `DYNAMIC_TOPIC_CONFIG` (1) when a client set it or a stored duration cannot be shown as milliseconds | | `cleanup.policy` | `delete` | yes | `DEFAULT_CONFIG` (5) | -`is_sensitive` is false for both. `include_synonyms` is honored and the synonym -list is empty either way. From v3, `include_documentation` is honored: true -attaches one sentence per known key, false sends no documentation. +`is_sensitive` is false for both. From v3, `include_documentation` is honored: +true attaches one sentence per known key, false sends no documentation. + +`retention.minutes` and `retention.hours` are not listed unless the client names +them in `configuration_keys`. A request that names one of those keys gets that +name back, with the value derived from the stored millisecond count (integer +division toward zero, and `-1` stays `-1`). The stored source of truth is still +`retention.ms`. Naming either key is not `INVALID_CONFIG`. + +`include_synonyms` is honored. The synonym list on `retention.ms` is empty when +the flag is false, or when this process does not remember `retention.minutes` or +`retention.hours` for that topic. When the flag is true and an alter on this +process used one or both of those names, the list has one entry per remembered +name. The synonym value is the stored millisecond count divided by 60000 or +3600000, truncating toward zero. `-1` stays `-1`. `cleanup.policy` has no +synonyms. A null or empty `configuration_keys` list still returns only +`retention.ms` and `cleanup.policy`; minutes and hours show up there only as +synonyms. Iggy stores `message_expiry` as `IggyExpiry`. A duration is `IggyDuration`, and that type counts microseconds. Retention applies to sealed segments only. Iggy @@ -34,29 +50,68 @@ the never-expire default. The gateway does not round. ## What an alter stores -Only `retention.ms` is written, through `update_topic`'s `message_expiry`. A resource -whose `configs` list does not name `retention.ms` is left unchanged. Omitting the key -does not clear a previously set expiry. +`retention.ms`, `retention.minutes`, and `retention.hours` are one writable +duration. Iggy stores milliseconds, through `update_topic`'s `message_expiry`. +Minutes and hours are not stored. -- `-1` stores `IggyExpiry::NeverExpire`. The server treats `0` - (`IggyExpiry::ServerDefault`) as "leave the current expiry", so `-1` is not - sent as 0. A following describe reports `-1` with source +- `retention.ms` is the source of truth. +- `retention.minutes` converts as milliseconds = minutes * 60000. A later + describe of the synonym divides the stored milliseconds by 60000. +- `retention.hours` converts as milliseconds = hours * 3600000. A later describe + of the synonym divides the stored milliseconds by 3600000. +- `-1` on any of the three stores `IggyExpiry::NeverExpire`. The server treats + `0` (`IggyExpiry::ServerDefault`) as "leave the current expiry", so `-1` is + not sent as 0. A following describe reports `-1` with source + `DYNAMIC_TOPIC_CONFIG`. +- A positive count must be the canonical decimal of that integer (`1`, not + `0001` or `+1`). The gateway rejects a product that overflows before it + multiplies into milliseconds. The resulting duration must be at most + `u32::MAX` seconds, the same limit `IggyExpiry::from_str` enforces. A longer + value is `INVALID_CONFIG` (40) and is not stored. A following describe of + `retention.ms` reports that millisecond count with source `DYNAMIC_TOPIC_CONFIG`. -- A positive millisecond count is multiplied by 1000 and stored as - `IggyExpiry::ExpireDuration`. The text must be the canonical decimal of that - integer (`1`, not `0001` or `+1`). The duration must be at most `u32::MAX` - seconds, the same limit `IggyExpiry::from_str` enforces. A longer value is - `INVALID_CONFIG` (40) and is not stored. A following describe reports the same - decimal with source `DYNAMIC_TOPIC_CONFIG`. - `0` and every other value are `INVALID_CONFIG` (40). -`cleanup.policy` and any other key are `INVALID_CONFIG` (40). They are not -stored. Kafka has no `NOT_CONFIGURABLE` code. One bad key fails that resource -and writes nothing for it. Other resources in the same request are still -processed. +If one resource sets more than one of the three names and the converted +millisecond values differ, that resource is `INVALID_CONFIG` (40) and nothing +is written. If they convert to the same millisecond count, one write is stored. +A resource whose `configs` list does not name any of the three is left +unchanged. Omitting them does not clear a previously set expiry. + +`cleanup.policy` and any other key, including `retention.mins`, are +`INVALID_CONFIG` (40). They are not stored. Kafka has no `NOT_CONFIGURABLE` +code. One bad key fails that resource and writes nothing for it. Other +resources in the same request are still processed. `validate_only` runs the same checks, including the topic lookup, and does not -call `update_topic`. +call `update_topic`. It also does not change which synonym names are remembered. + +## Synonym memory + +When an alter successfully applies and the request used `retention.minutes` +and/or `retention.hours`, the gateway remembers those names for that topic. +When an alter successfully applies and it used only `retention.ms`, the +remembered names for that topic are cleared. A failed alter does not change +them. A resource that does not store an expiry does not change them either. + +The key is the Iggy stream and topic the gateway already resolves from the +Kafka topic name, not the Kafka name itself. + +## Limitations + +- `IncrementalAlterConfigs` (API 44) is not implemented here. It is planned + after this work merges. A client that sends 44 still gets the connection + closed. Only `AlterConfigs` (33) writes retention. +- Synonym memory is process-local. It is lost on gateway restart and is not + shared across gateway processes. After a restart the synonym list is empty + until the next alter that uses minutes or hours. +- The memory is not cleared when a topic is deleted outside this gateway + (`DeleteTopics` is not in this branch). A later recreate can show a stale + synonym until the next alter. +- Synonym display truncates toward zero. A stored 90000 ms remembered as + minutes is reported as 1 minute on the synonym, while `retention.ms` still + shows 90000. +- `retention.ms` remains the only value Iggy stores. ## Other resource errors diff --git a/gateways/kafka/src/bridge/iggy_bridge/mod.rs b/gateways/kafka/src/bridge/iggy_bridge/mod.rs index 0d95e60026..7d6bf5a1e6 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/mod.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/mod.rs @@ -174,6 +174,16 @@ impl IggyBridge { }) } + /// Iggy stream and topic names `kafka_topic` resolves to. + /// + /// Callers have already checked the Kafka name. Retention synonym memory is + /// keyed by this pair. + #[must_use] + pub(crate) fn topic_identity(&self, kafka_topic: &str) -> (String, String) { + let (stream, topic) = self.config.topic_mapping.resolve(kafka_topic); + (stream.to_string(), topic.to_string()) + } + /// Tears down the underlying Iggy client, including its background heartbeat task. /// /// Not `IggyClient::disconnect`: that only tears down the transport diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 850a15928d..4ad4e46dae 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use bytes::Bytes; @@ -37,6 +37,9 @@ use crate::protocol::bounds_guard::{ validate_describe_acls_shape, validate_sasl_authenticate_shape, validate_sasl_handshake_shape, }; use crate::protocol::handlers::init_producer_id::ProducerIdAllocator; +use crate::protocol::handlers::topic_config::{ + IggyTopicKey, RetentionSynonymMemory, RetentionSynonyms, +}; use crate::protocol::handlers::{ alter_configs, api_versions, create_topics, decode_guarded, describe_configs, dispatch, fetch, find_coordinator, heartbeat, init_producer_id, join_group, leave_group, list_offsets, metadata, @@ -145,7 +148,8 @@ pub const ERROR_INVALID_REQUEST: i16 = 42; /// A config this gateway will not apply. /// /// `CreateTopics` rejects any per-topic config list and points the client at `AlterConfigs`. -/// `AlterConfigs` stores `retention.ms` and rejects every other key, and a value that cannot +/// `AlterConfigs` stores `retention.ms`, `retention.minutes`, or `retention.hours` as +/// `message_expiry` in milliseconds and rejects every other key, and a value that cannot /// be stored. `DescribeConfigs` sends this when a requested key is unknown or a stored expiry /// cannot be shown as `retention.ms`. pub const ERROR_INVALID_CONFIG: i16 = 40; @@ -315,6 +319,9 @@ pub struct GatewayState { /// Consumer group membership. Process-wide and independent of the bridge: a member outlives /// the connection that created it, and group coordination needs no Iggy call. pub groups: GroupCoordinator, + /// `retention.minutes` and `retention.hours` from the last successful alter on this + /// process, keyed by Iggy stream and topic. Not stored in Iggy. + pub(crate) retention_synonym_memory: Mutex, /// Per Kafka topic and partition, how long probes read it as loading. Fetch and `ListOffsets` /// share it, since every consumer sees the same probe. pub(crate) loading: Mutex, @@ -344,6 +351,7 @@ impl GatewayState { producer_ids: ProducerIdAllocator::new(instance_id), produce_slots: Semaphore::const_new(PRODUCE_SLOTS), groups, + retention_synonym_memory: Mutex::default(), loading: Mutex::default(), probe_board: ProbeBoard::default(), } @@ -363,6 +371,20 @@ impl GatewayState { GroupCoordinator::new(GroupCoordinatorConfig::default(), CancellationToken::new()), ) } + + pub(crate) fn remembered_retention_synonyms(&self, key: &IggyTopicKey) -> RetentionSynonyms { + self.retention_synonym_memory + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remembered(key) + } + + pub(crate) fn record_retention_synonyms(&self, key: IggyTopicKey, synonyms: RetentionSynonyms) { + self.retention_synonym_memory + .lock() + .unwrap_or_else(PoisonError::into_inner) + .record_applied(key, synonyms); + } } /// What a handler keeps for one Kafka connection, across its requests. diff --git a/gateways/kafka/src/protocol/handlers/alter_configs.rs b/gateways/kafka/src/protocol/handlers/alter_configs.rs index 8da1671b3d..9298ef1d56 100644 --- a/gateways/kafka/src/protocol/handlers/alter_configs.rs +++ b/gateways/kafka/src/protocol/handlers/alter_configs.rs @@ -37,7 +37,7 @@ use crate::protocol::api::{ }; use crate::protocol::bounds_guard::validate_alter_configs_shape; use crate::protocol::handlers::topic_config::{ - ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, bridge_failure, exceeds_topic_cap, + IggyTopicKey, ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, bridge_failure, exceeds_topic_cap, find_duplicate_names, name_reason, plan_retention_update, static_text, topic_cap_message, }; use crate::protocol::handlers::{ @@ -84,7 +84,7 @@ pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> Hand }; let deadline = Instant::now() + REQUEST_DEADLINE; - let responses = alter_all(bridge, &req, deadline).await; + let responses = alter_all(state, bridge, &req, deadline).await; respond_or_close( encode_message(&response(responses), api_version, 256), "AlterConfigs", @@ -112,6 +112,7 @@ fn response(responses: Vec) -> AlterConfigsRespons } async fn alter_all( + state: &GatewayState, bridge: &IggyBridge, req: &AlterConfigsRequest, deadline: Instant, @@ -177,7 +178,7 @@ async fn alter_all( continue; } - let result = alter_one(bridge, resource, req.validate_only, deadline).await; + let result = alter_one(state, bridge, resource, req.validate_only, deadline).await; if Instant::now() >= deadline { deadline_exceeded = true; } @@ -187,6 +188,7 @@ async fn alter_all( } async fn alter_one( + state: &GatewayState, bridge: &IggyBridge, resource: &AlterConfigsResource, validate_only: bool, @@ -249,13 +251,13 @@ async fn alter_one( } } - let expiry = match plan_retention_update(resource.configs.iter().map(|config| { + let planned = match plan_retention_update(resource.configs.iter().map(|config| { ( config.name.as_str(), config.value.as_ref().map(StrBytes::as_str), ) })) { - Ok(expiry) => expiry, + Ok(planned) => planned, Err(fault) => { return resource_error( resource, @@ -264,14 +266,14 @@ async fn alter_one( ); } }; - // A resource that does not name retention.ms has nothing to store. Omitting the key - // does not clear a previously set expiry. - let Some(expiry) = expiry else { + // A resource that names no retention key has nothing to store. Omitting the key + // does not clear a previously set expiry or the remembered synonym names. + let Some(planned) = planned else { return resource_error(resource, ERROR_NONE, None); }; // `validate_only` already performed the existence read and the same key checks. - // It does not call `update_topic`. + // It does not call `update_topic` and does not change synonym memory. if validate_only { return resource_error(resource, ERROR_NONE, None); } @@ -287,8 +289,15 @@ async fn alter_one( ); return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None); } - match bridge.update_kafka_topic_message_expiry(name, expiry).await { + match bridge + .update_kafka_topic_message_expiry(name, planned.expiry) + .await + { Ok(()) | Err(BridgeError::Iggy(IggyError::RequestAlreadyApplied)) => { + // Keyed by the mapped Iggy stream and topic, not the Kafka name. + let (stream, iggy_topic) = bridge.topic_identity(name); + state + .record_retention_synonyms(IggyTopicKey::new(stream, iggy_topic), planned.synonyms); resource_error(resource, ERROR_NONE, None) } Err(error) => { diff --git a/gateways/kafka/src/protocol/handlers/describe_configs.rs b/gateways/kafka/src/protocol/handlers/describe_configs.rs index 200261443e..807a029171 100644 --- a/gateways/kafka/src/protocol/handlers/describe_configs.rs +++ b/gateways/kafka/src/protocol/handlers/describe_configs.rs @@ -39,9 +39,11 @@ use crate::protocol::api::{ use crate::protocol::bounds_guard::validate_describe_configs_shape; use crate::protocol::handlers::topic_config::{ CLEANUP_DOC, CLEANUP_POLICY, CLEANUP_POLICY_VALUE, CONFIG_SOURCE_DEFAULT, CONFIG_SOURCE_TOPIC, - CONFIG_TYPE_LONG, CONFIG_TYPE_STRING, CONFIG_TYPE_UNKNOWN, ListedKey, ONLY_TOPIC_RESOURCES, - RESOURCE_TYPE_TOPIC, RETENTION_DOC, RETENTION_MS, bridge_failure, exceeds_topic_cap, - find_duplicate_names, listed_keys, message_expiry_is_explicit, name_reason, retention_ms, + CONFIG_TYPE_LONG, CONFIG_TYPE_STRING, CONFIG_TYPE_UNKNOWN, IggyTopicKey, ListedKey, + MILLIS_PER_HOUR, MILLIS_PER_MINUTE, ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, RETENTION_DOC, + RETENTION_HOURS, RETENTION_HOURS_DOC, RETENTION_MINUTES, RETENTION_MINUTES_DOC, RETENTION_MS, + RetentionMs, RetentionSynonyms, bridge_failure, exceeds_topic_cap, find_duplicate_names, + listed_keys, message_expiry_is_explicit, name_reason, retention_in_unit, retention_ms, static_text, topic_cap_message, }; use crate::protocol::handlers::{ @@ -88,7 +90,7 @@ pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> Hand }; let deadline = Instant::now() + REQUEST_DEADLINE; - let results = describe_all(bridge, &req, deadline).await; + let results = describe_all(state, bridge, &req, deadline).await; respond_or_close( encode_message(&response(results), api_version, 256), "DescribeConfigs", @@ -117,6 +119,7 @@ fn response(results: Vec) -> DescribeConfigsResponse { } async fn describe_all( + state: &GatewayState, bridge: &IggyBridge, req: &DescribeConfigsRequest, deadline: Instant, @@ -207,6 +210,7 @@ async fn describe_all( } let result = describe_one( + state, bridge, resource, req.include_synonyms, @@ -223,6 +227,7 @@ async fn describe_all( } async fn describe_one( + state: &GatewayState, bridge: &IggyBridge, resource: &DescribeConfigsResource, include_synonyms: bool, @@ -281,11 +286,14 @@ async fn describe_one( } }; + let (stream, iggy_topic) = bridge.topic_identity(name); + let remembered = state.remembered_retention_synonyms(&IggyTopicKey::new(stream, iggy_topic)); let (configs, unknown_key, unrepresentable) = described_entries( &topic, resource.configuration_keys.as_deref(), include_synonyms, include_documentation, + remembered, ); if let Some(unknown_key) = unknown_key { @@ -317,6 +325,7 @@ fn described_entries( configuration_keys: Option<&[StrBytes]>, include_synonyms: bool, include_documentation: bool, + remembered: RetentionSynonyms, ) -> (Vec, Option, bool) { let explicit = message_expiry_is_explicit(&topic.options); let retention = retention_ms(topic.message_expiry, explicit); @@ -327,32 +336,38 @@ fn described_entries( for key in keys { match key { ListedKey::Retention => { - let (value, source) = retention.as_ref().map_or_else( - |()| { - // A stored duration that cannot be shown as retention.ms is still the - // topic's expiry, not the never-expire default. - unrepresentable = true; - (None, CONFIG_SOURCE_TOPIC) - }, - |retention| (Some(retention.value.as_str()), retention.source), - ); + let (value, source) = retention_parts(&retention, &mut unrepresentable); configs.push(config_entry( RETENTION_MS, value, false, source, CONFIG_TYPE_LONG, - include_synonyms, + synonyms(include_synonyms, remembered, value, source), include_documentation.then_some(RETENTION_DOC), )); } + ListedKey::Minutes => configs.push(derived_retention_entry( + RETENTION_MINUTES, + MILLIS_PER_MINUTE, + &retention, + include_documentation.then_some(RETENTION_MINUTES_DOC), + &mut unrepresentable, + )), + ListedKey::Hours => configs.push(derived_retention_entry( + RETENTION_HOURS, + MILLIS_PER_HOUR, + &retention, + include_documentation.then_some(RETENTION_HOURS_DOC), + &mut unrepresentable, + )), ListedKey::Cleanup => configs.push(config_entry( CLEANUP_POLICY, Some(CLEANUP_POLICY_VALUE), true, CONFIG_SOURCE_DEFAULT, CONFIG_TYPE_STRING, - include_synonyms, + Vec::new(), include_documentation.then_some(CLEANUP_DOC), )), ListedKey::Unknown(name) => { @@ -365,7 +380,7 @@ fn described_entries( false, CONFIG_SOURCE_DEFAULT, CONFIG_TYPE_UNKNOWN, - include_synonyms, + Vec::new(), None, )); } @@ -374,13 +389,47 @@ fn described_entries( (configs, unknown_key, unrepresentable) } +const fn retention_parts<'a>( + retention: &'a Result, + unrepresentable: &mut bool, +) -> (Option<&'a str>, i8) { + if let Ok(retention) = retention { + (Some(retention.value.as_str()), retention.source) + } else { + // A stored duration that cannot be shown as retention.ms is still the + // topic's expiry, not the never-expire default. + *unrepresentable = true; + (None, CONFIG_SOURCE_TOPIC) + } +} + +fn derived_retention_entry( + name: &str, + unit_ms: u64, + retention: &Result, + documentation: Option<&str>, + unrepresentable: &mut bool, +) -> DescribeConfigsResourceResult { + let (millis, source) = retention_parts(retention, unrepresentable); + let value = millis.and_then(|millis| retention_in_unit(millis, unit_ms)); + config_entry( + name, + value.as_deref(), + false, + source, + CONFIG_TYPE_LONG, + Vec::new(), + documentation, + ) +} + fn config_entry( name: &str, value: Option<&str>, read_only: bool, source: i8, config_type: i8, - include_synonyms: bool, + synonyms: Vec, documentation: Option<&str>, ) -> DescribeConfigsResourceResult { DescribeConfigsResourceResult::default() @@ -389,22 +438,43 @@ fn config_entry( .with_read_only(read_only) .with_config_source(source) .with_is_sensitive(false) - .with_synonyms(synonyms(include_synonyms)) + .with_synonyms(synonyms) .with_config_type(config_type) .with_documentation(documentation.map(|text| StrBytes::from(text.to_string()))) } -/// Neither known key has a synonym. `true` asks for the list and receives none. -#[allow( - clippy::if_same_then_else, - reason = "true asks for synonyms and there are none" -)] -fn synonyms(include_synonyms: bool) -> Vec { - if include_synonyms { - Vec::new() - } else { - Vec::new() +fn synonyms( + include_synonyms: bool, + remembered: RetentionSynonyms, + retention_ms_value: Option<&str>, + source: i8, +) -> Vec { + if !include_synonyms || remembered.is_empty() { + return Vec::new(); + } + let mut listed = Vec::new(); + if remembered.minutes { + listed.push(synonym( + RETENTION_MINUTES, + retention_ms_value.and_then(|value| retention_in_unit(value, MILLIS_PER_MINUTE)), + source, + )); } + if remembered.hours { + listed.push(synonym( + RETENTION_HOURS, + retention_ms_value.and_then(|value| retention_in_unit(value, MILLIS_PER_HOUR)), + source, + )); + } + listed +} + +fn synonym(name: &'static str, value: Option, source: i8) -> DescribeConfigsSynonym { + DescribeConfigsSynonym::default() + .with_name(StrBytes::from_static_str(name)) + .with_value(value.map(StrBytes::from)) + .with_source(source) } fn resource_error( @@ -420,3 +490,192 @@ fn resource_error( .with_resource_name(resource.resource_name.clone()) .with_configs(configs) } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use iggy::prelude::{ + CompressionAlgorithm, HeaderKey, HeaderValue, IggyByteSize, IggyExpiry, IggyTimestamp, + MaxTopicSize, OptionValue, ResourceOptions, TopicDetails, topic_option_keys, + }; + use kafka_protocol::messages::describe_configs_response::DescribeConfigsResourceResult; + use kafka_protocol::protocol::StrBytes; + + use super::described_entries; + use crate::protocol::handlers::topic_config::{ + CLEANUP_POLICY, CONFIG_SOURCE_TOPIC, RETENTION_HOURS, RETENTION_MINUTES, RETENTION_MS, + RetentionSynonyms, parse_retention_ms, + }; + + fn topic_details(expiry: IggyExpiry, explicit: bool) -> TopicDetails { + let mut options = ResourceOptions::new(); + if explicit { + let key = HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY).expect("expiry key"); + options.insert( + key, + OptionValue::explicit(HeaderValue::from(u64::from(expiry))), + ); + } + TopicDetails { + id: 1, + created_at: IggyTimestamp::default(), + name: "orders".to_string(), + size: IggyByteSize::default(), + message_expiry: expiry, + compression_algorithm: CompressionAlgorithm::None, + max_topic_size: MaxTopicSize::default(), + messages_count: 0, + partitions_count: 1, + partitions: Vec::new(), + options, + } + } + + fn value_of<'a>(configs: &'a [DescribeConfigsResourceResult], name: &str) -> Option<&'a str> { + configs + .iter() + .find(|config| config.name.as_str() == name) + .and_then(|config| config.value.as_ref().map(StrBytes::as_str)) + } + + #[test] + fn minutes_and_hours_describe_as_retention_ms_with_synonyms_when_asked() { + let minutes = parse_retention_ms("120000").expect("2 minutes in ms"); + let remembered = RetentionSynonyms { + minutes: true, + hours: false, + }; + let details = topic_details(minutes, true); + let (configs, unknown, bad) = described_entries(&details, None, true, false, remembered); + assert!(unknown.is_none()); + assert!(!bad); + assert_eq!( + configs + .iter() + .map(|config| config.name.as_str()) + .collect::>(), + vec![RETENTION_MS, CLEANUP_POLICY] + ); + assert_eq!(value_of(&configs, RETENTION_MS), Some("120000")); + let retention = configs + .iter() + .find(|config| config.name.as_str() == RETENTION_MS) + .expect("retention.ms"); + assert_eq!(retention.synonyms.len(), 1); + assert_eq!(retention.synonyms[0].name.as_str(), RETENTION_MINUTES); + assert_eq!( + retention.synonyms[0].value.as_ref().map(StrBytes::as_str), + Some("2") + ); + assert_eq!(retention.synonyms[0].source, CONFIG_SOURCE_TOPIC); + let cleanup = configs + .iter() + .find(|config| config.name.as_str() == CLEANUP_POLICY) + .expect("cleanup"); + assert!(cleanup.synonyms.is_empty()); + + let hours = parse_retention_ms("3600000").expect("1 hour in ms"); + let (configs, unknown, bad) = described_entries( + &topic_details(hours, true), + None, + true, + false, + RetentionSynonyms { + minutes: false, + hours: true, + }, + ); + assert!(unknown.is_none() && !bad); + let retention = configs + .iter() + .find(|config| config.name.as_str() == RETENTION_MS) + .expect("retention.ms"); + assert_eq!( + retention.value.as_ref().map(StrBytes::as_str), + Some("3600000") + ); + assert_eq!(retention.synonyms.len(), 1); + assert_eq!(retention.synonyms[0].name.as_str(), RETENTION_HOURS); + assert_eq!( + retention.synonyms[0].value.as_ref().map(StrBytes::as_str), + Some("1") + ); + } + + #[test] + fn synonyms_are_empty_when_not_requested_or_only_milliseconds_were_set() { + let expiry = parse_retention_ms("120000").expect("ms"); + let details = topic_details(expiry, true); + let remembered = RetentionSynonyms { + minutes: true, + hours: true, + }; + let (configs, _, _) = described_entries(&details, None, false, false, remembered); + assert!(configs.iter().all(|config| config.synonyms.is_empty())); + + let (configs, _, _) = + described_entries(&details, None, true, false, RetentionSynonyms::default()); + assert!(configs.iter().all(|config| config.synonyms.is_empty())); + } + + #[test] + fn requested_minutes_or_hours_return_that_name_from_stored_milliseconds() { + let expiry = parse_retention_ms("90000").expect("ms"); + let details = topic_details(expiry, true); + let asked = [StrBytes::from_static_str(RETENTION_MINUTES)]; + let (configs, unknown, bad) = described_entries( + &details, + Some(&asked), + true, + false, + RetentionSynonyms { + minutes: true, + hours: false, + }, + ); + assert!(unknown.is_none()); + assert!(!bad); + assert_eq!(configs.len(), 1); + assert_eq!(configs[0].name.as_str(), RETENTION_MINUTES); + assert_eq!(configs[0].value.as_ref().map(StrBytes::as_str), Some("1")); + assert!(configs[0].synonyms.is_empty()); + + let asked = [StrBytes::from_static_str(RETENTION_HOURS)]; + let never = topic_details(IggyExpiry::NeverExpire, true); + let (configs, unknown, bad) = described_entries( + &never, + Some(&asked), + false, + false, + RetentionSynonyms::default(), + ); + assert!(unknown.is_none() && !bad); + assert_eq!(configs[0].name.as_str(), RETENTION_HOURS); + assert_eq!(configs[0].value.as_ref().map(StrBytes::as_str), Some("-1")); + } + + #[test] + fn stored_never_expire_reports_negative_one_for_a_remembered_hour_synonym() { + let (configs, unknown, bad) = described_entries( + &topic_details(IggyExpiry::NeverExpire, true), + None, + true, + false, + RetentionSynonyms { + minutes: false, + hours: true, + }, + ); + assert!(unknown.is_none() && !bad); + let retention = configs + .iter() + .find(|config| config.name.as_str() == RETENTION_MS) + .expect("retention.ms"); + assert_eq!(retention.value.as_ref().map(StrBytes::as_str), Some("-1")); + assert_eq!( + retention.synonyms[0].value.as_ref().map(StrBytes::as_str), + Some("-1") + ); + } +} diff --git a/gateways/kafka/src/protocol/handlers/topic_config.rs b/gateways/kafka/src/protocol/handlers/topic_config.rs index 1b6ccb56c0..627935e519 100644 --- a/gateways/kafka/src/protocol/handlers/topic_config.rs +++ b/gateways/kafka/src/protocol/handlers/topic_config.rs @@ -21,10 +21,11 @@ //! (`IggyDuration::from(u64)` / `as_micros`). `u64::MAX` is [`IggyExpiry::NeverExpire`] //! and `0` is [`IggyExpiry::ServerDefault`], so neither is a usable millisecond expiry. //! Kafka `retention.ms` is a decimal millisecond count, or `-1` when sealed segments -//! never expire. A positive count is capped at `u32::MAX` seconds, matching -//! [`IggyExpiry::from_str`]. +//! never expire. `retention.minutes` and `retention.hours` are the same duration, +//! converted to milliseconds before anything is stored. A positive count is capped +//! at `u32::MAX` seconds, matching [`IggyExpiry::from_str`]. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::str::FromStr; use iggy::prelude::{HeaderKey, IggyDuration, IggyExpiry, ResourceOptions, topic_option_keys}; @@ -51,12 +52,18 @@ pub const CONFIG_TYPE_LONG: i8 = 5; pub const CONFIG_TYPE_UNKNOWN: i8 = 0; pub const RETENTION_MS: &str = "retention.ms"; +pub const RETENTION_MINUTES: &str = "retention.minutes"; +pub const RETENTION_HOURS: &str = "retention.hours"; +pub const MILLIS_PER_MINUTE: u64 = 60_000; +pub const MILLIS_PER_HOUR: u64 = 3_600_000; pub const CLEANUP_POLICY: &str = "cleanup.policy"; pub const CLEANUP_POLICY_VALUE: &str = "delete"; pub const ONLY_TOPIC_RESOURCES: &str = "only topic resources are supported"; pub const RETENTION_DOC: &str = "How long a sealed segment is kept, in milliseconds. The active segment does not expire. -1 means sealed segments never expire."; +pub const RETENTION_MINUTES_DOC: &str = "How long a sealed segment is kept, in minutes, derived from retention.ms. The count truncates toward zero. -1 means sealed segments never expire."; +pub const RETENTION_HOURS_DOC: &str = "How long a sealed segment is kept, in hours, derived from retention.ms. The count truncates toward zero. -1 means sealed segments never expire."; pub const CLEANUP_DOC: &str = "Iggy deletes expired messages and does not compact a topic."; /// Distinct Kafka topic names one `DescribeConfigs` or `AlterConfigs` request may send to Iggy. @@ -72,6 +79,8 @@ pub const MAX_CONFIG_TOPICS: usize = 100; #[derive(Debug, Clone, PartialEq, Eq)] pub enum ListedKey { Retention, + Minutes, + Hours, Cleanup, Unknown(String), } @@ -83,6 +92,7 @@ pub enum ConfigFault { Repeated(String), ReadOnly(&'static str), InvalidRetention, + ConflictingRetention, } impl ConfigFault { @@ -93,13 +103,82 @@ impl ConfigFault { Self::Repeated(name) => format!("config key '{name}' is repeated"), Self::ReadOnly(message) => (*message).to_string(), Self::InvalidRetention => format!( - "retention.ms must be -1 or a positive number of milliseconds at most {} seconds", + "retention.ms, retention.minutes, and retention.hours must be -1 or a positive count whose duration is at most {} seconds", u32::MAX ), + Self::ConflictingRetention => "retention.ms, retention.minutes, and retention.hours must describe the same duration".to_string(), } } } +/// `retention.minutes` and `retention.hours` used by the last successful alter. +/// +/// Process-local, keyed by Iggy stream and topic, and not stored in Iggy. A +/// restart clears it. Deleting the topic outside this gateway does not. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct RetentionSynonyms { + pub minutes: bool, + pub hours: bool, +} + +impl RetentionSynonyms { + #[must_use] + pub const fn is_empty(self) -> bool { + !self.minutes && !self.hours + } +} + +/// Iggy stream and topic a Kafka name resolves to. Synonym memory uses this pair, +/// not the Kafka topic name. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct IggyTopicKey { + stream: String, + topic: String, +} + +impl IggyTopicKey { + #[must_use] + pub fn new(stream: impl Into, topic: impl Into) -> Self { + Self { + stream: stream.into(), + topic: topic.into(), + } + } +} + +/// Process memory of [`RetentionSynonyms`] per [`IggyTopicKey`]. +#[derive(Debug, Default)] +pub struct RetentionSynonymMemory { + topics: HashMap, +} + +impl RetentionSynonymMemory { + #[must_use] + pub fn remembered(&self, key: &IggyTopicKey) -> RetentionSynonyms { + self.topics.get(key).copied().unwrap_or_default() + } + + /// Remembers `synonyms` after a successful alter that stored an expiry. + /// + /// An empty set clears the key. That is an alter which used only + /// `retention.ms`. A failed alter, and a resource that stored nothing, + /// must not call this. + pub fn record_applied(&mut self, key: IggyTopicKey, synonyms: RetentionSynonyms) { + if synonyms.is_empty() { + self.topics.remove(&key); + } else { + self.topics.insert(key, synonyms); + } + } +} + +/// One duration to store, plus which synonym names the alter used. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PlannedRetention { + pub expiry: IggyExpiry, + pub synonyms: RetentionSynonyms, +} + /// `retention.ms` as Kafka should report it, plus the config source byte. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RetentionMs { @@ -166,19 +245,50 @@ pub fn retention_ms(expiry: IggyExpiry, explicit: bool) -> Result Result { - let Ok(millis) = value.parse::() else { + parse_scaled_retention(value, 1) +} + +/// `retention.ms` text scaled into `unit_ms` units. +/// +/// `-1` stays `-1`. Any other count is integer division toward zero, so a stored +/// `90000` reported as minutes is `1` while `retention.ms` stays `90000`. +#[must_use] +pub fn retention_in_unit(millis_text: &str, unit_ms: u64) -> Option { + let Ok(millis) = millis_text.parse::() else { + return None; + }; + if millis == -1 { + return Some("-1".to_string()); + } + if millis < 0 || unit_ms == 0 { + return None; + } + let Ok(millis) = u64::try_from(millis) else { + return None; + }; + Some((millis / unit_ms).to_string()) +} + +/// `count` is one unit of `unit_ms` milliseconds. `1` is `retention.ms`. +fn parse_scaled_retention(value: &str, unit_ms: u64) -> Result { + let Ok(count) = value.parse::() else { return Err(ConfigFault::InvalidRetention); }; - if millis.to_string() != value { + if count.to_string() != value { return Err(ConfigFault::InvalidRetention); } - if millis == -1 { + if count == -1 { return Ok(IggyExpiry::NeverExpire); } - if millis <= 0 { + if count <= 0 { + return Err(ConfigFault::InvalidRetention); + } + let count = u64::try_from(count).map_err(|_| ConfigFault::InvalidRetention)?; + // Reject a product that does not fit before multiplying into milliseconds. + if unit_ms == 0 || count > u64::MAX / unit_ms { return Err(ConfigFault::InvalidRetention); } - let millis = u64::try_from(millis).map_err(|_| ConfigFault::InvalidRetention)?; + let millis = count * unit_ms; // `IggyExpiry::from_str` rejects `as_secs() > u32::MAX`. The typed update path // does not, but a value the server's own parser refuses is not one this gateway // should store. @@ -268,8 +378,9 @@ pub fn bridge_failure(error: &BridgeError, handler: &str, action: &str) -> (i16, /// Keys to return for one describe resource. /// -/// `None` and an empty list both mean both known keys. Any other name is -/// [`ListedKey::Unknown`] and stays in the list. +/// `None` and an empty list both mean `retention.ms` and `cleanup.policy`. +/// `retention.minutes` and `retention.hours` are listed only when the client +/// names them. Any other name is [`ListedKey::Unknown`] and stays in the list. #[must_use] pub fn listed_keys(configuration_keys: Option<&[StrBytes]>) -> Vec { let Some(keys) = configuration_keys.filter(|keys| !keys.is_empty()) else { @@ -278,43 +389,72 @@ pub fn listed_keys(configuration_keys: Option<&[StrBytes]>) -> Vec { keys.iter() .map(|key| match key.as_str() { RETENTION_MS => ListedKey::Retention, + RETENTION_MINUTES => ListedKey::Minutes, + RETENTION_HOURS => ListedKey::Hours, CLEANUP_POLICY => ListedKey::Cleanup, other => ListedKey::Unknown(other.to_string()), }) .collect() } -/// The expiry to store, or `None` when the resource names no `retention.ms`. +/// The expiry to store, or `None` when the resource names no retention key. +/// +/// `retention.ms`, `retention.minutes`, and `retention.hours` are one duration. +/// Minutes and hours convert to milliseconds. Two names that convert to +/// different millisecond values fail the resource and select nothing. One +/// expiry is returned when they agree. [`PlannedRetention::synonyms`] is empty +/// when the resource used only `retention.ms`. /// /// One invalid or repeated key fails the whole resource. Nothing is selected -/// for storage in that case. +/// for storage in that case. `update_topic` stores the millisecond expiry, never +/// the minute or hour count. /// /// # Errors /// /// Returns the first [`ConfigFault`] in request order. pub fn plan_retention_update<'a>( configs: impl IntoIterator)>, -) -> Result, ConfigFault> { +) -> Result, ConfigFault> { let mut seen = HashSet::new(); let mut expiry = None; + let mut synonyms = RetentionSynonyms::default(); for (name, value) in configs { if !seen.insert(name) { return Err(ConfigFault::Repeated(name.to_string())); } - match name { - RETENTION_MS => { - let Some(value) = value else { - return Err(ConfigFault::InvalidRetention); - }; - expiry = Some(parse_retention_ms(value)?); - } - CLEANUP_POLICY => { - return Err(ConfigFault::ReadOnly("cleanup.policy is read-only")); - } - other => return Err(ConfigFault::UnknownKey(other.to_string())), + let unit_ms = retention_scale(name, &mut synonyms)?; + let Some(value) = value else { + return Err(ConfigFault::InvalidRetention); + }; + let parsed = if unit_ms == 1 { + parse_retention_ms(value)? + } else { + parse_scaled_retention(value, unit_ms)? + }; + if let Some(existing) = expiry + && existing != parsed + { + return Err(ConfigFault::ConflictingRetention); } + expiry = Some(parsed); + } + Ok(expiry.map(|expiry| PlannedRetention { expiry, synonyms })) +} + +fn retention_scale(name: &str, synonyms: &mut RetentionSynonyms) -> Result { + match name { + RETENTION_MS => Ok(1), + RETENTION_MINUTES => { + synonyms.minutes = true; + Ok(MILLIS_PER_MINUTE) + } + RETENTION_HOURS => { + synonyms.hours = true; + Ok(MILLIS_PER_HOUR) + } + CLEANUP_POLICY => Err(ConfigFault::ReadOnly("cleanup.policy is read-only")), + other => Err(ConfigFault::UnknownKey(other.to_string())), } - Ok(expiry) } #[cfg(test)] @@ -411,12 +551,175 @@ mod tests { assert!(matches!(err, ConfigFault::ReadOnly(_))); assert_eq!( - plan_retention_update([(RETENTION_MS, Some("-1"))]).expect("clear"), - Some(IggyExpiry::NeverExpire) + plan_retention_update([(RETENTION_MS, Some("-1"))]) + .expect("clear") + .expect("stores") + .expiry, + IggyExpiry::NeverExpire ); assert_eq!( plan_retention_update([] as [(&str, Option<&str>); 0]).expect("empty"), None ); } + + #[test] + fn minutes_and_hours_convert_to_milliseconds() { + let minutes = plan_retention_update([(RETENTION_MINUTES, Some("2"))]) + .expect("minutes") + .expect("stores"); + assert_eq!(u64::from(minutes.expiry), 2 * MILLIS_PER_MINUTE * 1_000); + assert!(minutes.synonyms.minutes); + assert!(!minutes.synonyms.hours); + let described = retention_ms(minutes.expiry, true).expect("representable"); + assert_eq!(described.value, "120000"); + assert_eq!( + retention_in_unit(&described.value, MILLIS_PER_MINUTE).as_deref(), + Some("2") + ); + + let hours = plan_retention_update([(RETENTION_HOURS, Some("1"))]) + .expect("hours") + .expect("stores"); + assert_eq!(u64::from(hours.expiry), MILLIS_PER_HOUR * 1_000); + assert!(hours.synonyms.hours); + let described = retention_ms(hours.expiry, true).expect("representable"); + assert_eq!(described.value, "3600000"); + assert_eq!( + retention_in_unit(&described.value, MILLIS_PER_HOUR).as_deref(), + Some("1") + ); + + let agreed = plan_retention_update([ + (RETENTION_MS, Some("3600000")), + (RETENTION_MINUTES, Some("60")), + (RETENTION_HOURS, Some("1")), + ]) + .expect("same duration") + .expect("one write"); + assert_eq!( + retention_ms(agreed.expiry, true) + .expect("representable") + .value, + "3600000" + ); + assert!(agreed.synonyms.minutes); + assert!(agreed.synonyms.hours); + + assert_eq!( + retention_in_unit("90000", MILLIS_PER_MINUTE).as_deref(), + Some("1"), + "synonym display truncates toward zero" + ); + assert_eq!( + retention_in_unit("-1", MILLIS_PER_HOUR).as_deref(), + Some("-1") + ); + } + + #[test] + fn synonym_memory_tracks_minutes_and_hours_and_clears_on_milliseconds_only() { + let mut memory = RetentionSynonymMemory::default(); + let orders = IggyTopicKey::new("kafka", "orders"); + let other_stream = IggyTopicKey::new("billing", "orders"); + let minutes = plan_retention_update([(RETENTION_MINUTES, Some("2"))]) + .expect("minutes") + .expect("stores"); + memory.record_applied(orders.clone(), minutes.synonyms); + assert!(memory.remembered(&orders).minutes); + assert!(memory.remembered(&other_stream).is_empty()); + + let ms_only = plan_retention_update([(RETENTION_MS, Some("1500"))]) + .expect("ms") + .expect("stores"); + assert!(ms_only.synonyms.is_empty()); + memory.record_applied(orders.clone(), ms_only.synonyms); + assert!(memory.remembered(&orders).is_empty()); + + assert_eq!( + listed_keys(None), + vec![ListedKey::Retention, ListedKey::Cleanup] + ); + let asked = [StrBytes::from_static_str(RETENTION_MINUTES)]; + assert_eq!(listed_keys(Some(&asked)), vec![ListedKey::Minutes]); + let asked = [StrBytes::from_static_str(RETENTION_HOURS)]; + assert_eq!(listed_keys(Some(&asked)), vec![ListedKey::Hours]); + } + + #[test] + fn disagreeing_retention_units_store_nothing_and_keep_remembered_names() { + let mut memory = RetentionSynonymMemory::default(); + let orders = IggyTopicKey::new("kafka", "orders"); + memory.record_applied( + orders.clone(), + RetentionSynonyms { + minutes: true, + hours: false, + }, + ); + let err = + plan_retention_update([(RETENTION_MS, Some("90000")), (RETENTION_HOURS, Some("1"))]) + .expect_err("different durations"); + assert_eq!(err, ConfigFault::ConflictingRetention); + assert!(memory.remembered(&orders).minutes); + assert!(!memory.remembered(&orders).hours); + + let err = plan_retention_update([("retention.mins", Some("1"))]).expect_err("not a name"); + assert_eq!(err, ConfigFault::UnknownKey("retention.mins".to_string())); + assert!(memory.remembered(&orders).minutes); + } + + #[test] + fn retention_hours_negative_one_stores_never_expire() { + let planned = plan_retention_update([(RETENTION_HOURS, Some("-1"))]) + .expect("hours") + .expect("stores"); + assert_eq!(planned.expiry, IggyExpiry::NeverExpire); + assert!(planned.synonyms.hours); + assert!(!planned.synonyms.minutes); + let described = retention_ms(planned.expiry, true).expect("never expire"); + assert_eq!(described.value, "-1"); + assert_eq!(described.source, CONFIG_SOURCE_TOPIC); + } + + #[test] + fn retention_unit_overflow_is_invalid_and_not_stored() { + let mut memory = RetentionSynonymMemory::default(); + let orders = IggyTopicKey::new("kafka", "orders"); + memory.record_applied( + orders.clone(), + RetentionSynonyms { + minutes: false, + hours: true, + }, + ); + + let overflow_hours = (u64::MAX / MILLIS_PER_HOUR + 1).to_string(); + let err = plan_retention_update([(RETENTION_HOURS, Some(overflow_hours.as_str()))]) + .expect_err("overflow before multiply"); + assert_eq!(err, ConfigFault::InvalidRetention); + + let past_cap_hours = (u64::from(u32::MAX) / 3_600 + 1).to_string(); + let err = plan_retention_update([(RETENTION_HOURS, Some(past_cap_hours.as_str()))]) + .expect_err("seconds above u32::MAX"); + assert_eq!(err, ConfigFault::InvalidRetention); + + let past_cap_minutes = (u64::from(u32::MAX) / 60 + 1).to_string(); + let err = plan_retention_update([(RETENTION_MINUTES, Some(past_cap_minutes.as_str()))]) + .expect_err("seconds above u32::MAX"); + assert_eq!(err, ConfigFault::InvalidRetention); + + assert!(plan_retention_update([(RETENTION_MINUTES, Some("0"))]).is_err()); + assert!(plan_retention_update([(RETENTION_HOURS, Some("0001"))]).is_err()); + assert!(plan_retention_update([(RETENTION_MINUTES, Some("+1"))]).is_err()); + assert!(memory.remembered(&orders).hours); + assert!(!memory.remembered(&orders).minutes); + + let at_cap_hours = (u64::from(u32::MAX) / 3_600).to_string(); + let stored = plan_retention_update([(RETENTION_HOURS, Some(at_cap_hours.as_str()))]) + .expect("at the second cap") + .expect("stores"); + let seconds = u64::from(stored.expiry) / 1_000 / 1_000; + assert!(u32::try_from(seconds).is_ok()); + } } diff --git a/gateways/kafka/tests/configs_handler_tests.rs b/gateways/kafka/tests/configs_handler_tests.rs index 4bf8c9346b..477fb43e5e 100644 --- a/gateways/kafka/tests/configs_handler_tests.rs +++ b/gateways/kafka/tests/configs_handler_tests.rs @@ -28,8 +28,13 @@ use kafka_protocol::messages::describe_configs_response::DescribeConfigsResponse use kafka_protocol::protocol::StrBytes; use tokio_util::sync::CancellationToken; +use bytes::Bytes; use iggy_gateway_kafka::group::{GroupCoordinator, GroupCoordinatorConfig}; -use iggy_gateway_kafka::protocol::api::{BrokerAdvertise, ERROR_NOT_CONTROLLER, GatewayState}; + +use iggy_gateway_kafka::protocol::api::{ + BrokerAdvertise, ERROR_NOT_CONTROLLER, GatewayState, handle_request, is_supported_version, + supported_api_ranges, +}; use iggy_gateway_kafka::protocol::handlers::{alter_configs, describe_configs}; #[path = "common/codec.rs"] @@ -106,3 +111,20 @@ async fn alter_configs_without_a_bridge_returns_not_controller() { assert_eq!(response.responses.len(), 1); assert_eq!(response.responses[0].error_code, ERROR_NOT_CONTROLLER); } + +#[test] +fn incremental_alter_configs_is_not_advertised() { + assert!( + supported_api_ranges() + .iter() + .all(|range| range.api_key != 44) + ); + assert!(!is_supported_version(44, 0)); + assert!(!is_supported_version(44, 1)); +} + +#[tokio::test] +async fn incremental_alter_configs_closes_the_connection() { + let outcome = handle_request(44, 1, Bytes::new(), &BrokerAdvertise::default()).await; + assert!(outcome.is_close()); +} From 927bfe63575e8008b2e22953bc4e3cae18946c77 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 4 Oct 2026 10:50:52 -0400 Subject: [PATCH 05/13] feat(gateways): fix the wire fixures.sh --- gateways/kafka/scripts/ci-wire-fixtures.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateways/kafka/scripts/ci-wire-fixtures.sh b/gateways/kafka/scripts/ci-wire-fixtures.sh index ec0793a49c..42e35a4c5c 100755 --- a/gateways/kafka/scripts/ci-wire-fixtures.sh +++ b/gateways/kafka/scripts/ci-wire-fixtures.sh @@ -24,7 +24,7 @@ set -euo pipefail FIXTURES_DIR="gateways/kafka/tools/kafka-tool/kafka_messages" # API keys requested by api_handler_tests, version_firewall_tests, and server_e2e_tests. -FIXTURE_API_KEYS=(0 1 2 10 11 12 13 14 19 22) +FIXTURE_API_KEYS=(0 1 2 10 11 12 13 14 19 22 32 33) usage() { echo "Usage: $0 {generate|cleanup}" >&2 From 42ec19cccb34e80c0f34901ed15f71d003e425b3 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 4 Oct 2026 17:45:45 -0400 Subject: [PATCH 06/13] Fixing team review skill --- gateways/kafka/docs/CONFIGS.md | 16 +- gateways/kafka/src/bridge/iggy_bridge/mod.rs | 2 +- .../kafka/src/bridge/iggy_bridge/topics.rs | 119 +++++++++++++- gateways/kafka/src/bridge/mod.rs | 4 +- .../src/protocol/handlers/alter_configs.rs | 51 +++--- .../src/protocol/handlers/create_topics.rs | 23 +-- .../src/protocol/handlers/describe_configs.rs | 154 ++++++++++-------- .../src/protocol/handlers/topic_config.rs | 26 ++- .../kafka/tests/configs_java_client_tests.rs | 64 +++++--- .../kafka/tests/configs_real_bridge_tests.rs | 27 ++- .../kafka/tests/version_firewall_tests.rs | 8 +- 11 files changed, 317 insertions(+), 177 deletions(-) diff --git a/gateways/kafka/docs/CONFIGS.md b/gateways/kafka/docs/CONFIGS.md index f8a93c7e5e..1ede6ff102 100644 --- a/gateways/kafka/docs/CONFIGS.md +++ b/gateways/kafka/docs/CONFIGS.md @@ -12,9 +12,9 @@ do not read or write Iggy. ## What a describe returns A null or empty `configuration_keys` list returns both keys below. A non-empty -list returns only the requested names, in request order. A name this gateway does -not know is still returned, and that resource's error code is `INVALID_CONFIG` -(40). +list returns only the requested names this gateway knows, in request order. A +name it does not know is omitted. The resource's error code stays `NONE` (0), +and the known keys in that request are still returned. | Key | Value | Read only | Source | | --- | ----- | --------- | ------ | @@ -75,8 +75,13 @@ Minutes and hours are not stored. If one resource sets more than one of the three names and the converted millisecond values differ, that resource is `INVALID_CONFIG` (40) and nothing is written. If they convert to the same millisecond count, one write is stored. -A resource whose `configs` list does not name any of the three is left -unchanged. Omitting them does not clear a previously set expiry. + +Kafka's `AlterConfigs` replaces a topic's full configuration. An empty `configs` +list on that API is a reset to defaults. This gateway patches the named keys +instead of replacing the set, so an empty list is `INVALID_CONFIG` (40) and +writes nothing. A previously set expiry stays in place. A list that names only +keys other than the three retention names fails for that key and also writes +nothing. `cleanup.policy` and any other key, including `retention.mins`, are `INVALID_CONFIG` (40). They are not stored. Kafka has no `NOT_CONFIGURABLE` @@ -120,6 +125,7 @@ Kafka topic name, not the Kafka name itself. | Resource type other than topic | `INVALID_REQUEST` (42) | Message is `only topic resources are supported`. Other resources in the batch still run | | Topic name fails the same rules as CreateTopics | `INVALID_TOPIC_EXCEPTION` (17) | The message is the validation reason and does not repeat the topic name | | Topic does not exist | `UNKNOWN_TOPIC_OR_PARTITION` (3) | Checked before config keys, so a missing topic is not `INVALID_CONFIG` | +| Empty `configs` list on alter | `INVALID_CONFIG` (40) | Kafka would replace the set, which is a reset. This gateway patches named keys, so an empty list is rejected and writes nothing | | More than 100 distinct topic names | `POLICY_VIOLATION` (44) | Every resource in the request | | A topic name repeated across resources | `INVALID_REQUEST` (42) | Every occurrence of the duplicate, not just the second one. Same choice CreateTopics makes for a repeated topic name | | Non-empty unknown tagged fields | `INVALID_REQUEST` (42) | Request-level tags fail every resource. A resource or config entry's own tags fail that resource. Empty tagged fields are the normal flexible encoding | diff --git a/gateways/kafka/src/bridge/iggy_bridge/mod.rs b/gateways/kafka/src/bridge/iggy_bridge/mod.rs index 7d6bf5a1e6..b1c7698afc 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/mod.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/mod.rs @@ -37,7 +37,7 @@ mod topics; use fetch::{FetchPool, LazyClient}; pub(crate) use fetch::{FetchSlot, PartitionProbe, TopicProbe}; -pub use topics::{KafkaTopicMetadata, TopicCreationOutcome}; +pub use topics::{KafkaTopicMetadata, StreamTopicCache, TopicCreationOutcome, TopicLoad}; /// Passes attempted, after the first, before [`IggyBridge::connect`] gives up and returns `Err`. /// diff --git a/gateways/kafka/src/bridge/iggy_bridge/topics.rs b/gateways/kafka/src/bridge/iggy_bridge/topics.rs index 83c4d7c041..200150ddf7 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/topics.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/topics.rs @@ -20,11 +20,12 @@ use std::collections::{HashMap, HashSet}; use iggy::prelude::{ - Identifier, IggyError, IggyExpiry, StreamClient, TopicClient, TopicCreateOptions, TopicDetails, - TopicUpdateOptions, + Identifier, IggyError, IggyExpiry, ResourceOptions, StreamClient, Topic, TopicClient, + TopicCreateOptions, TopicDetails, TopicUpdateOptions, }; use kafka_protocol::protocol::StrBytes; -use tracing::{debug, info}; +use tokio::time::{Instant, timeout_at}; +use tracing::{debug, info, warn}; use super::{IggyBridge, with_request_timeout}; use crate::bridge::error::BridgeError; @@ -315,6 +316,13 @@ impl IggyBridge { .await } + async fn topics_in_stream(&self, stream_name: &str) -> Result, BridgeError> { + let stream_id = Identifier::named(stream_name).map_err(BridgeError::Iggy)?; + // A missing stream lists as empty, the same fact `get_kafka_topics` relies on, so a + // name that resolves here and is absent from the list is "no such topic". + with_request_timeout(self.client.get_topics(&stream_id)).await + } + /// Resolves many Kafka-visible names at once, one entry per input in the same order. /// /// Batches by the Iggy stream each name resolves to, rather than paying one round trip per @@ -526,3 +534,108 @@ impl IggyBridge { Ok(results) } } + +/// `message_expiry` and admission options from one `get_topics` row. +/// +/// Enough for `DescribeConfigs`. The partition list on `TopicDetails` is not loaded. +#[derive(Debug, Clone)] +pub struct TopicConfigSnapshot { + pub message_expiry: IggyExpiry, + pub options: ResourceOptions, +} + +/// One stream's `get_topics` result, shared by every Kafka name that resolves there. +#[derive(Debug, Default)] +pub struct StreamTopicCache { + streams: HashMap, +} + +#[derive(Debug)] +enum StreamLoad { + Ready(HashMap), + Failed(BridgeError), + NotStarted, + TimedOut, +} + +/// What [`StreamTopicCache::lookup`] found for one Kafka topic name. +#[derive(Debug)] +pub enum TopicLoad<'a> { + Found(&'a TopicConfigSnapshot), + Missing, + Failed(&'a BridgeError), + NotStarted, + TimedOut, +} + +impl StreamTopicCache { + /// The topic behind `kafka_topic`, loading its stream on the first name that needs it. + /// + /// Later names that resolve to a stream already loaded, failed, or skipped reuse that + /// result. One `get_topics` call per distinct stream, including a stream that is missing. + /// A stream whose first name arrives at or after `deadline` is [`TopicLoad::NotStarted`] + /// and does not call Iggy. `api_name` is the handler in the deadline log line. + pub async fn lookup<'a>( + &'a mut self, + bridge: &IggyBridge, + kafka_topic: &str, + deadline: Instant, + api_name: &str, + ) -> TopicLoad<'a> { + let (stream, iggy_topic) = bridge.topic_identity(kafka_topic); + if !self.streams.contains_key(&stream) { + let load = load_stream(bridge, kafka_topic, &stream, deadline, api_name).await; + self.streams.insert(stream.clone(), load); + } + match self.streams.get(&stream) { + Some(StreamLoad::Ready(topics)) => topics + .get(&iggy_topic) + .map_or(TopicLoad::Missing, TopicLoad::Found), + Some(StreamLoad::Failed(error)) => TopicLoad::Failed(error), + Some(StreamLoad::NotStarted) => TopicLoad::NotStarted, + Some(StreamLoad::TimedOut) | None => TopicLoad::TimedOut, + } + } +} + +async fn load_stream( + bridge: &IggyBridge, + kafka_topic: &str, + stream: &str, + deadline: Instant, + api_name: &str, +) -> StreamLoad { + if Instant::now() >= deadline { + warn!( + resource = kafka_topic, + "{api_name} deadline passed before the topic lookup" + ); + return StreamLoad::NotStarted; + } + match timeout_at(deadline, bridge.topics_in_stream(stream)).await { + Ok(Ok(topics)) => { + let ready = topics + .into_iter() + .map(|topic| { + ( + topic.name, + TopicConfigSnapshot { + message_expiry: topic.message_expiry, + options: topic.options, + }, + ) + }) + .collect(); + StreamLoad::Ready(ready) + } + Ok(Err(error)) => StreamLoad::Failed(error), + Err(_elapsed) => { + warn!( + resource = kafka_topic, + "{api_name}: this resource's bridge work exceeded the request deadline; \ + answering retriable instead of blocking further" + ); + StreamLoad::TimedOut + } + } +} diff --git a/gateways/kafka/src/bridge/mod.rs b/gateways/kafka/src/bridge/mod.rs index abdec3f71a..f6b4f45202 100644 --- a/gateways/kafka/src/bridge/mod.rs +++ b/gateways/kafka/src/bridge/mod.rs @@ -28,5 +28,7 @@ pub mod topic_map; pub use config::{DEFAULT_MAX_MESSAGE_SIZE, IggyBridgeConfig}; pub use error::BridgeError; -pub use iggy_bridge::{IggyBridge, KafkaTopicMetadata, TopicCreationOutcome, TopicTarget}; +pub use iggy_bridge::{ + IggyBridge, KafkaTopicMetadata, StreamTopicCache, TopicCreationOutcome, TopicLoad, TopicTarget, +}; pub use topic_map::{TopicMapping, TopicOverride}; diff --git a/gateways/kafka/src/protocol/handlers/alter_configs.rs b/gateways/kafka/src/protocol/handlers/alter_configs.rs index 9298ef1d56..55661e44db 100644 --- a/gateways/kafka/src/protocol/handlers/alter_configs.rs +++ b/gateways/kafka/src/protocol/handlers/alter_configs.rs @@ -28,7 +28,7 @@ use kafka_protocol::protocol::StrBytes; use tokio::time::Instant; use crate::bridge::topic_map::validate_kafka_topic_name; -use crate::bridge::{BridgeError, IggyBridge}; +use crate::bridge::{BridgeError, IggyBridge, StreamTopicCache, TopicLoad}; use crate::protocol::api::{ API_KEY_ALTER_CONFIGS, ApiVersionRange, ERROR_INVALID_CONFIG, ERROR_INVALID_REQUEST, ERROR_INVALID_TOPIC_EXCEPTION, ERROR_NONE, ERROR_NOT_CONTROLLER, ERROR_POLICY_VIOLATION, @@ -157,6 +157,7 @@ async fn alter_all( // picking a winner - the same choice CreateTopics makes for a repeated topic name. let duplicate_names = find_duplicate_names(topic_names.iter().copied()); + let mut cache = StreamTopicCache::default(); let mut responses = Vec::with_capacity(req.resources.len()); let mut deadline_exceeded = false; for resource in &req.resources { @@ -178,7 +179,15 @@ async fn alter_all( continue; } - let result = alter_one(state, bridge, resource, req.validate_only, deadline).await; + let result = alter_one( + state, + bridge, + &mut cache, + resource, + req.validate_only, + deadline, + ) + .await; if Instant::now() >= deadline { deadline_exceeded = true; } @@ -190,6 +199,7 @@ async fn alter_all( async fn alter_one( state: &GatewayState, bridge: &IggyBridge, + cache: &mut StreamTopicCache, resource: &AlterConfigsResource, validate_only: bool, deadline: Instant, @@ -225,28 +235,16 @@ async fn alter_one( // Existence is checked before config keys. A missing topic is // UNKNOWN_TOPIC_OR_PARTITION even when a key would also be invalid. // A deadline that has already passed must not start the lockstep Iggy call. - if Instant::now() >= deadline { - tracing::warn!( - resource = name, - "AlterConfigs deadline passed before the topic lookup" - ); - return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None); - } - match tokio::time::timeout_at(deadline, bridge.get_kafka_topic(name)).await { - Ok(Ok(Some(_topic))) => {} - Ok(Ok(None)) => { + match cache.lookup(bridge, name, deadline, "AlterConfigs").await { + TopicLoad::Found(_) => {} + TopicLoad::Missing => { return resource_error(resource, ERROR_UNKNOWN_TOPIC_OR_PARTITION, None); } - Ok(Err(error)) => { - let (code, message) = bridge_failure(&error, "AlterConfigs", "altering"); + TopicLoad::Failed(error) => { + let (code, message) = bridge_failure(error, "AlterConfigs", "altering"); return resource_error(resource, code, message); } - Err(_elapsed) => { - tracing::warn!( - resource = name, - "AlterConfigs: this resource's bridge work exceeded the request deadline; \ - answering retriable instead of blocking further" - ); + TopicLoad::NotStarted | TopicLoad::TimedOut => { return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None); } } @@ -266,10 +264,17 @@ async fn alter_one( ); } }; - // A resource that names no retention key has nothing to store. Omitting the key - // does not clear a previously set expiry or the remembered synonym names. + // Kafka's AlterConfigs replaces the whole set, so an empty list is a reset. + // This gateway patches named keys. Reporting success would hide that the + // previous expiry is still stored. let Some(planned) = planned else { - return resource_error(resource, ERROR_NONE, None); + return resource_error( + resource, + ERROR_INVALID_CONFIG, + Some(static_text( + "an empty configs list is rejected because this gateway patches named keys and does not replace the topic configuration", + )), + ); }; // `validate_only` already performed the existence read and the same key checks. diff --git a/gateways/kafka/src/protocol/handlers/create_topics.rs b/gateways/kafka/src/protocol/handlers/create_topics.rs index 97e4903a85..f9fbd42341 100644 --- a/gateways/kafka/src/protocol/handlers/create_topics.rs +++ b/gateways/kafka/src/protocol/handlers/create_topics.rs @@ -38,6 +38,7 @@ use crate::protocol::api::{ ERROR_TOPIC_ALREADY_EXISTS, GatewayState, HandleOutcome, }; use crate::protocol::bounds_guard::validate_create_topics_shape; +use crate::protocol::handlers::topic_config::{MAX_CONFIG_TOPICS, topic_cap_message}; use crate::protocol::handlers::{ decode_guarded, encode_message, handle_versioned_request, is_supported_version, respond_or_close, unsupported_version_response, @@ -63,18 +64,6 @@ const DEFAULT_PARTITION_COUNT: u32 = 1; /// `NONE` for a partition count the real path can never actually create. const MAX_PARTITIONS_COUNT: u32 = 1000; -/// Cap on distinct topic names one `CreateTopics` request may address through the bridge. -/// -/// `bounds_guard`'s `MAX_REQUEST_ELEMENTS` (4,096) is a pre-decode `DoS` ceiling, not a usability -/// recommendation: each non-duplicate requested name here costs up to ~4 Iggy round trips -/// (`ensure_stream` + `create_topic`, plus a possible race-retry read on either) against the -/// single lockstep `IggyClient` every Kafka connection on this gateway shares -/// (`bridge/iggy_bridge/mod.rs`'s "Concurrency ceiling"). 100 keeps a worst-case batch's -/// aggregate bridge cost small relative to that shared resource while remaining generous for any -/// real admin batch. Duplicate names never count against this cap - they're rejected by -/// [`find_duplicate_names`] before ever reaching the bridge. -const MAX_BRIDGE_BACKED_TOPICS: usize = 100; - /// Bounds imposed on the request's own `timeout_ms` before it becomes the aggregate bridge-work /// deadline. That value is client-supplied and otherwise unchecked: `0` or negative would abort /// every topic on arrival, and an oversized one would tie up the shared `IggyClient` past any @@ -135,17 +124,15 @@ pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> Hand .map(|topic| &topic.name) .filter(|name| !duplicate_names.contains(*name)) .collect(); - if distinct_bridge_backed.len() > MAX_BRIDGE_BACKED_TOPICS { + if distinct_bridge_backed.len() > MAX_CONFIG_TOPICS { tracing::warn!( distinct_topics = distinct_bridge_backed.len(), - max = MAX_BRIDGE_BACKED_TOPICS, + max = MAX_CONFIG_TOPICS, "CreateTopics request addresses too many distinct topics; rejecting" ); - // A server-imposed limit, not a malformed request - INVALID_REQUEST would blame the + // A server-imposed limit, not a malformed request. INVALID_REQUEST would blame the // client for a request Kafka itself would accept. - let message = StrBytes::from(format!( - "this gateway addresses at most {MAX_BRIDGE_BACKED_TOPICS} distinct topics per CreateTopics request" - )); + let message = StrBytes::from(topic_cap_message("CreateTopics")); let results = req .topics .iter() diff --git a/gateways/kafka/src/protocol/handlers/describe_configs.rs b/gateways/kafka/src/protocol/handlers/describe_configs.rs index 807a029171..7f2dcc25c2 100644 --- a/gateways/kafka/src/protocol/handlers/describe_configs.rs +++ b/gateways/kafka/src/protocol/handlers/describe_configs.rs @@ -18,7 +18,7 @@ //! `DescribeConfigs` (API key 32). use bytes::Bytes; -use iggy::prelude::TopicDetails; +use iggy::prelude::{IggyExpiry, ResourceOptions}; use kafka_protocol::messages::DescribeConfigsRequest; use kafka_protocol::messages::describe_configs_request::DescribeConfigsResource; use kafka_protocol::messages::describe_configs_response::{ @@ -28,8 +28,8 @@ use kafka_protocol::messages::describe_configs_response::{ use kafka_protocol::protocol::StrBytes; use tokio::time::Instant; -use crate::bridge::IggyBridge; use crate::bridge::topic_map::validate_kafka_topic_name; +use crate::bridge::{IggyBridge, StreamTopicCache, TopicLoad}; use crate::protocol::api::{ API_KEY_DESCRIBE_CONFIGS, ApiVersionRange, ERROR_INVALID_CONFIG, ERROR_INVALID_REQUEST, ERROR_INVALID_TOPIC_EXCEPTION, ERROR_NONE, ERROR_NOT_CONTROLLER, ERROR_POLICY_VIOLATION, @@ -39,12 +39,12 @@ use crate::protocol::api::{ use crate::protocol::bounds_guard::validate_describe_configs_shape; use crate::protocol::handlers::topic_config::{ CLEANUP_DOC, CLEANUP_POLICY, CLEANUP_POLICY_VALUE, CONFIG_SOURCE_DEFAULT, CONFIG_SOURCE_TOPIC, - CONFIG_TYPE_LONG, CONFIG_TYPE_STRING, CONFIG_TYPE_UNKNOWN, IggyTopicKey, ListedKey, - MILLIS_PER_HOUR, MILLIS_PER_MINUTE, ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, RETENTION_DOC, - RETENTION_HOURS, RETENTION_HOURS_DOC, RETENTION_MINUTES, RETENTION_MINUTES_DOC, RETENTION_MS, - RetentionMs, RetentionSynonyms, bridge_failure, exceeds_topic_cap, find_duplicate_names, - listed_keys, message_expiry_is_explicit, name_reason, retention_in_unit, retention_ms, - static_text, topic_cap_message, + CONFIG_TYPE_LONG, CONFIG_TYPE_STRING, IggyTopicKey, ListedKey, MILLIS_PER_HOUR, + MILLIS_PER_MINUTE, ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, RETENTION_DOC, RETENTION_HOURS, + RETENTION_HOURS_DOC, RETENTION_MINUTES, RETENTION_MINUTES_DOC, RETENTION_MS, RetentionMs, + RetentionSynonyms, bridge_failure, exceeds_topic_cap, find_duplicate_names, listed_keys, + message_expiry_is_explicit, name_reason, retention_in_unit, retention_ms, static_text, + topic_cap_message, }; use crate::protocol::handlers::{ decode_guarded, encode_message, is_supported_version, respond_or_close, @@ -178,6 +178,7 @@ async fn describe_all( // `configuration_keys` list - the same choice CreateTopics makes for a repeated topic name. let duplicate_names = find_duplicate_names(topic_names.iter().copied()); + let mut cache = StreamTopicCache::default(); let mut results = Vec::with_capacity(req.resources.len()); let mut deadline_exceeded = false; for resource in &req.resources { @@ -212,6 +213,7 @@ async fn describe_all( let result = describe_one( state, bridge, + &mut cache, resource, req.include_synonyms, req.include_documentation, @@ -229,6 +231,7 @@ async fn describe_all( async fn describe_one( state: &GatewayState, bridge: &IggyBridge, + cache: &mut StreamTopicCache, resource: &DescribeConfigsResource, include_synonyms: bool, include_documentation: bool, @@ -260,52 +263,34 @@ async fn describe_one( ); } - if Instant::now() >= deadline { - tracing::warn!( - resource = name, - "DescribeConfigs deadline passed before the topic lookup" - ); - return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None, Vec::new()); - } - let topic = match tokio::time::timeout_at(deadline, bridge.get_kafka_topic(name)).await { - Ok(Ok(Some(topic))) => topic, - Ok(Ok(None)) => { + let snapshot = match cache + .lookup(bridge, name, deadline, "DescribeConfigs") + .await + { + TopicLoad::Found(snapshot) => snapshot, + TopicLoad::Missing => { return resource_error(resource, ERROR_UNKNOWN_TOPIC_OR_PARTITION, None, Vec::new()); } - Ok(Err(error)) => { - let (code, message) = bridge_failure(&error, "DescribeConfigs", "reading"); + TopicLoad::Failed(error) => { + let (code, message) = bridge_failure(error, "DescribeConfigs", "reading"); return resource_error(resource, code, message, Vec::new()); } - Err(_elapsed) => { - tracing::warn!( - resource = name, - "DescribeConfigs: this resource's bridge work exceeded the request deadline; \ - answering retriable instead of blocking further" - ); + TopicLoad::NotStarted | TopicLoad::TimedOut => { return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None, Vec::new()); } }; let (stream, iggy_topic) = bridge.topic_identity(name); let remembered = state.remembered_retention_synonyms(&IggyTopicKey::new(stream, iggy_topic)); - let (configs, unknown_key, unrepresentable) = described_entries( - &topic, + let (configs, unrepresentable) = described_entries( + snapshot.message_expiry, + &snapshot.options, resource.configuration_keys.as_deref(), include_synonyms, include_documentation, remembered, ); - if let Some(unknown_key) = unknown_key { - return resource_error( - resource, - ERROR_INVALID_CONFIG, - Some(StrBytes::from(format!( - "unknown config key '{unknown_key}'" - ))), - configs, - ); - } if unrepresentable { return resource_error( resource, @@ -321,16 +306,16 @@ async fn describe_one( #[must_use] fn described_entries( - topic: &TopicDetails, + message_expiry: IggyExpiry, + options: &ResourceOptions, configuration_keys: Option<&[StrBytes]>, include_synonyms: bool, include_documentation: bool, remembered: RetentionSynonyms, -) -> (Vec, Option, bool) { - let explicit = message_expiry_is_explicit(&topic.options); - let retention = retention_ms(topic.message_expiry, explicit); +) -> (Vec, bool) { + let explicit = message_expiry_is_explicit(options); + let retention = retention_ms(message_expiry, explicit); let keys = listed_keys(configuration_keys); - let mut unknown_key = None; let mut unrepresentable = false; let mut configs = Vec::with_capacity(keys.len()); for key in keys { @@ -370,23 +355,12 @@ fn described_entries( Vec::new(), include_documentation.then_some(CLEANUP_DOC), )), - ListedKey::Unknown(name) => { - if unknown_key.is_none() { - unknown_key = Some(name.clone()); - } - configs.push(config_entry( - &name, - None, - false, - CONFIG_SOURCE_DEFAULT, - CONFIG_TYPE_UNKNOWN, - Vec::new(), - None, - )); - } + // A resource error makes clients drop every returned config, including + // retention.ms. An unmodeled name is omitted and the resource stays successful. + ListedKey::Unknown(_) => {} } } - (configs, unknown_key, unrepresentable) + (configs, unrepresentable) } const fn retention_parts<'a>( @@ -532,6 +506,23 @@ mod tests { } } + fn entries( + details: &TopicDetails, + configuration_keys: Option<&[StrBytes]>, + include_synonyms: bool, + include_documentation: bool, + remembered: RetentionSynonyms, + ) -> (Vec, bool) { + described_entries( + details.message_expiry, + &details.options, + configuration_keys, + include_synonyms, + include_documentation, + remembered, + ) + } + fn value_of<'a>(configs: &'a [DescribeConfigsResourceResult], name: &str) -> Option<&'a str> { configs .iter() @@ -547,8 +538,7 @@ mod tests { hours: false, }; let details = topic_details(minutes, true); - let (configs, unknown, bad) = described_entries(&details, None, true, false, remembered); - assert!(unknown.is_none()); + let (configs, bad) = entries(&details, None, true, false, remembered); assert!(!bad); assert_eq!( configs @@ -576,7 +566,7 @@ mod tests { assert!(cleanup.synonyms.is_empty()); let hours = parse_retention_ms("3600000").expect("1 hour in ms"); - let (configs, unknown, bad) = described_entries( + let (configs, bad) = entries( &topic_details(hours, true), None, true, @@ -586,7 +576,7 @@ mod tests { hours: true, }, ); - assert!(unknown.is_none() && !bad); + assert!(!bad); let retention = configs .iter() .find(|config| config.name.as_str() == RETENTION_MS) @@ -611,11 +601,10 @@ mod tests { minutes: true, hours: true, }; - let (configs, _, _) = described_entries(&details, None, false, false, remembered); + let (configs, _) = entries(&details, None, false, false, remembered); assert!(configs.iter().all(|config| config.synonyms.is_empty())); - let (configs, _, _) = - described_entries(&details, None, true, false, RetentionSynonyms::default()); + let (configs, _) = entries(&details, None, true, false, RetentionSynonyms::default()); assert!(configs.iter().all(|config| config.synonyms.is_empty())); } @@ -624,7 +613,7 @@ mod tests { let expiry = parse_retention_ms("90000").expect("ms"); let details = topic_details(expiry, true); let asked = [StrBytes::from_static_str(RETENTION_MINUTES)]; - let (configs, unknown, bad) = described_entries( + let (configs, bad) = entries( &details, Some(&asked), true, @@ -634,7 +623,6 @@ mod tests { hours: false, }, ); - assert!(unknown.is_none()); assert!(!bad); assert_eq!(configs.len(), 1); assert_eq!(configs[0].name.as_str(), RETENTION_MINUTES); @@ -643,21 +631,21 @@ mod tests { let asked = [StrBytes::from_static_str(RETENTION_HOURS)]; let never = topic_details(IggyExpiry::NeverExpire, true); - let (configs, unknown, bad) = described_entries( + let (configs, bad) = entries( &never, Some(&asked), false, false, RetentionSynonyms::default(), ); - assert!(unknown.is_none() && !bad); + assert!(!bad); assert_eq!(configs[0].name.as_str(), RETENTION_HOURS); assert_eq!(configs[0].value.as_ref().map(StrBytes::as_str), Some("-1")); } #[test] fn stored_never_expire_reports_negative_one_for_a_remembered_hour_synonym() { - let (configs, unknown, bad) = described_entries( + let (configs, bad) = entries( &topic_details(IggyExpiry::NeverExpire, true), None, true, @@ -667,7 +655,7 @@ mod tests { hours: true, }, ); - assert!(unknown.is_none() && !bad); + assert!(!bad); let retention = configs .iter() .find(|config| config.name.as_str() == RETENTION_MS) @@ -678,4 +666,28 @@ mod tests { Some("-1") ); } + + #[test] + fn an_unmodeled_key_is_omitted_and_known_keys_remain() { + let expiry = parse_retention_ms("1500").expect("ms"); + let details = topic_details(expiry, true); + let asked = [ + StrBytes::from_static_str(RETENTION_MS), + StrBytes::from_static_str("no.such"), + ]; + let (configs, bad) = entries( + &details, + Some(&asked), + false, + false, + RetentionSynonyms::default(), + ); + assert!(!bad); + assert_eq!(configs.len(), 1); + assert_eq!(configs[0].name.as_str(), RETENTION_MS); + assert_eq!( + configs[0].value.as_ref().map(StrBytes::as_str), + Some("1500") + ); + } } diff --git a/gateways/kafka/src/protocol/handlers/topic_config.rs b/gateways/kafka/src/protocol/handlers/topic_config.rs index 627935e519..8bd472c200 100644 --- a/gateways/kafka/src/protocol/handlers/topic_config.rs +++ b/gateways/kafka/src/protocol/handlers/topic_config.rs @@ -48,9 +48,6 @@ pub const CONFIG_TYPE_STRING: i8 = 2; /// Kafka `ConfigType.LONG`. pub const CONFIG_TYPE_LONG: i8 = 5; -/// Kafka `ConfigType.UNKNOWN`. -pub const CONFIG_TYPE_UNKNOWN: i8 = 0; - pub const RETENTION_MS: &str = "retention.ms"; pub const RETENTION_MINUTES: &str = "retention.minutes"; pub const RETENTION_HOURS: &str = "retention.hours"; @@ -66,13 +63,14 @@ pub const RETENTION_MINUTES_DOC: &str = "How long a sealed segment is kept, in m pub const RETENTION_HOURS_DOC: &str = "How long a sealed segment is kept, in hours, derived from retention.ms. The count truncates toward zero. -1 means sealed segments never expire."; pub const CLEANUP_DOC: &str = "Iggy deletes expired messages and does not compact a topic."; -/// Distinct Kafka topic names one `DescribeConfigs` or `AlterConfigs` request may send to Iggy. +/// Distinct topic names one `CreateTopics`, `DescribeConfigs`, or `AlterConfigs` request +/// may address through the bridge. /// -/// Same ceiling as `CreateTopics`: each distinct name is a lockstep call on the shared client. -/// Duplicate names in one request count once for this cap, but a repeated name is itself -/// rejected outright - see [`find_duplicate_names`]. A larger batch is `POLICY_VIOLATION` on -/// every resource. The decode-time element ceiling is a separate, larger limit and is not this -/// cap. +/// `bounds_guard`'s element ceiling is a pre-decode limit. `CreateTopics` can spend several +/// lockstep Iggy calls per name on the shared client (`ensure_stream`, `create_topic`, and a +/// possible race-retry read). 100 keeps that worst case small and is shared with the config +/// APIs. Duplicate names count once toward the cap and are rejected on their own. A larger +/// batch is `POLICY_VIOLATION` on every resource. pub const MAX_CONFIG_TOPICS: usize = 100; /// One config name a describe request asked for, in request order. @@ -244,6 +242,7 @@ pub fn retention_ms(expiry: IggyExpiry, explicit: bool) -> Result Result { parse_scaled_retention(value, 1) } @@ -380,7 +379,8 @@ pub fn bridge_failure(error: &BridgeError, handler: &str, action: &str) -> (i16, /// /// `None` and an empty list both mean `retention.ms` and `cleanup.policy`. /// `retention.minutes` and `retention.hours` are listed only when the client -/// names them. Any other name is [`ListedKey::Unknown`] and stays in the list. +/// names them. Any other name is [`ListedKey::Unknown`]. `DescribeConfigs` omits +/// those names from the response and does not fail the resource. #[must_use] pub fn listed_keys(configuration_keys: Option<&[StrBytes]>) -> Vec { let Some(keys) = configuration_keys.filter(|keys| !keys.is_empty()) else { @@ -426,11 +426,7 @@ pub fn plan_retention_update<'a>( let Some(value) = value else { return Err(ConfigFault::InvalidRetention); }; - let parsed = if unit_ms == 1 { - parse_retention_ms(value)? - } else { - parse_scaled_retention(value, unit_ms)? - }; + let parsed = parse_scaled_retention(value, unit_ms)?; if let Some(existing) = expiry && existing != parsed { diff --git a/gateways/kafka/tests/configs_java_client_tests.rs b/gateways/kafka/tests/configs_java_client_tests.rs index 1fde0a4eca..76ce7bd343 100644 --- a/gateways/kafka/tests/configs_java_client_tests.rs +++ b/gateways/kafka/tests/configs_java_client_tests.rs @@ -75,13 +75,23 @@ fn java_missing() -> bool { } fn output_bounded(command: &mut Command, what: &str) -> Output { + try_output_bounded(command, what).unwrap_or_else(|error| panic!("{error}")) +} + +fn try_output_bounded(command: &mut Command, what: &str) -> Result { let mut child = command .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() - .unwrap_or_else(|error| panic!("spawn {what}: {error}")); - let mut stdout = child.stdout.take().expect("stdout"); - let mut stderr = child.stderr.take().expect("stderr"); + .map_err(|error| format!("spawn {what}: {error}"))?; + let mut stdout = child + .stdout + .take() + .ok_or_else(|| format!("{what} stdout"))?; + let mut stderr = child + .stderr + .take() + .ok_or_else(|| format!("{what} stderr"))?; let stdout_thread = std::thread::spawn(move || { let mut buf = Vec::new(); stdout.read_to_end(&mut buf).map(|_read| buf) @@ -97,25 +107,25 @@ fn output_bounded(command: &mut Command, what: &str) -> Output { Ok(None) if Instant::now() >= deadline => { let _ = child.kill(); let _ = child.wait(); - panic!("{what} exceeded {CHILD_LIMIT:?}"); + return Err(format!("{what} exceeded {CHILD_LIMIT:?}")); } Ok(None) => std::thread::sleep(Duration::from_millis(50)), - Err(error) => panic!("poll {what}: {error}"), + Err(error) => return Err(format!("poll {what}: {error}")), } }; let stdout = stdout_thread .join() - .unwrap_or_else(|_| panic!("{what} stdout thread")) - .unwrap_or_else(|error| panic!("read {what} stdout: {error}")); + .map_err(|_| format!("{what} stdout thread"))? + .map_err(|error| format!("read {what} stdout: {error}"))?; let stderr = stderr_thread .join() - .unwrap_or_else(|_| panic!("{what} stderr thread")) - .unwrap_or_else(|error| panic!("read {what} stderr: {error}")); - Output { + .map_err(|_| format!("{what} stderr thread"))? + .map_err(|error| format!("read {what} stderr: {error}"))?; + Ok(Output { status, stdout, stderr, - } + }) } const JARS: &[(&str, &str)] = &[ @@ -149,14 +159,14 @@ fn jar_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/kafka-3.9-admin-jars") } -fn ensure_jars(dir: &Path) { - std::fs::create_dir_all(dir).expect("create jar cache"); +fn ensure_jars(dir: &Path) -> Result<(), String> { + std::fs::create_dir_all(dir).map_err(|error| format!("create jar cache: {error}"))?; for (name, url) in JARS { let path = dir.join(name); if path.exists() { continue; } - let output = output_bounded( + let output = match try_output_bounded( Command::new("curl") .args([ "--fail", @@ -170,17 +180,23 @@ fn ensure_jars(dir: &Path) { .arg(&path) .arg(url), name, - ); + ) { + Ok(output) => output, + Err(error) => { + let _ = std::fs::remove_file(&path); + return Err(error); + } + }; if !output.status.success() { let _ = std::fs::remove_file(&path); + return Err(format!( + "download {url} failed: {}{}", + output.status, + String::from_utf8_lossy(&output.stderr) + )); } - assert!( - output.status.success(), - "download {url} failed: {}{}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); } + Ok(()) } fn classpath(jars: &Path, classes: &Path) -> String { @@ -221,7 +237,11 @@ async fn kafka_39_admin_client_describes_and_alters_topic_configs() { } let jars = jar_dir(); - ensure_jars(&jars); + if let Err(reason) = ensure_jars(&jars) + && skip(&reason) + { + return; + } let classes = tempfile::tempdir().expect("class dir"); let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/java/ConfigsProbe.java"); let javac = output_bounded( diff --git a/gateways/kafka/tests/configs_real_bridge_tests.rs b/gateways/kafka/tests/configs_real_bridge_tests.rs index 8893c6ee7e..eec05fa99e 100644 --- a/gateways/kafka/tests/configs_real_bridge_tests.rs +++ b/gateways/kafka/tests/configs_real_bridge_tests.rs @@ -304,17 +304,10 @@ async fn describe_and_alter_configs_reject_unknown_nontopic_missing_and_invalid_ false, ) .await; - assert_eq!(unknown.results[0].error_code, ERROR_INVALID_CONFIG); - assert_eq!(unknown.results[0].configs.len(), 2); - assert_eq!(unknown.results[0].configs[1].name.as_str(), "no.such"); - let message = unknown.results[0] - .error_message - .as_ref() - .map_or("", StrBytes::as_str); - assert!( - message.contains("no.such"), - "unknown-key message must name the key, matching AlterConfigs' own wording: {message}" - ); + assert_eq!(unknown.results[0].error_code, ERROR_NONE); + assert!(unknown.results[0].error_message.is_none()); + assert_eq!(unknown.results[0].configs.len(), 1); + assert_eq!(unknown.results[0].configs[0].name.as_str(), "retention.ms"); let broker = describe( &state, @@ -587,10 +580,10 @@ async fn describe_configs_rejects_every_occurrence_of_a_duplicate_resource_name( assert_eq!(described.results[1].error_code, ERROR_INVALID_REQUEST); } -/// An alter that does not name `retention.ms` does not clear a previously set expiry. +/// An empty alter list is not a reset. Kafka would replace the set; this gateway rejects it. #[tokio::test] #[serial] -async fn alter_configs_with_no_configs_leaves_retention_unchanged() { +async fn alter_configs_rejects_an_empty_list_and_leaves_retention_unchanged() { let data_dir = tempfile::tempdir().expect("tempdir"); let server = TestServer::spawn(data_dir.path()).await; let state = connected_state(&server).await; @@ -612,7 +605,13 @@ async fn alter_configs_with_no_configs_leaves_retention_unchanged() { false, ) .await; - assert_eq!(reset.responses[0].error_code, ERROR_NONE); + assert_eq!(reset.responses[0].error_code, ERROR_INVALID_CONFIG); + assert!( + reset.responses[0] + .error_message + .as_ref() + .is_some_and(|message| message.as_str().contains("empty")) + ); let described = describe( &state, diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index 549e77f00e..fefd0d7e13 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -101,10 +101,10 @@ fn is_supported_version_matches_scope_table() { /// KIP-511 compatibility) is hardcoded here rather than obtained by calling /// `advertised_min_version` - the same reasoning applies to it as the function under test. /// -/// Relies on `SUPPORTED_RANGES` (src) and `SCOPED_API_KEYS` (test) sharing declaration order -/// (Produce, Fetch, `ListOffsets`, Metadata, `ApiVersions`, `CreateTopics`, `FindCoordinator`, -/// `JoinGroup`, Heartbeat, `LeaveGroup`, `SyncGroup`) - `supported_ranges_table_has_fourteen_entries` plus -/// `is_supported_version_matches_scope_table` already pin that both tables cover the same keys. +/// The wire rows are `supported_api_ranges()` sorted by `api_key` (`api_versions.rs`). +/// `SCOPED_API_KEYS` is already in that ascending order, and this test reads the response +/// in that order. `supported_ranges_table_has_fourteen_entries` and +/// `is_supported_version_matches_scope_table` pin that both tables cover the same keys. #[tokio::test] async fn apiversions_advertises_exact_supported_ranges_v1() { let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &default_broker()) From f6eacae03d8085ca110668c3537bfb50e9d46e11 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 7 Oct 2026 05:23:46 -0400 Subject: [PATCH 07/13] Fixing review comments --- gateways/kafka/README.md | 9 +- gateways/kafka/docs/CONFIGS.md | 118 ++--- .../kafka/src/bridge/iggy_bridge/topics.rs | 41 +- gateways/kafka/src/protocol/api.rs | 32 +- gateways/kafka/src/protocol/bounds_guard.rs | 33 ++ .../src/protocol/handlers/alter_configs.rs | 232 ++++++--- .../src/protocol/handlers/create_topics.rs | 56 +- .../src/protocol/handlers/describe_configs.rs | 393 ++++---------- gateways/kafka/src/protocol/handlers/mod.rs | 4 +- .../src/protocol/handlers/topic_config.rs | 481 +++++------------- gateways/kafka/src/server.rs | 3 + gateways/kafka/tests/configs_handler_tests.rs | 82 ++- .../kafka/tests/configs_real_bridge_tests.rs | 2 +- .../kafka/tests/fetch_real_bridge_tests.rs | 10 +- .../kafka/tests/version_firewall_tests.rs | 16 +- 15 files changed, 618 insertions(+), 894 deletions(-) diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index 77b97c5c6d..b473f9ef33 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -6,10 +6,13 @@ Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/34 > CreateTopics, DescribeConfigs and AlterConfigs use Iggy. With the bridge off (the default), those > seven are stubs: Produce, Fetch and ListOffsets answer retriable `NOT_LEADER_OR_FOLLOWER` (6), > CreateTopics, DescribeConfigs and AlterConfigs answer `NOT_CONTROLLER` (41), and Metadata reports -> every requested topic unknown. **CreateTopics runs as the bridge's own +> every requested topic unknown. **CreateTopics and AlterConfigs run as the bridge's own > Iggy user**: with the bridge on and `IGGY_KAFKA_SASL_ENABLED` off (the default), any client that -> can reach this port can create topics (up to 1000 partitions each). Every client reads and writes -> as the bridge's Iggy user. See [docs/SCOPE.md](docs/SCOPE.md). +> can reach this port can create topics (up to 1000 partitions each) or shorten a topic's +> `retention.ms` low enough that the cleaner drops sealed segments, deleting data. With SASL on, +> `AlterConfigs` additionally requires the authenticated principal to hold `manage_topics`; without +> it every resource in the request fails `TOPIC_AUTHORIZATION_FAILED` (29) and nothing is written. +> Every client reads and writes as the bridge's Iggy user. See [docs/SCOPE.md](docs/SCOPE.md). > > InitProducerId does real work too, with or without the bridge: it allocates a producer id, so a stock idempotent producer starts instead of failing at startup. > diff --git a/gateways/kafka/docs/CONFIGS.md b/gateways/kafka/docs/CONFIGS.md index 1ede6ff102..19fa2e5704 100644 --- a/gateways/kafka/docs/CONFIGS.md +++ b/gateways/kafka/docs/CONFIGS.md @@ -2,19 +2,33 @@ `DescribeConfigs` (32) and `AlterConfigs` (33) apply to Kafka topic resources only (resource type 2). Only the deprecated `Admin.alterConfigs` (API 33) writes -retention. `IncrementalAlterConfigs` (44) is not in this change. It is planned -after this work merges. It is not advertised, and a client that sends 44 still -gets the connection closed. Only `AlterConfigs` (33) writes retention. +retention. `IncrementalAlterConfigs` (44) is not in this change and is not +advertised in `ApiVersions`. A client picks its admin API from that response, so +a Java or librdkafka client that would otherwise send 44 raises an +`UnsupportedVersionException` (or the librdkafka equivalent) locally and never +sends the request - the gateway never sees it, and the connection is not +affected. Only `AlterConfigs` (33) writes retention. With the bridge off, both APIs answer `NOT_CONTROLLER` (41) for each resource and do not read or write Iggy. +## Authorization + +`AlterConfigs` requires the `manage_topics` (or `manage_streams`) Iggy permission +on the authenticated principal when SASL is enabled. A principal without it gets +`TOPIC_AUTHORIZATION_FAILED` (29) for every resource in the request, and +`update_topic` is never called. With SASL disabled there is no principal to +check, and the request is not gated here. `DescribeConfigs` performs no +authorization check of its own. + ## What a describe returns A null or empty `configuration_keys` list returns both keys below. A non-empty -list returns only the requested names this gateway knows, in request order. A -name it does not know is omitted. The resource's error code stays `NONE` (0), -and the known keys in that request are still returned. +list returns only the requested names this gateway knows, in request order, +with a name repeated in the list collapsed to one entry. A name it does not +know, including `retention.minutes` and `retention.hours`, is omitted. The +resource's error code stays `NONE` (0), and the known keys in that request are +still returned. | Key | Value | Read only | Source | | --- | ----- | --------- | ------ | @@ -23,22 +37,9 @@ and the known keys in that request are still returned. `is_sensitive` is false for both. From v3, `include_documentation` is honored: true attaches one sentence per known key, false sends no documentation. - -`retention.minutes` and `retention.hours` are not listed unless the client names -them in `configuration_keys`. A request that names one of those keys gets that -name back, with the value derived from the stored millisecond count (integer -division toward zero, and `-1` stays `-1`). The stored source of truth is still -`retention.ms`. Naming either key is not `INVALID_CONFIG`. - -`include_synonyms` is honored. The synonym list on `retention.ms` is empty when -the flag is false, or when this process does not remember `retention.minutes` or -`retention.hours` for that topic. When the flag is true and an alter on this -process used one or both of those names, the list has one entry per remembered -name. The synonym value is the stored millisecond count divided by 60000 or -3600000, truncating toward zero. `-1` stays `-1`. `cleanup.policy` has no -synonyms. A null or empty `configuration_keys` list still returns only -`retention.ms` and `cleanup.policy`; minutes and hours show up there only as -synonyms. +`include_synonyms` has no effect: this gateway models no config-inheritance +chain beyond `retention.ms` itself, so the synonym list on every entry is +always empty. Iggy stores `message_expiry` as `IggyExpiry`. A duration is `IggyDuration`, and that type counts microseconds. Retention applies to sealed segments only. Iggy @@ -50,82 +51,59 @@ the never-expire default. The gateway does not round. ## What an alter stores -`retention.ms`, `retention.minutes`, and `retention.hours` are one writable -duration. Iggy stores milliseconds, through `update_topic`'s `message_expiry`. -Minutes and hours are not stored. - -- `retention.ms` is the source of truth. -- `retention.minutes` converts as milliseconds = minutes * 60000. A later - describe of the synonym divides the stored milliseconds by 60000. -- `retention.hours` converts as milliseconds = hours * 3600000. A later describe - of the synonym divides the stored milliseconds by 3600000. -- `-1` on any of the three stores `IggyExpiry::NeverExpire`. The server treats - `0` (`IggyExpiry::ServerDefault`) as "leave the current expiry", so `-1` is +`retention.ms` is the only writable duration. Iggy stores milliseconds, through +`update_topic`'s `message_expiry`. `retention.minutes` and `retention.hours` are +not real Kafka topic-level configs - only the broker-level `log.retention.minutes` +and `log.retention.hours` exist in real Kafka - so this gateway rejects both with +`INVALID_CONFIG` (40), the same as any other unknown key. + +- `retention.ms` is the source of truth, and the only retention key this + gateway writes. +- `-1` stores `IggyExpiry::NeverExpire`. The server treats `0` + (`IggyExpiry::ServerDefault`) as "leave the current expiry", so `-1` is not sent as 0. A following describe reports `-1` with source `DYNAMIC_TOPIC_CONFIG`. - A positive count must be the canonical decimal of that integer (`1`, not - `0001` or `+1`). The gateway rejects a product that overflows before it - multiplies into milliseconds. The resulting duration must be at most - `u32::MAX` seconds, the same limit `IggyExpiry::from_str` enforces. A longer - value is `INVALID_CONFIG` (40) and is not stored. A following describe of + `0001` or `+1`). The resulting duration must be at most `u32::MAX` seconds, + the same limit `IggyExpiry::from_str` enforces. A longer value is + `INVALID_CONFIG` (40) and is not stored. A following describe of `retention.ms` reports that millisecond count with source `DYNAMIC_TOPIC_CONFIG`. - `0` and every other value are `INVALID_CONFIG` (40). -If one resource sets more than one of the three names and the converted -millisecond values differ, that resource is `INVALID_CONFIG` (40) and nothing -is written. If they convert to the same millisecond count, one write is stored. - Kafka's `AlterConfigs` replaces a topic's full configuration. An empty `configs` list on that API is a reset to defaults. This gateway patches the named keys instead of replacing the set, so an empty list is `INVALID_CONFIG` (40) and -writes nothing. A previously set expiry stays in place. A list that names only -keys other than the three retention names fails for that key and also writes -nothing. +writes nothing. A previously set expiry stays in place. -`cleanup.policy` and any other key, including `retention.mins`, are +`cleanup.policy`, `retention.minutes`, `retention.hours`, and any other key are `INVALID_CONFIG` (40). They are not stored. Kafka has no `NOT_CONFIGURABLE` code. One bad key fails that resource and writes nothing for it. Other resources in the same request are still processed. `validate_only` runs the same checks, including the topic lookup, and does not -call `update_topic`. It also does not change which synonym names are remembered. - -## Synonym memory - -When an alter successfully applies and the request used `retention.minutes` -and/or `retention.hours`, the gateway remembers those names for that topic. -When an alter successfully applies and it used only `retention.ms`, the -remembered names for that topic are cleared. A failed alter does not change -them. A resource that does not store an expiry does not change them either. - -The key is the Iggy stream and topic the gateway already resolves from the -Kafka topic name, not the Kafka name itself. +call `update_topic`. ## Limitations - `IncrementalAlterConfigs` (API 44) is not implemented here. It is planned - after this work merges. A client that sends 44 still gets the connection - closed. Only `AlterConfigs` (33) writes retention. -- Synonym memory is process-local. It is lost on gateway restart and is not - shared across gateway processes. After a restart the synonym list is empty - until the next alter that uses minutes or hours. -- The memory is not cleared when a topic is deleted outside this gateway - (`DeleteTopics` is not in this branch). A later recreate can show a stale - synonym until the next alter. -- Synonym display truncates toward zero. A stored 90000 ms remembered as - minutes is reported as 1 minute on the synonym, while `retention.ms` still - shows 90000. + after this work merges. It is not advertised, so a client falls back to + `AlterConfigs` (33); see "Authorization" above for what an unadvertised key + does to the client. - `retention.ms` remains the only value Iggy stores. ## Other resource errors | Condition | Code | Notes | | --------- | ---- | ----- | +| Caller lacks `manage_topics` (SASL on, `AlterConfigs` only) | `TOPIC_AUTHORIZATION_FAILED` (29) | Checked before the topic lookup. Every resource in the request | | Resource type other than topic | `INVALID_REQUEST` (42) | Message is `only topic resources are supported`. Other resources in the batch still run | | Topic name fails the same rules as CreateTopics | `INVALID_TOPIC_EXCEPTION` (17) | The message is the validation reason and does not repeat the topic name | | Topic does not exist | `UNKNOWN_TOPIC_OR_PARTITION` (3) | Checked before config keys, so a missing topic is not `INVALID_CONFIG` | | Empty `configs` list on alter | `INVALID_CONFIG` (40) | Kafka would replace the set, which is a reset. This gateway patches named keys, so an empty list is rejected and writes nothing | -| More than 100 distinct topic names | `POLICY_VIOLATION` (44) | Every resource in the request | +| More than 100 distinct topic names | `POLICY_VIOLATION` (44) | Every resource in the request, checked before the bridge-availability check | | A topic name repeated across resources | `INVALID_REQUEST` (42) | Every occurrence of the duplicate, not just the second one. Same choice CreateTopics makes for a repeated topic name | -| Non-empty unknown tagged fields | `INVALID_REQUEST` (42) | Request-level tags fail every resource. A resource or config entry's own tags fail that resource. Empty tagged fields are the normal flexible encoding | + +Unknown tagged fields at the request, resource, or config-entry level are +ignored, not rejected: Kafka's flexible versions define them as a forward-compatible +extension point a server that does not recognize them must skip, not refuse. diff --git a/gateways/kafka/src/bridge/iggy_bridge/topics.rs b/gateways/kafka/src/bridge/iggy_bridge/topics.rs index 200150ddf7..948b3512c5 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/topics.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/topics.rs @@ -20,8 +20,8 @@ use std::collections::{HashMap, HashSet}; use iggy::prelude::{ - Identifier, IggyError, IggyExpiry, ResourceOptions, StreamClient, Topic, TopicClient, - TopicCreateOptions, TopicDetails, TopicUpdateOptions, + Identifier, IggyError, IggyExpiry, StreamClient, Topic, TopicClient, TopicCreateOptions, + TopicDetails, TopicUpdateOptions, }; use kafka_protocol::protocol::StrBytes; use tokio::time::{Instant, timeout_at}; @@ -30,6 +30,7 @@ use tracing::{debug, info, warn}; use super::{IggyBridge, with_request_timeout}; use crate::bridge::error::BridgeError; use crate::bridge::topic_map::validate_kafka_topic_name; +use crate::protocol::handlers::topic_config::message_expiry_is_explicit; /// Outcome of [`IggyBridge::create_kafka_topic`]. /// @@ -535,13 +536,16 @@ impl IggyBridge { } } -/// `message_expiry` and admission options from one `get_topics` row. +/// `message_expiry` and whether it was set explicitly, from one `get_topics` row. /// -/// Enough for `DescribeConfigs`. The partition list on `TopicDetails` is not loaded. -#[derive(Debug, Clone)] +/// Enough for `DescribeConfigs`. The partition list on `TopicDetails` is not loaded. Only +/// [`message_expiry_is_explicit`] of the full `ResourceOptions` map a row carries is ever read +/// downstream, so this stores that one bit instead of cloning the whole map into the cache for +/// every topic in the stream, including every topic no describe ever names. +#[derive(Debug, Clone, Copy)] pub struct TopicConfigSnapshot { pub message_expiry: IggyExpiry, - pub options: ResourceOptions, + pub message_expiry_explicit: bool, } /// One stream's `get_topics` result, shared by every Kafka name that resolves there. @@ -575,6 +579,11 @@ impl StreamTopicCache { /// result. One `get_topics` call per distinct stream, including a stream that is missing. /// A stream whose first name arrives at or after `deadline` is [`TopicLoad::NotStarted`] /// and does not call Iggy. `api_name` is the handler in the deadline log line. + /// + /// # Panics + /// + /// Never, in practice: `stream` is inserted immediately above whenever it is not already a + /// key, so the lookup right after always finds it. pub async fn lookup<'a>( &'a mut self, bridge: &IggyBridge, @@ -587,13 +596,20 @@ impl StreamTopicCache { let load = load_stream(bridge, kafka_topic, &stream, deadline, api_name).await; self.streams.insert(stream.clone(), load); } - match self.streams.get(&stream) { - Some(StreamLoad::Ready(topics)) => topics + // The branch above guarantees `stream` is now a key, whether it was already cached or + // just inserted - `None` is not a real outcome here, unlike the `StreamLoad` variants + // below, so it is not given one of its own. + match self + .streams + .get(&stream) + .expect("stream was just inserted, or was already present") + { + StreamLoad::Ready(topics) => topics .get(&iggy_topic) .map_or(TopicLoad::Missing, TopicLoad::Found), - Some(StreamLoad::Failed(error)) => TopicLoad::Failed(error), - Some(StreamLoad::NotStarted) => TopicLoad::NotStarted, - Some(StreamLoad::TimedOut) | None => TopicLoad::TimedOut, + StreamLoad::Failed(error) => TopicLoad::Failed(error), + StreamLoad::NotStarted => TopicLoad::NotStarted, + StreamLoad::TimedOut => TopicLoad::TimedOut, } } } @@ -617,11 +633,12 @@ async fn load_stream( let ready = topics .into_iter() .map(|topic| { + let message_expiry_explicit = message_expiry_is_explicit(&topic.options); ( topic.name, TopicConfigSnapshot { message_expiry: topic.message_expiry, - options: topic.options, + message_expiry_explicit, }, ) }) diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 4ad4e46dae..d17a4aa475 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::sync::{Arc, Mutex, PoisonError}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use bytes::Bytes; @@ -26,6 +26,7 @@ use kafka_protocol::messages::{ use tokio::sync::Semaphore; use tokio_util::sync::CancellationToken; +use crate::auth::AuthenticatedPrincipal; use crate::bridge::IggyBridge; use crate::error::Result; use crate::group::{GroupCoordinator, GroupCoordinatorConfig}; @@ -37,9 +38,6 @@ use crate::protocol::bounds_guard::{ validate_describe_acls_shape, validate_sasl_authenticate_shape, validate_sasl_handshake_shape, }; use crate::protocol::handlers::init_producer_id::ProducerIdAllocator; -use crate::protocol::handlers::topic_config::{ - IggyTopicKey, RetentionSynonymMemory, RetentionSynonyms, -}; use crate::protocol::handlers::{ alter_configs, api_versions, create_topics, decode_guarded, describe_configs, dispatch, fetch, find_coordinator, heartbeat, init_producer_id, join_group, leave_group, list_offsets, metadata, @@ -319,9 +317,6 @@ pub struct GatewayState { /// Consumer group membership. Process-wide and independent of the bridge: a member outlives /// the connection that created it, and group coordination needs no Iggy call. pub groups: GroupCoordinator, - /// `retention.minutes` and `retention.hours` from the last successful alter on this - /// process, keyed by Iggy stream and topic. Not stored in Iggy. - pub(crate) retention_synonym_memory: Mutex, /// Per Kafka topic and partition, how long probes read it as loading. Fetch and `ListOffsets` /// share it, since every consumer sees the same probe. pub(crate) loading: Mutex, @@ -351,7 +346,6 @@ impl GatewayState { producer_ids: ProducerIdAllocator::new(instance_id), produce_slots: Semaphore::const_new(PRODUCE_SLOTS), groups, - retention_synonym_memory: Mutex::default(), loading: Mutex::default(), probe_board: ProbeBoard::default(), } @@ -371,20 +365,6 @@ impl GatewayState { GroupCoordinator::new(GroupCoordinatorConfig::default(), CancellationToken::new()), ) } - - pub(crate) fn remembered_retention_synonyms(&self, key: &IggyTopicKey) -> RetentionSynonyms { - self.retention_synonym_memory - .lock() - .unwrap_or_else(PoisonError::into_inner) - .remembered(key) - } - - pub(crate) fn record_retention_synonyms(&self, key: IggyTopicKey, synonyms: RetentionSynonyms) { - self.retention_synonym_memory - .lock() - .unwrap_or_else(PoisonError::into_inner) - .record_applied(key, synonyms); - } } /// What a handler keeps for one Kafka connection, across its requests. @@ -434,18 +414,22 @@ pub async fn handle_request_bounded( body: Bytes, ) -> HandleOutcome { let connection = ConnectionState::default(); - handle_connection_request(state, &connection, api_key, api_version, body).await + handle_connection_request(state, &connection, None, api_key, api_version, body).await } /// [`handle_request_bounded`] for one request of `connection`. +/// +/// `principal` is `None` when SASL is off, or for a key the connection reached before +/// authenticating - only [`crate::server::route_frame`] ever has a principal to pass. pub async fn handle_connection_request( state: &GatewayState, connection: &ConnectionState, + principal: Option<&AuthenticatedPrincipal>, api_key: i16, api_version: i16, body: Bytes, ) -> HandleOutcome { - dispatch(state, connection, api_key, api_version, body).await + dispatch(state, connection, principal, api_key, api_version, body).await } #[must_use] diff --git a/gateways/kafka/src/protocol/bounds_guard.rs b/gateways/kafka/src/protocol/bounds_guard.rs index 5fd688d221..d348c37e76 100644 --- a/gateways/kafka/src/protocol/bounds_guard.rs +++ b/gateways/kafka/src/protocol/bounds_guard.rs @@ -1727,4 +1727,37 @@ mod tests { assert!(validate_leave_group_shape(4, &body, 8 * 1024 * 1024).is_ok()); assert!(validate_leave_group_shape(4, &body, 1_024).is_err()); } + + /// `validate_describe_configs_shape` and `validate_alter_configs_shape` had no POC of their + /// own before this - every sibling guard above pins a rejection for both the legacy + /// (`i32`) and flexible (compact varint) count forms, and these two never did. + #[test] + fn describe_configs_v1_huge_legacy_count_rejected() { + let body = Bytes::from_static(&[0x7F, 0xFF, 0xFF, 0xFF]); + assert!(validate_describe_configs_shape(1, &body, TEST_MAX_FRAME_SIZE).is_err()); + } + + /// The flexible (v4+) sibling of the legacy test above: a huge compact-array-count varint + /// for `resources` on its own, exceeding `MAX_COLLECTION_LEN` regardless of what bytes + /// remain in the frame. + #[test] + fn describe_configs_v4_huge_compact_count_rejected() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF, 0xFF, 0x0F]); // u32::MAX, 5-byte varint + assert!(validate_describe_configs_shape(4, &body, TEST_MAX_FRAME_SIZE).is_err()); + } + + #[test] + fn alter_configs_v0_huge_legacy_count_rejected() { + let body = Bytes::from_static(&[0x7F, 0xFF, 0xFF, 0xFF]); + assert!(validate_alter_configs_shape(0, &body, TEST_MAX_FRAME_SIZE).is_err()); + } + + /// The flexible (v2+) sibling of the legacy test above: a huge compact-array-count varint + /// for `resources` on its own, exceeding `MAX_COLLECTION_LEN` regardless of what bytes + /// remain in the frame. + #[test] + fn alter_configs_v2_huge_compact_count_rejected() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF, 0xFF, 0x0F]); // u32::MAX, 5-byte varint + assert!(validate_alter_configs_shape(2, &body, TEST_MAX_FRAME_SIZE).is_err()); + } } diff --git a/gateways/kafka/src/protocol/handlers/alter_configs.rs b/gateways/kafka/src/protocol/handlers/alter_configs.rs index 55661e44db..8386db6f98 100644 --- a/gateways/kafka/src/protocol/handlers/alter_configs.rs +++ b/gateways/kafka/src/protocol/handlers/alter_configs.rs @@ -17,6 +17,8 @@ //! `AlterConfigs` (API key 33). +use std::collections::HashSet; + use bytes::Bytes; use iggy::prelude::IggyError; use kafka_protocol::messages::AlterConfigsRequest; @@ -27,18 +29,19 @@ use kafka_protocol::messages::alter_configs_response::{ use kafka_protocol::protocol::StrBytes; use tokio::time::Instant; +use crate::auth::AuthenticatedPrincipal; use crate::bridge::topic_map::validate_kafka_topic_name; use crate::bridge::{BridgeError, IggyBridge, StreamTopicCache, TopicLoad}; use crate::protocol::api::{ API_KEY_ALTER_CONFIGS, ApiVersionRange, ERROR_INVALID_CONFIG, ERROR_INVALID_REQUEST, ERROR_INVALID_TOPIC_EXCEPTION, ERROR_NONE, ERROR_NOT_CONTROLLER, ERROR_POLICY_VIOLATION, - ERROR_REQUEST_TIMED_OUT, ERROR_UNKNOWN_TOPIC_OR_PARTITION, GatewayState, HandleOutcome, - REQUEST_DEADLINE, + ERROR_REQUEST_TIMED_OUT, ERROR_TOPIC_AUTHORIZATION_FAILED, ERROR_UNKNOWN_TOPIC_OR_PARTITION, + GatewayState, HandleOutcome, REQUEST_DEADLINE, }; use crate::protocol::bounds_guard::validate_alter_configs_shape; use crate::protocol::handlers::topic_config::{ - IggyTopicKey, ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, bridge_failure, exceeds_topic_cap, - find_duplicate_names, name_reason, plan_retention_update, static_text, topic_cap_message, + ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, bridge_failure, name_reason, plan_retention_update, + topic_cap_and_duplicates, topic_cap_message, }; use crate::protocol::handlers::{ decode_guarded, encode_message, is_supported_version, respond_or_close, @@ -51,7 +54,12 @@ pub const RANGE: ApiVersionRange = ApiVersionRange { max_version: 2, }; -pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> HandleOutcome { +pub async fn handle( + state: &GatewayState, + principal: Option<&AuthenticatedPrincipal>, + api_version: i16, + body: Bytes, +) -> HandleOutcome { if !is_supported_version(API_KEY_ALTER_CONFIGS, api_version) { return unsupported_version_response(API_KEY_ALTER_CONFIGS, api_version, |version| { encode_error_response(version, ERROR_INVALID_REQUEST) @@ -71,6 +79,30 @@ pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> Hand } }; + // Checked before the bridge-availability stub, not inside `alter_all`: a cap or an + // authorization decision does not depend on the controller being reachable, and a check + // that only runs once the bridge is up is never exercised by a serverless/stub test. + if let Some(responses) = authorize(principal, &req.resources) { + return respond_or_close( + encode_message(&response(responses), api_version, 256), + "AlterConfigs", + ); + } + let (exceeds_cap, duplicate_names) = topic_cap_and_duplicates(&req.resources); + if exceeds_cap { + let message = StrBytes::from(topic_cap_message("AlterConfigs")); + tracing::warn!("AlterConfigs request addresses too many distinct topics; rejecting"); + let responses = req + .resources + .iter() + .map(|resource| resource_error(resource, ERROR_POLICY_VIOLATION, Some(message.clone()))) + .collect(); + return respond_or_close( + encode_message(&response(responses), api_version, 256), + "AlterConfigs", + ); + } + let Some(bridge) = &state.bridge else { let responses = req .resources @@ -84,13 +116,51 @@ pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> Hand }; let deadline = Instant::now() + REQUEST_DEADLINE; - let responses = alter_all(state, bridge, &req, deadline).await; + let responses = alter_all(bridge, &req, &duplicate_names, deadline).await; respond_or_close( encode_message(&response(responses), api_version, 256), "AlterConfigs", ) } +/// Denies every resource when `principal` is known but is not allowed to alter topic +/// configuration, or when its permissions could not be read. `None` means SASL is off, which +/// this gateway treats as no principal to enforce against - the same choice the rest of this +/// gateway makes when authentication itself is not configured. +fn authorize( + principal: Option<&AuthenticatedPrincipal>, + resources: &[AlterConfigsResource], +) -> Option> { + let principal = principal?; + if principal.permissions_known && principal.permissions.manage_topics { + return None; + } + if !principal.permissions_known { + // The permission read failed after a successful login, so this connection holds no + // real answer. Denying is the fail-closed choice: granting would authorize a write off + // a value nothing ever actually read. + tracing::warn!( + principal = %principal.username, + "AlterConfigs asked on a connection whose permissions were never read; denying" + ); + } + let message = StrBytes::from_static_str( + "the authenticated principal is not authorized to alter topic configuration", + ); + Some( + resources + .iter() + .map(|resource| { + resource_error( + resource, + ERROR_TOPIC_AUTHORIZATION_FAILED, + Some(message.clone()), + ) + }) + .collect(), + ) +} + /// # Errors /// /// Returns an error when `kafka_protocol` cannot encode the response at `version`. @@ -112,51 +182,11 @@ fn response(responses: Vec) -> AlterConfigsRespons } async fn alter_all( - state: &GatewayState, bridge: &IggyBridge, req: &AlterConfigsRequest, + duplicate_names: &HashSet<&str>, deadline: Instant, ) -> Vec { - if !req.unknown_tagged_fields.is_empty() { - let message = static_text("unknown tagged fields are not supported"); - return if req.resources.is_empty() { - vec![ - AlterConfigsResourceResponse::default() - .with_error_code(ERROR_INVALID_REQUEST) - .with_error_message(Some(message)) - .with_resource_type(RESOURCE_TYPE_TOPIC) - .with_resource_name(StrBytes::from_static_str("")), - ] - } else { - req.resources - .iter() - .map(|resource| { - resource_error(resource, ERROR_INVALID_REQUEST, Some(message.clone())) - }) - .collect() - }; - } - - let topic_names: Vec<&str> = req - .resources - .iter() - .filter(|resource| resource.resource_type == RESOURCE_TYPE_TOPIC) - .map(|resource| resource.resource_name.as_str()) - .collect(); - if exceeds_topic_cap(topic_names.iter().copied()) { - let message = StrBytes::from(topic_cap_message("AlterConfigs")); - tracing::warn!("AlterConfigs request addresses too many distinct topics; rejecting"); - return req - .resources - .iter() - .map(|resource| resource_error(resource, ERROR_POLICY_VIOLATION, Some(message.clone()))) - .collect(); - } - - // Real Kafka refuses every occurrence of a duplicate resource name outright rather than - // picking a winner - the same choice CreateTopics makes for a repeated topic name. - let duplicate_names = find_duplicate_names(topic_names.iter().copied()); - let mut cache = StreamTopicCache::default(); let mut responses = Vec::with_capacity(req.resources.len()); let mut deadline_exceeded = false; @@ -179,15 +209,7 @@ async fn alter_all( continue; } - let result = alter_one( - state, - bridge, - &mut cache, - resource, - req.validate_only, - deadline, - ) - .await; + let result = alter_one(bridge, &mut cache, resource, req.validate_only, deadline).await; if Instant::now() >= deadline { deadline_exceeded = true; } @@ -197,30 +219,17 @@ async fn alter_all( } async fn alter_one( - state: &GatewayState, bridge: &IggyBridge, cache: &mut StreamTopicCache, resource: &AlterConfigsResource, validate_only: bool, deadline: Instant, ) -> AlterConfigsResourceResponse { - if !resource.unknown_tagged_fields.is_empty() - || resource - .configs - .iter() - .any(|config| !config.unknown_tagged_fields.is_empty()) - { - return resource_error( - resource, - ERROR_INVALID_REQUEST, - Some(static_text("unknown tagged fields are not supported")), - ); - } if resource.resource_type != RESOURCE_TYPE_TOPIC { return resource_error( resource, ERROR_INVALID_REQUEST, - Some(static_text(ONLY_TOPIC_RESOURCES)), + Some(StrBytes::from_static_str(ONLY_TOPIC_RESOURCES)), ); } let name = resource.resource_name.as_str(); @@ -249,13 +258,13 @@ async fn alter_one( } } - let planned = match plan_retention_update(resource.configs.iter().map(|config| { + let expiry = match plan_retention_update(resource.configs.iter().map(|config| { ( config.name.as_str(), config.value.as_ref().map(StrBytes::as_str), ) })) { - Ok(planned) => planned, + Ok(expiry) => expiry, Err(fault) => { return resource_error( resource, @@ -267,18 +276,18 @@ async fn alter_one( // Kafka's AlterConfigs replaces the whole set, so an empty list is a reset. // This gateway patches named keys. Reporting success would hide that the // previous expiry is still stored. - let Some(planned) = planned else { + let Some(expiry) = expiry else { return resource_error( resource, ERROR_INVALID_CONFIG, - Some(static_text( + Some(StrBytes::from_static_str( "an empty configs list is rejected because this gateway patches named keys and does not replace the topic configuration", )), ); }; // `validate_only` already performed the existence read and the same key checks. - // It does not call `update_topic` and does not change synonym memory. + // It does not call `update_topic`. if validate_only { return resource_error(resource, ERROR_NONE, None); } @@ -286,7 +295,12 @@ async fn alter_one( // Do not start the write once the deadline has passed, and do not wrap it in // `timeout_at`. Dropping that future does not abort the SDK task, so // `UpdateTopic` can commit after this handler has already answered - // REQUEST_TIMED_OUT. A short retention then deletes sealed segments. + // REQUEST_TIMED_OUT. A short retention then deletes sealed segments. This is a + // policy choice about *this* handler's own deadline, independent of + // `update_kafka_topic_message_expiry` itself: that bridge call already wraps its + // SDK round trip in `with_request_timeout` (see `topics.rs`), so the write it + // starts is never unbounded - it can simply still be in flight when this + // handler's own, shorter deadline has already elapsed. if Instant::now() >= deadline { tracing::warn!( resource = name, @@ -294,15 +308,8 @@ async fn alter_one( ); return resource_error(resource, ERROR_REQUEST_TIMED_OUT, None); } - match bridge - .update_kafka_topic_message_expiry(name, planned.expiry) - .await - { + match bridge.update_kafka_topic_message_expiry(name, expiry).await { Ok(()) | Err(BridgeError::Iggy(IggyError::RequestAlreadyApplied)) => { - // Keyed by the mapped Iggy stream and topic, not the Kafka name. - let (stream, iggy_topic) = bridge.topic_identity(name); - state - .record_retention_synonyms(IggyTopicKey::new(stream, iggy_topic), planned.synonyms); resource_error(resource, ERROR_NONE, None) } Err(error) => { @@ -323,3 +330,60 @@ fn resource_error( .with_resource_type(resource.resource_type) .with_resource_name(resource.resource_name.clone()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::acl::PrincipalPermissions; + + fn principal(manage_topics: bool, permissions_known: bool) -> AuthenticatedPrincipal { + AuthenticatedPrincipal { + username: "alice".to_string(), + permissions: PrincipalPermissions { + manage_topics, + ..PrincipalPermissions::default() + }, + permissions_known, + } + } + + fn resource(name: &str) -> AlterConfigsResource { + AlterConfigsResource::default() + .with_resource_type(RESOURCE_TYPE_TOPIC) + .with_resource_name(StrBytes::from(name.to_string())) + } + + #[test] + fn no_principal_is_not_gated() { + assert!(authorize(None, &[resource("orders")]).is_none()); + } + + #[test] + fn a_principal_with_manage_topics_is_allowed() { + let principal = principal(true, true); + assert!(authorize(Some(&principal), &[resource("orders")]).is_none()); + } + + #[test] + fn a_principal_without_manage_topics_is_denied_every_resource() { + let principal = principal(false, true); + let responses = authorize( + Some(&principal), + &[resource("orders"), resource("payments")], + ) + .expect("denied"); + assert_eq!(responses.len(), 2); + assert!( + responses + .iter() + .all(|r| r.error_code == ERROR_TOPIC_AUTHORIZATION_FAILED) + ); + } + + #[test] + fn unread_permissions_fail_closed() { + let principal = principal(true, false); + let responses = authorize(Some(&principal), &[resource("orders")]).expect("denied"); + assert_eq!(responses[0].error_code, ERROR_TOPIC_AUTHORIZATION_FAILED); + } +} diff --git a/gateways/kafka/src/protocol/handlers/create_topics.rs b/gateways/kafka/src/protocol/handlers/create_topics.rs index f9fbd42341..760d39bc62 100644 --- a/gateways/kafka/src/protocol/handlers/create_topics.rs +++ b/gateways/kafka/src/protocol/handlers/create_topics.rs @@ -24,7 +24,7 @@ use bytes::Bytes; use iggy::prelude::IggyError; use kafka_protocol::messages::create_topics_request::{CreatableReplicaAssignment, CreatableTopic}; use kafka_protocol::messages::create_topics_response::CreatableTopicResult; -use kafka_protocol::messages::{BrokerId, CreateTopicsRequest, CreateTopicsResponse, TopicName}; +use kafka_protocol::messages::{BrokerId, CreateTopicsRequest, CreateTopicsResponse}; use kafka_protocol::protocol::StrBytes; use tokio::time::Instant; @@ -38,7 +38,9 @@ use crate::protocol::api::{ ERROR_TOPIC_ALREADY_EXISTS, GatewayState, HandleOutcome, }; use crate::protocol::bounds_guard::validate_create_topics_shape; -use crate::protocol::handlers::topic_config::{MAX_CONFIG_TOPICS, topic_cap_message}; +use crate::protocol::handlers::topic_config::{ + MAX_CONFIG_TOPICS, find_duplicate_names, topic_cap_message, +}; use crate::protocol::handlers::{ decode_guarded, encode_message, handle_versioned_request, is_supported_version, respond_or_close, unsupported_version_response, @@ -116,13 +118,13 @@ pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> Hand } }; - let duplicate_names = find_duplicate_names(&req.topics); + let topic_names: Vec<&str> = req.topics.iter().map(|topic| topic.name.as_str()).collect(); + let duplicate_names = find_duplicate_names(topic_names.iter().copied()); - let distinct_bridge_backed: HashSet<&TopicName> = req - .topics + let distinct_bridge_backed: HashSet<&str> = topic_names .iter() - .map(|topic| &topic.name) - .filter(|name| !duplicate_names.contains(*name)) + .copied() + .filter(|name| !duplicate_names.contains(name)) .collect(); if distinct_bridge_backed.len() > MAX_CONFIG_TOPICS { tracing::warn!( @@ -178,13 +180,13 @@ async fn create_all_topics( bridge: &IggyBridge, api_version: i16, topics: &[CreatableTopic], - duplicate_names: &HashSet, + duplicate_names: &HashSet<&str>, validate_only: bool, deadline: Instant, ) -> Vec { let mut results = Vec::with_capacity(topics.len()); for topic in topics { - let result = if duplicate_names.contains(&topic.name) { + let result = if duplicate_names.contains(topic.name.as_str()) { CreatableTopicResult::default() .with_name(topic.name.clone()) .with_error_code(ERROR_INVALID_REQUEST) @@ -215,21 +217,6 @@ async fn create_all_topics( results } -/// Every topic name that appears more than once in `topics` - real Kafka -/// (`ControllerApis.createTopics`) refuses every occurrence of a duplicate name with -/// `INVALID_REQUEST` (42) and creates nothing for it, rather than creating the first occurrence -/// and reporting the rest as already existing. -fn find_duplicate_names(topics: &[CreatableTopic]) -> HashSet { - let mut seen = HashSet::with_capacity(topics.len()); - let mut duplicates = HashSet::new(); - for topic in topics { - if !seen.insert(topic.name.clone()) { - duplicates.insert(topic.name.clone()); - } - } - duplicates -} - /// Validates and, when the topic is not rejected outright, provisions one requested topic. /// /// Existence beats a local (config/shape) rejection, not the other way around: real Kafka's @@ -578,6 +565,8 @@ fn encode_inner(version: i16, topics: &[CreatableTopic], forced_error: i16) -> R #[cfg(test)] mod tests { + use kafka_protocol::messages::TopicName; + use super::*; fn topic_name(name: &str) -> TopicName { @@ -760,25 +749,6 @@ mod tests { ); } - #[test] - fn find_duplicate_names_finds_a_name_repeated_across_two_requested_topics() { - let topics = vec![creatable_topic(1, 1), creatable_topic(1, 1)]; - let duplicates = find_duplicate_names(&topics); - assert_eq!(duplicates, HashSet::from([topic_name("orders")])); - } - - #[test] - fn find_duplicate_names_is_empty_when_every_name_is_unique() { - let topics = vec![ - creatable_topic(1, 1), - CreatableTopic::default() - .with_name(topic_name("payments")) - .with_num_partitions(1) - .with_replication_factor(1), - ]; - assert!(find_duplicate_names(&topics).is_empty()); - } - #[test] fn clamp_request_timeout_rejects_a_zero_or_negative_value_up_to_the_floor() { assert_eq!(clamp_request_timeout(0), MIN_REQUEST_TIMEOUT); diff --git a/gateways/kafka/src/protocol/handlers/describe_configs.rs b/gateways/kafka/src/protocol/handlers/describe_configs.rs index 7f2dcc25c2..ed48a931bf 100644 --- a/gateways/kafka/src/protocol/handlers/describe_configs.rs +++ b/gateways/kafka/src/protocol/handlers/describe_configs.rs @@ -17,13 +17,14 @@ //! `DescribeConfigs` (API key 32). +use std::collections::HashSet; + use bytes::Bytes; -use iggy::prelude::{IggyExpiry, ResourceOptions}; +use iggy::prelude::IggyExpiry; use kafka_protocol::messages::DescribeConfigsRequest; use kafka_protocol::messages::describe_configs_request::DescribeConfigsResource; use kafka_protocol::messages::describe_configs_response::{ DescribeConfigsResourceResult, DescribeConfigsResponse, DescribeConfigsResult, - DescribeConfigsSynonym, }; use kafka_protocol::protocol::StrBytes; use tokio::time::Instant; @@ -39,12 +40,9 @@ use crate::protocol::api::{ use crate::protocol::bounds_guard::validate_describe_configs_shape; use crate::protocol::handlers::topic_config::{ CLEANUP_DOC, CLEANUP_POLICY, CLEANUP_POLICY_VALUE, CONFIG_SOURCE_DEFAULT, CONFIG_SOURCE_TOPIC, - CONFIG_TYPE_LONG, CONFIG_TYPE_STRING, IggyTopicKey, ListedKey, MILLIS_PER_HOUR, - MILLIS_PER_MINUTE, ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, RETENTION_DOC, RETENTION_HOURS, - RETENTION_HOURS_DOC, RETENTION_MINUTES, RETENTION_MINUTES_DOC, RETENTION_MS, RetentionMs, - RetentionSynonyms, bridge_failure, exceeds_topic_cap, find_duplicate_names, listed_keys, - message_expiry_is_explicit, name_reason, retention_in_unit, retention_ms, static_text, - topic_cap_message, + CONFIG_TYPE_LONG, CONFIG_TYPE_STRING, ListedKey, ONLY_TOPIC_RESOURCES, RESOURCE_TYPE_TOPIC, + RETENTION_DOC, RETENTION_MS, bridge_failure, listed_keys, name_reason, retention_ms, + topic_cap_and_duplicates, topic_cap_message, }; use crate::protocol::handlers::{ decode_guarded, encode_message, is_supported_version, respond_or_close, @@ -77,6 +75,31 @@ pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> Hand } }; + // Checked before the bridge-availability stub, not inside `describe_all`: the cap is a + // request-shape policy, not a controller-availability one, and a check that only runs once + // the bridge is up is never exercised by a serverless/stub test. + let (exceeds_cap, duplicate_names) = topic_cap_and_duplicates(&req.resources); + if exceeds_cap { + let message = StrBytes::from(topic_cap_message("DescribeConfigs")); + tracing::warn!("DescribeConfigs request addresses too many distinct topics; rejecting"); + let results = req + .resources + .iter() + .map(|resource| { + resource_error( + resource, + ERROR_POLICY_VIOLATION, + Some(message.clone()), + Vec::new(), + ) + }) + .collect(); + return respond_or_close( + encode_message(&response(results), api_version, 256), + "DescribeConfigs", + ); + } + let Some(bridge) = &state.bridge else { let results = req .resources @@ -90,7 +113,7 @@ pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> Hand }; let deadline = Instant::now() + REQUEST_DEADLINE; - let results = describe_all(state, bridge, &req, deadline).await; + let results = describe_all(bridge, &req, &duplicate_names, deadline).await; respond_or_close( encode_message(&response(results), api_version, 256), "DescribeConfigs", @@ -119,65 +142,11 @@ fn response(results: Vec) -> DescribeConfigsResponse { } async fn describe_all( - state: &GatewayState, bridge: &IggyBridge, req: &DescribeConfigsRequest, + duplicate_names: &HashSet<&str>, deadline: Instant, ) -> Vec { - if !req.unknown_tagged_fields.is_empty() { - let message = static_text("unknown tagged fields are not supported"); - return if req.resources.is_empty() { - vec![ - DescribeConfigsResult::default() - .with_error_code(ERROR_INVALID_REQUEST) - .with_error_message(Some(message)) - .with_resource_type(RESOURCE_TYPE_TOPIC) - .with_resource_name(StrBytes::from_static_str("")) - .with_configs(Vec::new()), - ] - } else { - req.resources - .iter() - .map(|resource| { - resource_error( - resource, - ERROR_INVALID_REQUEST, - Some(message.clone()), - Vec::new(), - ) - }) - .collect() - }; - } - - let topic_names: Vec<&str> = req - .resources - .iter() - .filter(|resource| resource.resource_type == RESOURCE_TYPE_TOPIC) - .map(|resource| resource.resource_name.as_str()) - .collect(); - if exceeds_topic_cap(topic_names.iter().copied()) { - let message = StrBytes::from(topic_cap_message("DescribeConfigs")); - tracing::warn!("DescribeConfigs request addresses too many distinct topics; rejecting"); - return req - .resources - .iter() - .map(|resource| { - resource_error( - resource, - ERROR_POLICY_VIOLATION, - Some(message.clone()), - Vec::new(), - ) - }) - .collect(); - } - - // Real Kafka refuses every occurrence of a duplicate resource name outright rather than - // answering it from a cached result that may have been filtered by a different - // `configuration_keys` list - the same choice CreateTopics makes for a repeated topic name. - let duplicate_names = find_duplicate_names(topic_names.iter().copied()); - let mut cache = StreamTopicCache::default(); let mut results = Vec::with_capacity(req.resources.len()); let mut deadline_exceeded = false; @@ -211,11 +180,9 @@ async fn describe_all( } let result = describe_one( - state, bridge, &mut cache, resource, - req.include_synonyms, req.include_documentation, deadline, ) @@ -229,27 +196,17 @@ async fn describe_all( } async fn describe_one( - state: &GatewayState, bridge: &IggyBridge, cache: &mut StreamTopicCache, resource: &DescribeConfigsResource, - include_synonyms: bool, include_documentation: bool, deadline: Instant, ) -> DescribeConfigsResult { - if !resource.unknown_tagged_fields.is_empty() { - return resource_error( - resource, - ERROR_INVALID_REQUEST, - Some(static_text("unknown tagged fields are not supported")), - Vec::new(), - ); - } if resource.resource_type != RESOURCE_TYPE_TOPIC { return resource_error( resource, ERROR_INVALID_REQUEST, - Some(static_text(ONLY_TOPIC_RESOURCES)), + Some(StrBytes::from_static_str(ONLY_TOPIC_RESOURCES)), Vec::new(), ); } @@ -280,22 +237,18 @@ async fn describe_one( } }; - let (stream, iggy_topic) = bridge.topic_identity(name); - let remembered = state.remembered_retention_synonyms(&IggyTopicKey::new(stream, iggy_topic)); let (configs, unrepresentable) = described_entries( snapshot.message_expiry, - &snapshot.options, + snapshot.message_expiry_explicit, resource.configuration_keys.as_deref(), - include_synonyms, include_documentation, - remembered, ); if unrepresentable { return resource_error( resource, ERROR_INVALID_CONFIG, - Some(static_text( + Some(StrBytes::from_static_str( "stored message expiry is not a whole number of milliseconds", )), configs, @@ -307,103 +260,57 @@ async fn describe_one( #[must_use] fn described_entries( message_expiry: IggyExpiry, - options: &ResourceOptions, + message_expiry_explicit: bool, configuration_keys: Option<&[StrBytes]>, - include_synonyms: bool, include_documentation: bool, - remembered: RetentionSynonyms, ) -> (Vec, bool) { - let explicit = message_expiry_is_explicit(options); - let retention = retention_ms(message_expiry, explicit); + let retention = retention_ms(message_expiry, message_expiry_explicit); let keys = listed_keys(configuration_keys); let mut unrepresentable = false; let mut configs = Vec::with_capacity(keys.len()); for key in keys { match key { ListedKey::Retention => { - let (value, source) = retention_parts(&retention, &mut unrepresentable); + let (value, source) = retention.as_ref().map_or_else( + |()| { + // A stored duration that cannot be shown as retention.ms is still the + // topic's expiry, not the never-expire default. + unrepresentable = true; + (None, CONFIG_SOURCE_TOPIC) + }, + |retention| (Some(retention.value.as_str()), retention.source), + ); configs.push(config_entry( RETENTION_MS, value, false, source, CONFIG_TYPE_LONG, - synonyms(include_synonyms, remembered, value, source), include_documentation.then_some(RETENTION_DOC), )); } - ListedKey::Minutes => configs.push(derived_retention_entry( - RETENTION_MINUTES, - MILLIS_PER_MINUTE, - &retention, - include_documentation.then_some(RETENTION_MINUTES_DOC), - &mut unrepresentable, - )), - ListedKey::Hours => configs.push(derived_retention_entry( - RETENTION_HOURS, - MILLIS_PER_HOUR, - &retention, - include_documentation.then_some(RETENTION_HOURS_DOC), - &mut unrepresentable, - )), ListedKey::Cleanup => configs.push(config_entry( CLEANUP_POLICY, Some(CLEANUP_POLICY_VALUE), true, CONFIG_SOURCE_DEFAULT, CONFIG_TYPE_STRING, - Vec::new(), include_documentation.then_some(CLEANUP_DOC), )), // A resource error makes clients drop every returned config, including // retention.ms. An unmodeled name is omitted and the resource stays successful. - ListedKey::Unknown(_) => {} + ListedKey::Unknown => {} } } (configs, unrepresentable) } -const fn retention_parts<'a>( - retention: &'a Result, - unrepresentable: &mut bool, -) -> (Option<&'a str>, i8) { - if let Ok(retention) = retention { - (Some(retention.value.as_str()), retention.source) - } else { - // A stored duration that cannot be shown as retention.ms is still the - // topic's expiry, not the never-expire default. - *unrepresentable = true; - (None, CONFIG_SOURCE_TOPIC) - } -} - -fn derived_retention_entry( - name: &str, - unit_ms: u64, - retention: &Result, - documentation: Option<&str>, - unrepresentable: &mut bool, -) -> DescribeConfigsResourceResult { - let (millis, source) = retention_parts(retention, unrepresentable); - let value = millis.and_then(|millis| retention_in_unit(millis, unit_ms)); - config_entry( - name, - value.as_deref(), - false, - source, - CONFIG_TYPE_LONG, - Vec::new(), - documentation, - ) -} - fn config_entry( name: &str, value: Option<&str>, read_only: bool, source: i8, config_type: i8, - synonyms: Vec, documentation: Option<&str>, ) -> DescribeConfigsResourceResult { DescribeConfigsResourceResult::default() @@ -412,45 +319,11 @@ fn config_entry( .with_read_only(read_only) .with_config_source(source) .with_is_sensitive(false) - .with_synonyms(synonyms) + .with_synonyms(Vec::new()) .with_config_type(config_type) .with_documentation(documentation.map(|text| StrBytes::from(text.to_string()))) } -fn synonyms( - include_synonyms: bool, - remembered: RetentionSynonyms, - retention_ms_value: Option<&str>, - source: i8, -) -> Vec { - if !include_synonyms || remembered.is_empty() { - return Vec::new(); - } - let mut listed = Vec::new(); - if remembered.minutes { - listed.push(synonym( - RETENTION_MINUTES, - retention_ms_value.and_then(|value| retention_in_unit(value, MILLIS_PER_MINUTE)), - source, - )); - } - if remembered.hours { - listed.push(synonym( - RETENTION_HOURS, - retention_ms_value.and_then(|value| retention_in_unit(value, MILLIS_PER_HOUR)), - source, - )); - } - listed -} - -fn synonym(name: &'static str, value: Option, source: i8) -> DescribeConfigsSynonym { - DescribeConfigsSynonym::default() - .with_name(StrBytes::from_static_str(name)) - .with_value(value.map(StrBytes::from)) - .with_source(source) -} - fn resource_error( resource: &DescribeConfigsResource, error_code: i16, @@ -478,8 +351,7 @@ mod tests { use super::described_entries; use crate::protocol::handlers::topic_config::{ - CLEANUP_POLICY, CONFIG_SOURCE_TOPIC, RETENTION_HOURS, RETENTION_MINUTES, RETENTION_MS, - RetentionSynonyms, parse_retention_ms, + CLEANUP_POLICY, RETENTION_MS, message_expiry_is_explicit, parse_retention_ms, }; fn topic_details(expiry: IggyExpiry, explicit: bool) -> TopicDetails { @@ -509,17 +381,13 @@ mod tests { fn entries( details: &TopicDetails, configuration_keys: Option<&[StrBytes]>, - include_synonyms: bool, include_documentation: bool, - remembered: RetentionSynonyms, ) -> (Vec, bool) { described_entries( details.message_expiry, - &details.options, + message_expiry_is_explicit(&details.options), configuration_keys, - include_synonyms, include_documentation, - remembered, ) } @@ -531,14 +399,10 @@ mod tests { } #[test] - fn minutes_and_hours_describe_as_retention_ms_with_synonyms_when_asked() { - let minutes = parse_retention_ms("120000").expect("2 minutes in ms"); - let remembered = RetentionSynonyms { - minutes: true, - hours: false, - }; - let details = topic_details(minutes, true); - let (configs, bad) = entries(&details, None, true, false, remembered); + fn default_describe_returns_retention_and_cleanup() { + let expiry = parse_retention_ms("120000").expect("2 minutes in ms"); + let details = topic_details(expiry, true); + let (configs, bad) = entries(&details, None, false); assert!(!bad); assert_eq!( configs @@ -548,146 +412,67 @@ mod tests { vec![RETENTION_MS, CLEANUP_POLICY] ); assert_eq!(value_of(&configs, RETENTION_MS), Some("120000")); - let retention = configs - .iter() - .find(|config| config.name.as_str() == RETENTION_MS) - .expect("retention.ms"); - assert_eq!(retention.synonyms.len(), 1); - assert_eq!(retention.synonyms[0].name.as_str(), RETENTION_MINUTES); - assert_eq!( - retention.synonyms[0].value.as_ref().map(StrBytes::as_str), - Some("2") - ); - assert_eq!(retention.synonyms[0].source, CONFIG_SOURCE_TOPIC); let cleanup = configs .iter() .find(|config| config.name.as_str() == CLEANUP_POLICY) .expect("cleanup"); assert!(cleanup.synonyms.is_empty()); - - let hours = parse_retention_ms("3600000").expect("1 hour in ms"); - let (configs, bad) = entries( - &topic_details(hours, true), - None, - true, - false, - RetentionSynonyms { - minutes: false, - hours: true, - }, - ); - assert!(!bad); - let retention = configs - .iter() - .find(|config| config.name.as_str() == RETENTION_MS) - .expect("retention.ms"); - assert_eq!( - retention.value.as_ref().map(StrBytes::as_str), - Some("3600000") - ); - assert_eq!(retention.synonyms.len(), 1); - assert_eq!(retention.synonyms[0].name.as_str(), RETENTION_HOURS); - assert_eq!( - retention.synonyms[0].value.as_ref().map(StrBytes::as_str), - Some("1") - ); } #[test] - fn synonyms_are_empty_when_not_requested_or_only_milliseconds_were_set() { + fn synonyms_are_always_empty() { let expiry = parse_retention_ms("120000").expect("ms"); let details = topic_details(expiry, true); - let remembered = RetentionSynonyms { - minutes: true, - hours: true, - }; - let (configs, _) = entries(&details, None, false, false, remembered); - assert!(configs.iter().all(|config| config.synonyms.is_empty())); - - let (configs, _) = entries(&details, None, true, false, RetentionSynonyms::default()); + let (configs, _) = entries(&details, None, false); assert!(configs.iter().all(|config| config.synonyms.is_empty())); } #[test] - fn requested_minutes_or_hours_return_that_name_from_stored_milliseconds() { - let expiry = parse_retention_ms("90000").expect("ms"); + fn requested_unknown_names_are_omitted_and_known_keys_remain() { + let expiry = parse_retention_ms("1500").expect("ms"); let details = topic_details(expiry, true); - let asked = [StrBytes::from_static_str(RETENTION_MINUTES)]; - let (configs, bad) = entries( - &details, - Some(&asked), - true, - false, - RetentionSynonyms { - minutes: true, - hours: false, - }, - ); + let asked = [ + StrBytes::from_static_str(RETENTION_MS), + StrBytes::from_static_str("no.such"), + StrBytes::from_static_str("retention.minutes"), + StrBytes::from_static_str("retention.hours"), + ]; + let (configs, bad) = entries(&details, Some(&asked), false); assert!(!bad); assert_eq!(configs.len(), 1); - assert_eq!(configs[0].name.as_str(), RETENTION_MINUTES); - assert_eq!(configs[0].value.as_ref().map(StrBytes::as_str), Some("1")); - assert!(configs[0].synonyms.is_empty()); - - let asked = [StrBytes::from_static_str(RETENTION_HOURS)]; - let never = topic_details(IggyExpiry::NeverExpire, true); - let (configs, bad) = entries( - &never, - Some(&asked), - false, - false, - RetentionSynonyms::default(), - ); - assert!(!bad); - assert_eq!(configs[0].name.as_str(), RETENTION_HOURS); - assert_eq!(configs[0].value.as_ref().map(StrBytes::as_str), Some("-1")); - } - - #[test] - fn stored_never_expire_reports_negative_one_for_a_remembered_hour_synonym() { - let (configs, bad) = entries( - &topic_details(IggyExpiry::NeverExpire, true), - None, - true, - false, - RetentionSynonyms { - minutes: false, - hours: true, - }, - ); - assert!(!bad); - let retention = configs - .iter() - .find(|config| config.name.as_str() == RETENTION_MS) - .expect("retention.ms"); - assert_eq!(retention.value.as_ref().map(StrBytes::as_str), Some("-1")); + assert_eq!(configs[0].name.as_str(), RETENTION_MS); assert_eq!( - retention.synonyms[0].value.as_ref().map(StrBytes::as_str), - Some("-1") + configs[0].value.as_ref().map(StrBytes::as_str), + Some("1500") ); } #[test] - fn an_unmodeled_key_is_omitted_and_known_keys_remain() { + fn a_name_repeated_in_the_request_is_returned_once() { let expiry = parse_retention_ms("1500").expect("ms"); let details = topic_details(expiry, true); let asked = [ StrBytes::from_static_str(RETENTION_MS), - StrBytes::from_static_str("no.such"), + StrBytes::from_static_str(RETENTION_MS), ]; - let (configs, bad) = entries( - &details, - Some(&asked), - false, - false, - RetentionSynonyms::default(), - ); - assert!(!bad); + let (configs, _) = entries(&details, Some(&asked), false); assert_eq!(configs.len(), 1); - assert_eq!(configs[0].name.as_str(), RETENTION_MS); - assert_eq!( - configs[0].value.as_ref().map(StrBytes::as_str), - Some("1500") - ); + } + + #[test] + fn never_expire_reports_negative_one() { + let (configs, bad) = entries(&topic_details(IggyExpiry::NeverExpire, true), None, false); + assert!(!bad); + assert_eq!(value_of(&configs, RETENTION_MS), Some("-1")); + } + + #[test] + fn documentation_is_attached_only_when_asked() { + let expiry = parse_retention_ms("1500").expect("ms"); + let details = topic_details(expiry, true); + let (configs, _) = entries(&details, None, true); + assert!(configs.iter().all(|config| config.documentation.is_some())); + let (configs, _) = entries(&details, None, false); + assert!(configs.iter().all(|config| config.documentation.is_none())); } } diff --git a/gateways/kafka/src/protocol/handlers/mod.rs b/gateways/kafka/src/protocol/handlers/mod.rs index 229d29b802..394040acc8 100644 --- a/gateways/kafka/src/protocol/handlers/mod.rs +++ b/gateways/kafka/src/protocol/handlers/mod.rs @@ -45,6 +45,7 @@ use kafka_protocol::messages::TransactionalId; use kafka_protocol::protocol::{Decodable, Encodable}; use tokio::runtime::{Handle, RuntimeFlavor}; +use crate::auth::AuthenticatedPrincipal; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::api::{ API_KEY_ALTER_CONFIGS, API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_DESCRIBE_CONFIGS, @@ -79,6 +80,7 @@ pub(crate) fn off_worker(heavy: bool, work: impl FnOnce() -> T) -> T { pub async fn dispatch( state: &GatewayState, connection: &ConnectionState, + principal: Option<&AuthenticatedPrincipal>, api_key: i16, api_version: i16, body: Bytes, @@ -91,7 +93,7 @@ pub async fn dispatch( API_KEY_API_VERSIONS => api_versions::handle(state, api_version, body).await, API_KEY_CREATE_TOPICS => create_topics::handle(state, api_version, body).await, API_KEY_DESCRIBE_CONFIGS => describe_configs::handle(state, api_version, body).await, - API_KEY_ALTER_CONFIGS => alter_configs::handle(state, api_version, body).await, + API_KEY_ALTER_CONFIGS => alter_configs::handle(state, principal, api_version, body).await, API_KEY_FIND_COORDINATOR => find_coordinator::handle(state, api_version, body).await, API_KEY_JOIN_GROUP => join_group::handle(state, api_version, body).await, API_KEY_HEARTBEAT => heartbeat::handle(state, api_version, body).await, diff --git a/gateways/kafka/src/protocol/handlers/topic_config.rs b/gateways/kafka/src/protocol/handlers/topic_config.rs index 8bd472c200..9da428bc5d 100644 --- a/gateways/kafka/src/protocol/handlers/topic_config.rs +++ b/gateways/kafka/src/protocol/handlers/topic_config.rs @@ -21,14 +21,16 @@ //! (`IggyDuration::from(u64)` / `as_micros`). `u64::MAX` is [`IggyExpiry::NeverExpire`] //! and `0` is [`IggyExpiry::ServerDefault`], so neither is a usable millisecond expiry. //! Kafka `retention.ms` is a decimal millisecond count, or `-1` when sealed segments -//! never expire. `retention.minutes` and `retention.hours` are the same duration, -//! converted to milliseconds before anything is stored. A positive count is capped -//! at `u32::MAX` seconds, matching [`IggyExpiry::from_str`]. +//! never expire. `retention.minutes` and `retention.hours` are not Kafka topic-level +//! configs - only the broker-level `log.retention.minutes`/`log.retention.hours` exist +//! in real Kafka - so both are rejected the same as any other unknown key. -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::str::FromStr; use iggy::prelude::{HeaderKey, IggyDuration, IggyExpiry, ResourceOptions, topic_option_keys}; +use kafka_protocol::messages::alter_configs_request::AlterConfigsResource; +use kafka_protocol::messages::describe_configs_request::DescribeConfigsResource; use kafka_protocol::protocol::StrBytes; use crate::bridge::BridgeError; @@ -49,18 +51,12 @@ pub const CONFIG_TYPE_STRING: i8 = 2; pub const CONFIG_TYPE_LONG: i8 = 5; pub const RETENTION_MS: &str = "retention.ms"; -pub const RETENTION_MINUTES: &str = "retention.minutes"; -pub const RETENTION_HOURS: &str = "retention.hours"; -pub const MILLIS_PER_MINUTE: u64 = 60_000; -pub const MILLIS_PER_HOUR: u64 = 3_600_000; pub const CLEANUP_POLICY: &str = "cleanup.policy"; pub const CLEANUP_POLICY_VALUE: &str = "delete"; pub const ONLY_TOPIC_RESOURCES: &str = "only topic resources are supported"; pub const RETENTION_DOC: &str = "How long a sealed segment is kept, in milliseconds. The active segment does not expire. -1 means sealed segments never expire."; -pub const RETENTION_MINUTES_DOC: &str = "How long a sealed segment is kept, in minutes, derived from retention.ms. The count truncates toward zero. -1 means sealed segments never expire."; -pub const RETENTION_HOURS_DOC: &str = "How long a sealed segment is kept, in hours, derived from retention.ms. The count truncates toward zero. -1 means sealed segments never expire."; pub const CLEANUP_DOC: &str = "Iggy deletes expired messages and does not compact a topic."; /// Distinct topic names one `CreateTopics`, `DescribeConfigs`, or `AlterConfigs` request @@ -77,10 +73,12 @@ pub const MAX_CONFIG_TOPICS: usize = 100; #[derive(Debug, Clone, PartialEq, Eq)] pub enum ListedKey { Retention, - Minutes, - Hours, Cleanup, - Unknown(String), + /// An unmodeled name, including `retention.minutes` and `retention.hours`. The name + /// itself is never read back out: `DescribeConfigs` omits every `Unknown` entry from + /// its response, so carrying the name here would only pay a `String` allocation for a + /// value no caller inspects. + Unknown, } /// Why one resource's alter cannot be stored. @@ -90,7 +88,6 @@ pub enum ConfigFault { Repeated(String), ReadOnly(&'static str), InvalidRetention, - ConflictingRetention, } impl ConfigFault { @@ -101,82 +98,13 @@ impl ConfigFault { Self::Repeated(name) => format!("config key '{name}' is repeated"), Self::ReadOnly(message) => (*message).to_string(), Self::InvalidRetention => format!( - "retention.ms, retention.minutes, and retention.hours must be -1 or a positive count whose duration is at most {} seconds", + "retention.ms must be -1 or a positive count whose duration is at most {} seconds", u32::MAX ), - Self::ConflictingRetention => "retention.ms, retention.minutes, and retention.hours must describe the same duration".to_string(), } } } -/// `retention.minutes` and `retention.hours` used by the last successful alter. -/// -/// Process-local, keyed by Iggy stream and topic, and not stored in Iggy. A -/// restart clears it. Deleting the topic outside this gateway does not. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct RetentionSynonyms { - pub minutes: bool, - pub hours: bool, -} - -impl RetentionSynonyms { - #[must_use] - pub const fn is_empty(self) -> bool { - !self.minutes && !self.hours - } -} - -/// Iggy stream and topic a Kafka name resolves to. Synonym memory uses this pair, -/// not the Kafka topic name. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct IggyTopicKey { - stream: String, - topic: String, -} - -impl IggyTopicKey { - #[must_use] - pub fn new(stream: impl Into, topic: impl Into) -> Self { - Self { - stream: stream.into(), - topic: topic.into(), - } - } -} - -/// Process memory of [`RetentionSynonyms`] per [`IggyTopicKey`]. -#[derive(Debug, Default)] -pub struct RetentionSynonymMemory { - topics: HashMap, -} - -impl RetentionSynonymMemory { - #[must_use] - pub fn remembered(&self, key: &IggyTopicKey) -> RetentionSynonyms { - self.topics.get(key).copied().unwrap_or_default() - } - - /// Remembers `synonyms` after a successful alter that stored an expiry. - /// - /// An empty set clears the key. That is an alter which used only - /// `retention.ms`. A failed alter, and a resource that stored nothing, - /// must not call this. - pub fn record_applied(&mut self, key: IggyTopicKey, synonyms: RetentionSynonyms) { - if synonyms.is_empty() { - self.topics.remove(&key); - } else { - self.topics.insert(key, synonyms); - } - } -} - -/// One duration to store, plus which synonym names the alter used. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PlannedRetention { - pub expiry: IggyExpiry, - pub synonyms: RetentionSynonyms, -} - /// `retention.ms` as Kafka should report it, plus the config source byte. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RetentionMs { @@ -242,34 +170,7 @@ pub fn retention_ms(expiry: IggyExpiry, explicit: bool) -> Result Result { - parse_scaled_retention(value, 1) -} - -/// `retention.ms` text scaled into `unit_ms` units. -/// -/// `-1` stays `-1`. Any other count is integer division toward zero, so a stored -/// `90000` reported as minutes is `1` while `retention.ms` stays `90000`. -#[must_use] -pub fn retention_in_unit(millis_text: &str, unit_ms: u64) -> Option { - let Ok(millis) = millis_text.parse::() else { - return None; - }; - if millis == -1 { - return Some("-1".to_string()); - } - if millis < 0 || unit_ms == 0 { - return None; - } - let Ok(millis) = u64::try_from(millis) else { - return None; - }; - Some((millis / unit_ms).to_string()) -} - -/// `count` is one unit of `unit_ms` milliseconds. `1` is `retention.ms`. -fn parse_scaled_retention(value: &str, unit_ms: u64) -> Result { let Ok(count) = value.parse::() else { return Err(ConfigFault::InvalidRetention); }; @@ -282,22 +183,19 @@ fn parse_scaled_retention(value: &str, unit_ms: u64) -> Result u64::MAX / unit_ms { - return Err(ConfigFault::InvalidRetention); - } - let millis = count * unit_ms; + // `count > 0` was just checked, so this conversion is lossless for every `i64`. + let millis = u64::try_from(count).map_err(|_| ConfigFault::InvalidRetention)?; // `IggyExpiry::from_str` rejects `as_secs() > u32::MAX`. The typed update path // does not, but a value the server's own parser refuses is not one this gateway // should store. if millis / 1_000 > u64::from(u32::MAX) { return Err(ConfigFault::InvalidRetention); } - let Some(micros) = millis.checked_mul(1_000) else { - return Err(ConfigFault::InvalidRetention); - }; - Ok(IggyExpiry::ExpireDuration(IggyDuration::from(micros))) + // `millis <= u32::MAX * 1_000` was just proven above, so `millis * 1_000` is far + // short of `u64::MAX` and cannot overflow. + Ok(IggyExpiry::ExpireDuration(IggyDuration::from( + millis * 1_000, + ))) } /// Whether `names` contains more than [`MAX_CONFIG_TOPICS`] distinct strings. @@ -317,8 +215,8 @@ pub fn topic_cap_message(api_name: &str) -> String { /// Names in `names` that occur more than once. /// /// Real Kafka refuses every occurrence of a duplicate resource name with `INVALID_REQUEST` -/// (42) rather than silently picking a winner - the same choice `CreateTopics`' own -/// `find_duplicate_names` makes for a repeated topic name in one batch. +/// (42) rather than silently picking a winner. Shared by `CreateTopics`, `DescribeConfigs` +/// and `AlterConfigs`, which all make the same choice for a repeated topic name in one batch. #[must_use] pub fn find_duplicate_names<'a>(names: impl IntoIterator) -> HashSet<&'a str> { let mut seen = HashSet::new(); @@ -331,6 +229,52 @@ pub fn find_duplicate_names<'a>(names: impl IntoIterator) -> Has duplicates } +/// A resource `DescribeConfigs` or `AlterConfigs` can filter down to the ones naming a topic, +/// and check for the cap and for a duplicate name. The two APIs' generated request resource +/// types have no shared trait of their own, so this is the smallest common surface letting +/// [`topic_cap_and_duplicates`] serve both without either handler re-walking its resource list. +pub trait TopicResource { + fn kafka_resource_type(&self) -> i8; + fn kafka_resource_name(&self) -> &str; +} + +impl TopicResource for AlterConfigsResource { + fn kafka_resource_type(&self) -> i8 { + self.resource_type + } + + fn kafka_resource_name(&self) -> &str { + self.resource_name.as_str() + } +} + +impl TopicResource for DescribeConfigsResource { + fn kafka_resource_type(&self) -> i8 { + self.resource_type + } + + fn kafka_resource_name(&self) -> &str { + self.resource_name.as_str() + } +} + +/// The cap/duplicate verdict shared by `DescribeConfigs` and `AlterConfigs`: both filter +/// resources down to [`RESOURCE_TYPE_TOPIC`] before deciding whether the request exceeds +/// [`MAX_CONFIG_TOPICS`] or repeats a name, and the business decision is identical even +/// though the two APIs render it into different response types. +#[must_use] +pub fn topic_cap_and_duplicates(resources: &[R]) -> (bool, HashSet<&str>) { + let names: Vec<&str> = resources + .iter() + .filter(|resource| resource.kafka_resource_type() == RESOURCE_TYPE_TOPIC) + .map(TopicResource::kafka_resource_name) + .collect(); + ( + exceeds_topic_cap(names.iter().copied()), + find_duplicate_names(names.iter().copied()), + ) +} + /// The reason a topic name failed Kafka's own naming rules, or a fixed fallback. #[must_use] pub fn name_reason(error: &BridgeError) -> String { @@ -340,26 +284,24 @@ pub fn name_reason(error: &BridgeError) -> String { } } -#[must_use] -pub const fn static_text(message: &'static str) -> StrBytes { - StrBytes::from_static_str(message) -} - /// Maps a bridge failure to a Kafka error code and client-facing message, logging the real /// cause server-side. /// /// `handler` names the API in the log line (`"DescribeConfigs"`/`"AlterConfigs"`); `action` is /// the present participle of what the bridge call was doing (`"reading"`/`"altering"`), reused /// in both the timeout log and the generic internal-error message. +/// +/// Only [`BridgeError::Timeout`] gets its own arm. `InvalidKafkaTopicName` and `SendLost` are +/// unreachable here: the handler already validated the Kafka name before calling the bridge (so +/// the bridge's own, redundant validation inside `get_kafka_topic`/`update_kafka_topic_message_expiry` +/// cannot fail), and `SendLost` is only ever produced by the Produce path (`iggy_bridge/produce.rs`), +/// never by a config read or write. Both fall into the generic arm below rather than keeping a +/// branch no call site here can reach. #[must_use] pub fn bridge_failure(error: &BridgeError, handler: &str, action: &str) -> (i16, Option) { let code = error.to_kafka_error_code(); match error { - BridgeError::InvalidKafkaTopicName { reason, .. } => { - tracing::debug!(reason, "{handler} rejected an invalid topic name"); - (code, Some(StrBytes::from(reason.clone()))) - } - BridgeError::Timeout | BridgeError::SendLost(_) => { + BridgeError::Timeout => { tracing::warn!(%error, "{handler} {action} exceeded the bridge deadline"); (code, None) } @@ -377,80 +319,55 @@ pub fn bridge_failure(error: &BridgeError, handler: &str, action: &str) -> (i16, /// Keys to return for one describe resource. /// -/// `None` and an empty list both mean `retention.ms` and `cleanup.policy`. -/// `retention.minutes` and `retention.hours` are listed only when the client -/// names them. Any other name is [`ListedKey::Unknown`]. `DescribeConfigs` omits -/// those names from the response and does not fail the resource. +/// `None` and an empty list both mean `retention.ms` and `cleanup.policy`. A name repeated +/// in `configuration_keys` is collapsed to one entry, in its first position. Any other name, +/// including `retention.minutes` and `retention.hours`, is [`ListedKey::Unknown`]. +/// `DescribeConfigs` omits those names from the response and does not fail the resource. #[must_use] pub fn listed_keys(configuration_keys: Option<&[StrBytes]>) -> Vec { let Some(keys) = configuration_keys.filter(|keys| !keys.is_empty()) else { return vec![ListedKey::Retention, ListedKey::Cleanup]; }; + let mut seen = HashSet::new(); keys.iter() + .filter(|key| seen.insert(key.as_str())) .map(|key| match key.as_str() { RETENTION_MS => ListedKey::Retention, - RETENTION_MINUTES => ListedKey::Minutes, - RETENTION_HOURS => ListedKey::Hours, CLEANUP_POLICY => ListedKey::Cleanup, - other => ListedKey::Unknown(other.to_string()), + _ => ListedKey::Unknown, }) .collect() } -/// The expiry to store, or `None` when the resource names no retention key. -/// -/// `retention.ms`, `retention.minutes`, and `retention.hours` are one duration. -/// Minutes and hours convert to milliseconds. Two names that convert to -/// different millisecond values fail the resource and select nothing. One -/// expiry is returned when they agree. [`PlannedRetention::synonyms`] is empty -/// when the resource used only `retention.ms`. +/// The expiry to store, or `None` when the resource names no `retention.ms` key. /// -/// One invalid or repeated key fails the whole resource. Nothing is selected -/// for storage in that case. `update_topic` stores the millisecond expiry, never -/// the minute or hour count. +/// One invalid, unknown, or repeated key fails the whole resource and selects nothing. +/// `update_topic` stores the millisecond expiry this returns. /// /// # Errors /// /// Returns the first [`ConfigFault`] in request order. pub fn plan_retention_update<'a>( configs: impl IntoIterator)>, -) -> Result, ConfigFault> { +) -> Result, ConfigFault> { let mut seen = HashSet::new(); let mut expiry = None; - let mut synonyms = RetentionSynonyms::default(); for (name, value) in configs { if !seen.insert(name) { return Err(ConfigFault::Repeated(name.to_string())); } - let unit_ms = retention_scale(name, &mut synonyms)?; - let Some(value) = value else { - return Err(ConfigFault::InvalidRetention); - }; - let parsed = parse_scaled_retention(value, unit_ms)?; - if let Some(existing) = expiry - && existing != parsed - { - return Err(ConfigFault::ConflictingRetention); - } - expiry = Some(parsed); - } - Ok(expiry.map(|expiry| PlannedRetention { expiry, synonyms })) -} - -fn retention_scale(name: &str, synonyms: &mut RetentionSynonyms) -> Result { - match name { - RETENTION_MS => Ok(1), - RETENTION_MINUTES => { - synonyms.minutes = true; - Ok(MILLIS_PER_MINUTE) - } - RETENTION_HOURS => { - synonyms.hours = true; - Ok(MILLIS_PER_HOUR) + match name { + RETENTION_MS => { + let Some(value) = value else { + return Err(ConfigFault::InvalidRetention); + }; + expiry = Some(parse_retention_ms(value)?); + } + CLEANUP_POLICY => return Err(ConfigFault::ReadOnly("cleanup.policy is read-only")), + other => return Err(ConfigFault::UnknownKey(other.to_string())), } - CLEANUP_POLICY => Err(ConfigFault::ReadOnly("cleanup.policy is read-only")), - other => Err(ConfigFault::UnknownKey(other.to_string())), } + Ok(expiry) } #[cfg(test)] @@ -519,11 +436,42 @@ mod tests { ]; assert_eq!( listed_keys(Some(&asked)), - vec![ - ListedKey::Cleanup, - ListedKey::Unknown("no.such".to_string()) - ] + vec![ListedKey::Cleanup, ListedKey::Unknown] + ); + } + + #[test] + fn listed_keys_collapses_a_name_repeated_in_the_request() { + let asked = [ + StrBytes::from_static_str(RETENTION_MS), + StrBytes::from_static_str(RETENTION_MS), + StrBytes::from_static_str(CLEANUP_POLICY), + ]; + assert_eq!( + listed_keys(Some(&asked)), + vec![ListedKey::Retention, ListedKey::Cleanup] + ); + } + + #[test] + fn retention_minutes_and_hours_are_unknown_keys() { + let asked = [ + StrBytes::from_static_str("retention.minutes"), + StrBytes::from_static_str("retention.hours"), + ]; + assert_eq!( + listed_keys(Some(&asked)), + vec![ListedKey::Unknown, ListedKey::Unknown] ); + let err = plan_retention_update([("retention.minutes", Some("2"))]) + .expect_err("not a real topic config"); + assert_eq!( + err, + ConfigFault::UnknownKey("retention.minutes".to_string()) + ); + let err = plan_retention_update([("retention.hours", Some("1"))]) + .expect_err("not a real topic config"); + assert_eq!(err, ConfigFault::UnknownKey("retention.hours".to_string())); } #[test] @@ -547,11 +495,8 @@ mod tests { assert!(matches!(err, ConfigFault::ReadOnly(_))); assert_eq!( - plan_retention_update([(RETENTION_MS, Some("-1"))]) - .expect("clear") - .expect("stores") - .expiry, - IggyExpiry::NeverExpire + plan_retention_update([(RETENTION_MS, Some("-1"))]).expect("clear"), + Some(IggyExpiry::NeverExpire) ); assert_eq!( plan_retention_update([] as [(&str, Option<&str>); 0]).expect("empty"), @@ -560,162 +505,10 @@ mod tests { } #[test] - fn minutes_and_hours_convert_to_milliseconds() { - let minutes = plan_retention_update([(RETENTION_MINUTES, Some("2"))]) - .expect("minutes") - .expect("stores"); - assert_eq!(u64::from(minutes.expiry), 2 * MILLIS_PER_MINUTE * 1_000); - assert!(minutes.synonyms.minutes); - assert!(!minutes.synonyms.hours); - let described = retention_ms(minutes.expiry, true).expect("representable"); - assert_eq!(described.value, "120000"); - assert_eq!( - retention_in_unit(&described.value, MILLIS_PER_MINUTE).as_deref(), - Some("2") - ); - - let hours = plan_retention_update([(RETENTION_HOURS, Some("1"))]) - .expect("hours") - .expect("stores"); - assert_eq!(u64::from(hours.expiry), MILLIS_PER_HOUR * 1_000); - assert!(hours.synonyms.hours); - let described = retention_ms(hours.expiry, true).expect("representable"); - assert_eq!(described.value, "3600000"); - assert_eq!( - retention_in_unit(&described.value, MILLIS_PER_HOUR).as_deref(), - Some("1") - ); - - let agreed = plan_retention_update([ - (RETENTION_MS, Some("3600000")), - (RETENTION_MINUTES, Some("60")), - (RETENTION_HOURS, Some("1")), - ]) - .expect("same duration") - .expect("one write"); - assert_eq!( - retention_ms(agreed.expiry, true) - .expect("representable") - .value, - "3600000" - ); - assert!(agreed.synonyms.minutes); - assert!(agreed.synonyms.hours); - - assert_eq!( - retention_in_unit("90000", MILLIS_PER_MINUTE).as_deref(), - Some("1"), - "synonym display truncates toward zero" - ); - assert_eq!( - retention_in_unit("-1", MILLIS_PER_HOUR).as_deref(), - Some("-1") - ); - } - - #[test] - fn synonym_memory_tracks_minutes_and_hours_and_clears_on_milliseconds_only() { - let mut memory = RetentionSynonymMemory::default(); - let orders = IggyTopicKey::new("kafka", "orders"); - let other_stream = IggyTopicKey::new("billing", "orders"); - let minutes = plan_retention_update([(RETENTION_MINUTES, Some("2"))]) - .expect("minutes") - .expect("stores"); - memory.record_applied(orders.clone(), minutes.synonyms); - assert!(memory.remembered(&orders).minutes); - assert!(memory.remembered(&other_stream).is_empty()); - - let ms_only = plan_retention_update([(RETENTION_MS, Some("1500"))]) - .expect("ms") - .expect("stores"); - assert!(ms_only.synonyms.is_empty()); - memory.record_applied(orders.clone(), ms_only.synonyms); - assert!(memory.remembered(&orders).is_empty()); - - assert_eq!( - listed_keys(None), - vec![ListedKey::Retention, ListedKey::Cleanup] - ); - let asked = [StrBytes::from_static_str(RETENTION_MINUTES)]; - assert_eq!(listed_keys(Some(&asked)), vec![ListedKey::Minutes]); - let asked = [StrBytes::from_static_str(RETENTION_HOURS)]; - assert_eq!(listed_keys(Some(&asked)), vec![ListedKey::Hours]); - } - - #[test] - fn disagreeing_retention_units_store_nothing_and_keep_remembered_names() { - let mut memory = RetentionSynonymMemory::default(); - let orders = IggyTopicKey::new("kafka", "orders"); - memory.record_applied( - orders.clone(), - RetentionSynonyms { - minutes: true, - hours: false, - }, - ); + fn a_repeated_retention_ms_key_fails_the_resource() { let err = - plan_retention_update([(RETENTION_MS, Some("90000")), (RETENTION_HOURS, Some("1"))]) - .expect_err("different durations"); - assert_eq!(err, ConfigFault::ConflictingRetention); - assert!(memory.remembered(&orders).minutes); - assert!(!memory.remembered(&orders).hours); - - let err = plan_retention_update([("retention.mins", Some("1"))]).expect_err("not a name"); - assert_eq!(err, ConfigFault::UnknownKey("retention.mins".to_string())); - assert!(memory.remembered(&orders).minutes); - } - - #[test] - fn retention_hours_negative_one_stores_never_expire() { - let planned = plan_retention_update([(RETENTION_HOURS, Some("-1"))]) - .expect("hours") - .expect("stores"); - assert_eq!(planned.expiry, IggyExpiry::NeverExpire); - assert!(planned.synonyms.hours); - assert!(!planned.synonyms.minutes); - let described = retention_ms(planned.expiry, true).expect("never expire"); - assert_eq!(described.value, "-1"); - assert_eq!(described.source, CONFIG_SOURCE_TOPIC); - } - - #[test] - fn retention_unit_overflow_is_invalid_and_not_stored() { - let mut memory = RetentionSynonymMemory::default(); - let orders = IggyTopicKey::new("kafka", "orders"); - memory.record_applied( - orders.clone(), - RetentionSynonyms { - minutes: false, - hours: true, - }, - ); - - let overflow_hours = (u64::MAX / MILLIS_PER_HOUR + 1).to_string(); - let err = plan_retention_update([(RETENTION_HOURS, Some(overflow_hours.as_str()))]) - .expect_err("overflow before multiply"); - assert_eq!(err, ConfigFault::InvalidRetention); - - let past_cap_hours = (u64::from(u32::MAX) / 3_600 + 1).to_string(); - let err = plan_retention_update([(RETENTION_HOURS, Some(past_cap_hours.as_str()))]) - .expect_err("seconds above u32::MAX"); - assert_eq!(err, ConfigFault::InvalidRetention); - - let past_cap_minutes = (u64::from(u32::MAX) / 60 + 1).to_string(); - let err = plan_retention_update([(RETENTION_MINUTES, Some(past_cap_minutes.as_str()))]) - .expect_err("seconds above u32::MAX"); - assert_eq!(err, ConfigFault::InvalidRetention); - - assert!(plan_retention_update([(RETENTION_MINUTES, Some("0"))]).is_err()); - assert!(plan_retention_update([(RETENTION_HOURS, Some("0001"))]).is_err()); - assert!(plan_retention_update([(RETENTION_MINUTES, Some("+1"))]).is_err()); - assert!(memory.remembered(&orders).hours); - assert!(!memory.remembered(&orders).minutes); - - let at_cap_hours = (u64::from(u32::MAX) / 3_600).to_string(); - let stored = plan_retention_update([(RETENTION_HOURS, Some(at_cap_hours.as_str()))]) - .expect("at the second cap") - .expect("stores"); - let seconds = u64::from(stored.expiry) / 1_000 / 1_000; - assert!(u32::try_from(seconds).is_ok()); + plan_retention_update([(RETENTION_MS, Some("1000")), (RETENTION_MS, Some("1000"))]) + .expect_err("repeated key"); + assert_eq!(err, ConfigFault::Repeated(RETENTION_MS.to_string())); } } diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 8315e73fa0..70ee83e2e1 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -550,6 +550,7 @@ async fn route_frame( handle_connection_request( ctx.state, ctx.connection, + principal.as_ref(), req.request_api_key, req.request_api_version, body, @@ -563,9 +564,11 @@ async fn route_frame( // peer repeat a rejected version forever, resetting the pre-authentication read budget // on every frame and holding a `max_connections` permit with it. sasl_state.count_api_versions_answer(); + // Pre-authentication by construction: no principal exists yet at this point. handle_connection_request( ctx.state, ctx.connection, + None, req.request_api_key, req.request_api_version, body, diff --git a/gateways/kafka/tests/configs_handler_tests.rs b/gateways/kafka/tests/configs_handler_tests.rs index 477fb43e5e..a74a5f5011 100644 --- a/gateways/kafka/tests/configs_handler_tests.rs +++ b/gateways/kafka/tests/configs_handler_tests.rs @@ -32,8 +32,8 @@ use bytes::Bytes; use iggy_gateway_kafka::group::{GroupCoordinator, GroupCoordinatorConfig}; use iggy_gateway_kafka::protocol::api::{ - BrokerAdvertise, ERROR_NOT_CONTROLLER, GatewayState, handle_request, is_supported_version, - supported_api_ranges, + BrokerAdvertise, ERROR_NOT_CONTROLLER, ERROR_POLICY_VIOLATION, GatewayState, handle_request, + is_supported_version, supported_api_ranges, }; use iggy_gateway_kafka::protocol::handlers::{alter_configs, describe_configs}; @@ -100,6 +100,7 @@ async fn alter_configs_without_a_bridge_returns_not_controller() { .with_validate_only(false); let outcome = alter_configs::handle( &state_without_bridge(), + None, ALTER_VERSION, encode(&request, ALTER_VERSION), ) @@ -128,3 +129,80 @@ async fn incremental_alter_configs_closes_the_connection() { let outcome = handle_request(44, 1, Bytes::new(), &BrokerAdvertise::default()).await; assert!(outcome.is_close()); } + +/// The over-100-distinct-topics cap is checked before the bridge-availability stub: with the +/// bridge off (as every test in this file is), a request that would previously fall straight +/// through to `NOT_CONTROLLER` must still be rejected `POLICY_VIOLATION` first. Before this, the +/// cap lived inside `describe_all`/`alter_all`, past the bridge-off early return, so no +/// bridge-off test could ever reach it. +#[tokio::test] +async fn describe_configs_over_the_cap_is_rejected_even_with_the_bridge_off() { + const OVER_CAP: usize = 101; + let resources = (0..OVER_CAP) + .map(|i| { + DescribeConfigsResource::default() + .with_resource_type(2) + .with_resource_name(StrBytes::from(format!("topic-{i}"))) + .with_configuration_keys(None) + }) + .collect(); + let request = DescribeConfigsRequest::default() + .with_resources(resources) + .with_include_synonyms(false) + .with_include_documentation(false); + let outcome = describe_configs::handle( + &state_without_bridge(), + DESCRIBE_VERSION, + encode(&request, DESCRIBE_VERSION), + ) + .await; + let response: DescribeConfigsResponse = decode( + outcome.expect_response("DescribeConfigs answers"), + DESCRIBE_VERSION, + ); + assert_eq!(response.results.len(), OVER_CAP); + assert!( + response + .results + .iter() + .all(|result| result.error_code == ERROR_POLICY_VIOLATION) + ); +} + +#[tokio::test] +async fn alter_configs_over_the_cap_is_rejected_even_with_the_bridge_off() { + const OVER_CAP: usize = 101; + let resources = (0..OVER_CAP) + .map(|i| { + AlterConfigsResource::default() + .with_resource_type(2) + .with_resource_name(StrBytes::from(format!("topic-{i}"))) + .with_configs(vec![ + AlterableConfig::default() + .with_name(StrBytes::from_static_str("retention.ms")) + .with_value(Some(StrBytes::from_static_str("1000"))), + ]) + }) + .collect(); + let request = AlterConfigsRequest::default() + .with_resources(resources) + .with_validate_only(false); + let outcome = alter_configs::handle( + &state_without_bridge(), + None, + ALTER_VERSION, + encode(&request, ALTER_VERSION), + ) + .await; + let response: AlterConfigsResponse = decode( + outcome.expect_response("AlterConfigs answers"), + ALTER_VERSION, + ); + assert_eq!(response.responses.len(), OVER_CAP); + assert!( + response + .responses + .iter() + .all(|r| r.error_code == ERROR_POLICY_VIOLATION) + ); +} diff --git a/gateways/kafka/tests/configs_real_bridge_tests.rs b/gateways/kafka/tests/configs_real_bridge_tests.rs index eec05fa99e..2f3ef32482 100644 --- a/gateways/kafka/tests/configs_real_bridge_tests.rs +++ b/gateways/kafka/tests/configs_real_bridge_tests.rs @@ -136,7 +136,7 @@ async fn alter( let request = AlterConfigsRequest::default() .with_resources(resources) .with_validate_only(validate_only); - let outcome = alter_configs::handle(state, version, encode(&request, version)).await; + let outcome = alter_configs::handle(state, None, version, encode(&request, version)).await; decode(outcome.expect_response("AlterConfigs answers"), version) } diff --git a/gateways/kafka/tests/fetch_real_bridge_tests.rs b/gateways/kafka/tests/fetch_real_bridge_tests.rs index 1340aedad6..ecf480ff0f 100644 --- a/gateways/kafka/tests/fetch_real_bridge_tests.rs +++ b/gateways/kafka/tests/fetch_real_bridge_tests.rs @@ -249,8 +249,14 @@ async fn fetch_on( request .encode(&mut body, VERSION) .expect("the request encodes at this version"); - let handled = - handle_connection_request(state, connection, API_KEY_FETCH, VERSION, body.freeze()); + let handled = handle_connection_request( + state, + connection, + None, + API_KEY_FETCH, + VERSION, + body.freeze(), + ); let mut response = tokio::time::timeout(PROMPT, handled) .await .expect("records or an error answer at once") diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index fefd0d7e13..665e4ef70c 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -868,10 +868,16 @@ async fn corrupt_describe_configs_truncated_body_returns_invalid_request_error() assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); } -/// A resource count past what the remaining frame can hold is rejected by the shape walk. +/// A resource count past what the remaining frame can hold is rejected by the shape walk. Uses +/// `i32::MAX` (the same boundary `metadata_v0_huge_topics_count_rejected` exercises in +/// `bounds_guard.rs`'s own test module), not a value `kafka_protocol`'s own decode would already +/// choke on first: a small frame declaring `100000` fails before this guard gets credit for +/// catching anything, since the crate's own `Vec::with_capacity` path runs out of bytes to read +/// well before it would allocate that much. `i32::MAX` is the actual amplification class this +/// module exists to stop (see its module doc's `handle_alloc_error` reproduction). #[tokio::test] async fn corrupt_describe_configs_overlong_count_returns_invalid_request_error() { - let body = Bytes::from_static(&[0x00, 0x01, 0x86, 0xA0]); + let body = Bytes::from_static(&[0x7F, 0xFF, 0xFF, 0xFF]); let resp = handle_request(API_KEY_DESCRIBE_CONFIGS, 1, body, &default_broker()) .await .expect_response("DescribeConfigs v1 has an encodable error response"); @@ -894,10 +900,12 @@ async fn corrupt_alter_configs_truncated_body_returns_invalid_request_error() { assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); } -/// A resource count past what the remaining frame can hold is rejected by the shape walk. +/// A resource count past what the remaining frame can hold is rejected by the shape walk. See +/// `corrupt_describe_configs_overlong_count_returns_invalid_request_error`'s doc for why +/// `i32::MAX`, not a smaller value `kafka_protocol`'s own decode would already fail on. #[tokio::test] async fn corrupt_alter_configs_overlong_count_returns_invalid_request_error() { - let body = Bytes::from_static(&[0x00, 0x01, 0x86, 0xA0]); + let body = Bytes::from_static(&[0x7F, 0xFF, 0xFF, 0xFF]); let resp = handle_request(API_KEY_ALTER_CONFIGS, 0, body, &default_broker()) .await .expect_response("AlterConfigs v0 has an encodable error response"); From c4afa1b97ead680930ab5c88f12125465b1eaa35 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 7 Oct 2026 16:53:17 -0400 Subject: [PATCH 08/13] Fixing Doc tests --- gateways/kafka/src/protocol/api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index d17a4aa475..5af935308e 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -420,7 +420,7 @@ pub async fn handle_request_bounded( /// [`handle_request_bounded`] for one request of `connection`. /// /// `principal` is `None` when SASL is off, or for a key the connection reached before -/// authenticating - only [`crate::server::route_frame`] ever has a principal to pass. +/// authenticating - only `crate::server::route_frame` ever has a principal to pass. pub async fn handle_connection_request( state: &GatewayState, connection: &ConnectionState, From 92295870ee7b0179eb8836984c0a27b514fccbdc Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 7 Oct 2026 17:16:12 -0400 Subject: [PATCH 09/13] Fixing mac clippy tests --- gateways/kafka/tests/configs_handler_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateways/kafka/tests/configs_handler_tests.rs b/gateways/kafka/tests/configs_handler_tests.rs index a74a5f5011..8658dc304b 100644 --- a/gateways/kafka/tests/configs_handler_tests.rs +++ b/gateways/kafka/tests/configs_handler_tests.rs @@ -81,7 +81,7 @@ async fn describe_configs_without_a_bridge_returns_not_controller() { ); assert_eq!(response.results.len(), 1); assert_eq!(response.results[0].error_code, ERROR_NOT_CONTROLLER); - assert!(response.results[0].configs.is_empty()); + assert_eq!(response.results[0].configs, [] as [kafka_protocol::messages::describe_configs_response::DescribeConfigsResourceResult; 0]); } #[tokio::test] From 1f644fa626a21355a3cf6045978348f40a0c8d07 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 7 Oct 2026 17:31:19 -0400 Subject: [PATCH 10/13] Fixing fmt error --- gateways/kafka/tests/configs_handler_tests.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gateways/kafka/tests/configs_handler_tests.rs b/gateways/kafka/tests/configs_handler_tests.rs index 8658dc304b..f2da4f3bcc 100644 --- a/gateways/kafka/tests/configs_handler_tests.rs +++ b/gateways/kafka/tests/configs_handler_tests.rs @@ -81,7 +81,11 @@ async fn describe_configs_without_a_bridge_returns_not_controller() { ); assert_eq!(response.results.len(), 1); assert_eq!(response.results[0].error_code, ERROR_NOT_CONTROLLER); - assert_eq!(response.results[0].configs, [] as [kafka_protocol::messages::describe_configs_response::DescribeConfigsResourceResult; 0]); + assert_eq!( + response.results[0].configs, + [] as [kafka_protocol::messages::describe_configs_response::DescribeConfigsResourceResult; + 0] + ); } #[tokio::test] From 41134ed7499f70baf08b133231e86da181849f40 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 7 Oct 2026 20:04:17 -0400 Subject: [PATCH 11/13] Fixing clippy and fmt error --- .../kafka/tests/configs_real_bridge_tests.rs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/gateways/kafka/tests/configs_real_bridge_tests.rs b/gateways/kafka/tests/configs_real_bridge_tests.rs index 2f3ef32482..7fa62837f2 100644 --- a/gateways/kafka/tests/configs_real_bridge_tests.rs +++ b/gateways/kafka/tests/configs_real_bridge_tests.rs @@ -27,7 +27,7 @@ use kafka_protocol::messages::describe_configs_request::{ DescribeConfigsRequest, DescribeConfigsResource, }; use kafka_protocol::messages::describe_configs_response::{ - DescribeConfigsResourceResult, DescribeConfigsResponse, + DescribeConfigsResourceResult, DescribeConfigsResponse, DescribeConfigsSynonym, }; use kafka_protocol::protocol::StrBytes; use serial_test::serial; @@ -174,13 +174,13 @@ async fn describe_configs_returns_retention_and_cleanup_and_alter_persists_reten assert!(!retention.read_only); assert!(!retention.is_sensitive); assert_eq!(retention.config_source, SOURCE_DEFAULT); - assert!(retention.synonyms.is_empty()); + assert_eq!(retention.synonyms, [] as [DescribeConfigsSynonym; 0]); assert!(retention.documentation.is_some()); let cleanup = entry(&result.configs, "cleanup.policy"); assert_eq!(cleanup.value.as_ref().map(StrBytes::as_str), Some("delete")); assert!(cleanup.read_only); assert_eq!(cleanup.config_source, SOURCE_DEFAULT); - assert!(cleanup.synonyms.is_empty()); + assert_eq!(cleanup.synonyms, [] as [DescribeConfigsSynonym; 0]); assert!(cleanup.documentation.is_some()); let without_docs = describe( @@ -193,7 +193,10 @@ async fn describe_configs_returns_retention_and_cleanup_and_alter_persists_reten .await; let bare = &without_docs.results[0].configs; assert!(entry(bare, "retention.ms").documentation.is_none()); - assert!(entry(bare, "retention.ms").synonyms.is_empty()); + assert_eq!( + entry(bare, "retention.ms").synonyms, + [] as [DescribeConfigsSynonym; 0] + ); let filtered = describe( &state, @@ -324,7 +327,10 @@ async fn describe_and_alter_configs_reject_unknown_nontopic_missing_and_invalid_ ) .await; assert_eq!(broker.results[0].error_code, ERROR_INVALID_REQUEST); - assert!(broker.results[0].configs.is_empty()); + assert_eq!( + broker.results[0].configs, + [] as [DescribeConfigsResourceResult; 0] + ); assert_eq!( broker.results[0] .error_message @@ -346,7 +352,10 @@ async fn describe_and_alter_configs_reject_unknown_nontopic_missing_and_invalid_ missing.results[0].error_code, ERROR_UNKNOWN_TOPIC_OR_PARTITION ); - assert!(missing.results[0].configs.is_empty()); + assert_eq!( + missing.results[0].configs, + [] as [DescribeConfigsResourceResult; 0] + ); let invalid = describe( &state, From 6f8a92783cb1bb03f9eef6c19119b81187b1922f Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 7 Oct 2026 20:51:57 -0400 Subject: [PATCH 12/13] Fixing clippy and fmt error --- gateways/kafka/src/protocol/handlers/describe_configs.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gateways/kafka/src/protocol/handlers/describe_configs.rs b/gateways/kafka/src/protocol/handlers/describe_configs.rs index ed48a931bf..54017c66ef 100644 --- a/gateways/kafka/src/protocol/handlers/describe_configs.rs +++ b/gateways/kafka/src/protocol/handlers/describe_configs.rs @@ -346,7 +346,9 @@ mod tests { CompressionAlgorithm, HeaderKey, HeaderValue, IggyByteSize, IggyExpiry, IggyTimestamp, MaxTopicSize, OptionValue, ResourceOptions, TopicDetails, topic_option_keys, }; - use kafka_protocol::messages::describe_configs_response::DescribeConfigsResourceResult; + use kafka_protocol::messages::describe_configs_response::{ + DescribeConfigsResourceResult, DescribeConfigsSynonym, + }; use kafka_protocol::protocol::StrBytes; use super::described_entries; @@ -416,7 +418,7 @@ mod tests { .iter() .find(|config| config.name.as_str() == CLEANUP_POLICY) .expect("cleanup"); - assert!(cleanup.synonyms.is_empty()); + assert_eq!(cleanup.synonyms, [] as [DescribeConfigsSynonym; 0]); } #[test] From ca7fa81eac8bc7048e3248bc4ee8a3b3a538f2b3 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 8 Oct 2026 13:41:19 -0400 Subject: [PATCH 13/13] Fixing the merge issues --- gateways/kafka/docs/SCOPE.md | 8 ++++---- gateways/kafka/src/bridge/iggy_bridge/mod.rs | 2 +- gateways/kafka/src/bridge/iggy_bridge/topics.rs | 4 ++-- gateways/kafka/src/bridge/mod.rs | 3 ++- gateways/kafka/src/protocol/api.rs | 4 ++-- gateways/kafka/src/protocol/bounds_guard.rs | 2 ++ gateways/kafka/src/protocol/handlers/mod.rs | 8 ++++---- gateways/kafka/tests/golden_wire_fixtures_tests.rs | 12 ++++++------ gateways/kafka/tests/version_firewall_tests.rs | 14 +++++++------- 9 files changed, 30 insertions(+), 27 deletions(-) diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index a5353f1236..649f2ba71c 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -4,11 +4,11 @@ A TCP listener on the Kafka wire port. It decodes requests, validates scoped API keys, versions and wire formats, and answers them. With a bridge, Produce, Fetch, ListOffsets, Metadata, CreateTopics, DescribeConfigs and AlterConfigs use Iggy. -**Stub semantics (important):** without a bridge, every API answers with a stub. Produce discards -the payload and answers with retriable `NOT_LEADER_OR_FOLLOWER` (6). Fetch and ListOffsets answer 6 -too. CreateTopics, DescribeConfigs and AlterConfigs answer `NOT_CONTROLLER` (41), so clients do not believe +**Stub semantics (important):** without a bridge, every API answers with a stub. Produce discards +the payload and answers with retriable `NOT_LEADER_OR_FOLLOWER` (6). Fetch and ListOffsets answer 6 +too. CreateTopics, DescribeConfigs and AlterConfigs answer `NOT_CONTROLLER` (41), so clients do not believe topics were created or configs were read or changed. OffsetCommit and OffsetFetch answer retriable `COORDINATOR_LOAD_IN_PROGRESS` -(14). Do not read `ec=0` from a stub as durable storage. Produce is the one API that stores records once you configure +(14). Do not read `ec=0` from a stub as durable storage. Produce is the one API that stores records once you configure a bridge ([#3535](https://github.com/apache/iggy/issues/3535)). | Deliverable | Status | Location | diff --git a/gateways/kafka/src/bridge/iggy_bridge/mod.rs b/gateways/kafka/src/bridge/iggy_bridge/mod.rs index 13003776a0..b0c8032ef7 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/mod.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/mod.rs @@ -38,9 +38,9 @@ mod topics; use fetch::{FetchPool, LazyClient}; pub(crate) use fetch::{FetchSlot, PartitionProbe, TopicProbe}; -pub use topics::{KafkaTopicMetadata, StreamTopicCache, TopicCreationOutcome, TopicLoad}; use group_offsets::OffsetPool; pub use group_offsets::{OFFSET_GROUP_PREFIX, OffsetCalls}; +pub use topics::{KafkaTopicMetadata, StreamTopicCache, TopicCreationOutcome, TopicLoad}; /// Passes attempted, after the first, before [`IggyBridge::connect`] gives up and returns `Err`. /// diff --git a/gateways/kafka/src/bridge/iggy_bridge/topics.rs b/gateways/kafka/src/bridge/iggy_bridge/topics.rs index 8c3d5dac44..b7dda3335e 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/topics.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/topics.rs @@ -20,8 +20,8 @@ use std::collections::{HashMap, HashSet}; use iggy::prelude::{ - Identifier,IggyClient, IggyError, IggyExpiry, StreamClient, Topic, TopicClient, TopicCreateOptions, - TopicDetails, TopicUpdateOptions, + Identifier, IggyClient, IggyError, IggyExpiry, StreamClient, Topic, TopicClient, + TopicCreateOptions, TopicDetails, TopicUpdateOptions, }; use kafka_protocol::protocol::StrBytes; use tokio::time::{Instant, timeout_at}; diff --git a/gateways/kafka/src/bridge/mod.rs b/gateways/kafka/src/bridge/mod.rs index 8be006ae66..6083448f4d 100644 --- a/gateways/kafka/src/bridge/mod.rs +++ b/gateways/kafka/src/bridge/mod.rs @@ -29,6 +29,7 @@ pub mod topic_map; pub use config::{DEFAULT_MAX_MESSAGE_SIZE, IggyBridgeConfig}; pub use error::BridgeError; pub use iggy_bridge::{ - IggyBridge, KafkaTopicMetadata, OFFSET_GROUP_PREFIX, OffsetCalls, StreamTopicCache, TopicCreationOutcome, TopicLoad, TopicTarget, + IggyBridge, KafkaTopicMetadata, OFFSET_GROUP_PREFIX, OffsetCalls, StreamTopicCache, + TopicCreationOutcome, TopicLoad, TopicTarget, }; pub use topic_map::{TopicMapping, TopicOverride}; diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 87e335b994..b0f27feb51 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -40,8 +40,8 @@ use crate::protocol::bounds_guard::{ use crate::protocol::handlers::init_producer_id::ProducerIdAllocator; use crate::protocol::handlers::{ alter_configs, api_versions, create_topics, decode_guarded, describe_configs, dispatch, fetch, - find_coordinator, heartbeat, init_producer_id, join_group, leave_group, list_offsets, metadata, offset_commit, offset_fetch, - produce, respond_or_close, sync_group, + find_coordinator, heartbeat, init_producer_id, join_group, leave_group, list_offsets, metadata, + offset_commit, offset_fetch, produce, respond_or_close, sync_group, }; use crate::protocol::probe_board::ProbeBoard; use crate::protocol::sasl::{ diff --git a/gateways/kafka/src/protocol/bounds_guard.rs b/gateways/kafka/src/protocol/bounds_guard.rs index 811500a309..3d9f237e8e 100644 --- a/gateways/kafka/src/protocol/bounds_guard.rs +++ b/gateways/kafka/src/protocol/bounds_guard.rs @@ -1909,6 +1909,8 @@ mod tests { fn alter_configs_v2_huge_compact_count_rejected() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF, 0xFF, 0x0F]); // u32::MAX, 5-byte varint assert!(validate_alter_configs_shape(2, &body, TEST_MAX_FRAME_SIZE).is_err()); + } + #[test] fn offset_commit_v2_null_metadata_accepted() { let body = Bytes::from_static(&[ diff --git a/gateways/kafka/src/protocol/handlers/mod.rs b/gateways/kafka/src/protocol/handlers/mod.rs index 4a7b2b6937..ea706f2219 100644 --- a/gateways/kafka/src/protocol/handlers/mod.rs +++ b/gateways/kafka/src/protocol/handlers/mod.rs @@ -55,10 +55,10 @@ use crate::error::{KafkaProtocolError, Result}; use crate::protocol::api::{ API_KEY_ALTER_CONFIGS, API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_DESCRIBE_CONFIGS, API_KEY_FETCH, API_KEY_FIND_COORDINATOR, API_KEY_HEARTBEAT, API_KEY_INIT_PRODUCER_ID, - API_KEY_JOIN_GROUP, API_KEY_LEAVE_GROUP, API_KEY_LIST_OFFSETS, API_KEY_METADATA, API_KEY_OFFSET_COMMIT, API_KEY_OFFSET_FETCH, - API_KEY_PRODUCE, API_KEY_SYNC_GROUP, ConnectionState, ERROR_INVALID_REQUEST, - ERROR_UNSUPPORTED_VERSION, GatewayState, HandleOutcome, is_supported_version, - supported_max_version, + API_KEY_JOIN_GROUP, API_KEY_LEAVE_GROUP, API_KEY_LIST_OFFSETS, API_KEY_METADATA, + API_KEY_OFFSET_COMMIT, API_KEY_OFFSET_FETCH, API_KEY_PRODUCE, API_KEY_SYNC_GROUP, + ConnectionState, ERROR_INVALID_REQUEST, ERROR_UNSUPPORTED_VERSION, GatewayState, HandleOutcome, + is_supported_version, supported_max_version, }; /// Record encodes and decodes of this many bytes or more run off the async worker. diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index 3ad2d27d7c..b020d6ea08 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -41,11 +41,11 @@ async fn golden_apiversions_v3_flexible_response_fixture() { .await .expect_response("test request has acks != 0 and expects a response"); - // error_code=0, api_count=14 (compact array: N+1=15) + // error_code=0, api_count=16 (compact array: N+1=17) // each entry followed by an empty tagged-fields byte; throttle_ms=0; top-level tagged fields - let expected: [u8; 106] = [ + let expected: [u8; 120] = [ 0x00, 0x00, // error_code - 0x0F, // compact array count (14+1) + 0x11, // compact array count (16+1) 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, // key 0: Produce 0-9 (advertised) 0x00, 0x01, 0x00, 0x04, 0x00, 0x0C, 0x00, // key 1: Fetch 4-12 0x00, 0x02, 0x00, 0x01, 0x00, 0x06, 0x00, // key 2: ListOffsets 1-6 @@ -75,10 +75,10 @@ async fn golden_apiversions_v1_response_fixture() { .await .expect_response("test request has acks != 0 and expects a response"); - // error_code=0, api_count=14; Produce advertises min=0 per KAFKA-18659; throttle_ms=0 - let expected: [u8; 94] = [ + // error_code=0, api_count=16; Produce advertises min=0 per KAFKA-18659; throttle_ms=0 + let expected: [u8; 106] = [ 0x00, 0x00, // error_code - 0x00, 0x00, 0x00, 0x0E, // api count = 14 + 0x00, 0x00, 0x00, 0x10, // api count = 16 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, // key 0: Produce 0-9 (advertised) 0x00, 0x01, 0x00, 0x04, 0x00, 0x0C, // key 1: Fetch 4-12 0x00, 0x02, 0x00, 0x01, 0x00, 0x06, // key 2: ListOffsets 1-6 diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index d0fc438dd9..31240a70a9 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -39,10 +39,10 @@ use tokio::net::TcpStream; use iggy_gateway_kafka::protocol::api::{ API_KEY_ALTER_CONFIGS, API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_DESCRIBE_CONFIGS, API_KEY_FETCH, API_KEY_FIND_COORDINATOR, API_KEY_HEARTBEAT, API_KEY_INIT_PRODUCER_ID, - API_KEY_JOIN_GROUP, API_KEY_LEAVE_GROUP, API_KEY_LIST_OFFSETS, API_KEY_METADATA, API_KEY_OFFSET_COMMIT, API_KEY_OFFSET_FETCH, - API_KEY_PRODUCE, API_KEY_SYNC_GROUP, ERROR_INVALID_REQUEST, ERROR_NONE, - ERROR_UNSUPPORTED_VERSION, advertised_min_version, handle_request, is_supported_version, - supported_api_ranges, + API_KEY_JOIN_GROUP, API_KEY_LEAVE_GROUP, API_KEY_LIST_OFFSETS, API_KEY_METADATA, + API_KEY_OFFSET_COMMIT, API_KEY_OFFSET_FETCH, API_KEY_PRODUCE, API_KEY_SYNC_GROUP, + ERROR_INVALID_REQUEST, ERROR_NONE, ERROR_UNSUPPORTED_VERSION, advertised_min_version, + handle_request, is_supported_version, supported_api_ranges, }; use codec::Decoder; @@ -66,8 +66,8 @@ use wire::{ }; #[test] -fn supported_ranges_table_has_fourteen_entries() { - assert_eq!(supported_api_ranges().len(), 14); +fn supported_ranges_table_has_sixteen_entries() { + assert_eq!(supported_api_ranges().len(), 16); } #[test] @@ -104,7 +104,7 @@ fn is_supported_version_matches_scope_table() { /// /// The wire rows are `supported_api_ranges()` sorted by `api_key` (`api_versions.rs`). /// `SCOPED_API_KEYS` is already in that ascending order, and this test reads the response -/// in that order. `supported_ranges_table_has_fourteen_entries` and +/// in that order. `supported_ranges_table_has_sixteen_entries` and /// `is_supported_version_matches_scope_table` pin that both tables cover the same keys. #[tokio::test] async fn apiversions_advertises_exact_supported_ranges_v1() {