[v26.1.x] rpk: use pagination when listing secrets from cloud

src/go/rpk/pkg/cli/security/secret/list.go

@@ -14,7 +14,6 @@ import (
 	"strings"
 
 	dataplanev1 "buf.build/gen/go/redpandadata/dataplane/protocolbuffers/go/redpanda/api/dataplane/v1"
-	"connectrpc.com/connect"
 	"github.com/redpanda-data/redpanda/src/go/rpk/pkg/config"
 	"github.com/redpanda-data/redpanda/src/go/rpk/pkg/out"
 	"github.com/redpanda-data/redpanda/src/go/rpk/pkg/publicapi"
@@ -48,17 +47,14 @@ func newListCommand(fs afero.Fs, p *config.Params) *cobra.Command {
 			cl, err := publicapi.NewDataPlaneClientSet(url, p.CurrentAuth().AuthToken)
 			out.MaybeDie(err, "unable to initialize cloud client: %v", err)
 
-			request := &dataplanev1.ListSecretsRequest{
-				Filter: &dataplanev1.ListSecretsFilter{
-					NameContains: nameContains,
-				},
-			}
-			response, err := cl.Secret.ListSecrets(cmd.Context(), connect.NewRequest(request))
+			secrets, err := cl.ListAllSecrets(cmd.Context(), &dataplanev1.ListSecretsFilter{
+				NameContains: nameContains,
+			})
 			out.MaybeDie(err, "unable to list secrets: %v", err)
 
 			tw := out.NewTable("NAME", "SCOPES")
 			defer tw.Flush()
-			for _, secret := range response.Msg.Secrets {
+			for _, secret := range secrets {
 				var secretScopes []string
 				for _, scope := range secret.Scopes {
 					name, ok := mapScopeToName()[scope]

src/go/rpk/pkg/cli/shadow/create.go

@@ -297,18 +297,15 @@ func validateCloudSecrets(ctx context.Context, prof *config.RpkProfile, slCfg *S
 	if err != nil {
 		return err
 	}
-	secrets, err := dpClient.Secret.ListSecrets(ctx, connect.NewRequest(&dataplanev1.ListSecretsRequest{
-		PageSize: 500, // 500 is a reasonable upper limit for now.
-		Filter: &dataplanev1.ListSecretsFilter{
-			Scopes: []dataplanev1.Scope{dataplanev1.Scope_SCOPE_REDPANDA_CLUSTER},
-		},
-	}))
+	secrets, err := dpClient.ListAllSecrets(ctx, &dataplanev1.ListSecretsFilter{
+		Scopes: []dataplanev1.Scope{dataplanev1.Scope_SCOPE_REDPANDA_CLUSTER},
+	})
 	if err != nil {
 		return fmt.Errorf("unable to list secrets at REDPANDA_CLUSTER scope: %v", err)
 	}
 
 	secretRefs := make(map[string]struct{})
-	for _, secret := range secrets.Msg.GetSecrets() {
+	for _, secret := range secrets {
 		secretRefs[fmt.Sprintf("%s%s}", secretsPrefix, secret.Id)] = struct{}{}
 	}
 

src/go/rpk/pkg/publicapi/BUILD

@@ -36,7 +36,15 @@ go_library(
 
 go_test(
     name = "publicapi_test",
-    srcs = ["controlplane_test.go"],
+    srcs = [
+        "controlplane_test.go",
+        "dataplane_test.go",
+    ],
     embed = [":publicapi"],
-    deps = ["@com_github_stretchr_testify//require"],
+    deps = [
+        "@build_buf_gen_go_redpandadata_dataplane_connectrpc_go//redpanda/api/dataplane/v1/dataplanev1connect",
+        "@build_buf_gen_go_redpandadata_dataplane_protocolbuffers_go//redpanda/api/dataplane/v1:dataplane",
+        "@com_connectrpc_connect//:connect",
+        "@com_github_stretchr_testify//require",
+    ],
 )

src/go/rpk/pkg/publicapi/dataplane.go

@@ -139,6 +139,25 @@ func DataplaneClientFromRpkProfile(p *config.RpkProfile, opts ...connect.ClientO
 	return NewDataPlaneClientSet(url, p.CurrentAuth().AuthToken, opts...)
 }
 
+// ListAllSecrets returns all the Secrets matching the given filter using the
+// pagination feature to traverse all pages of the list. The filter may be nil
+// to list every secret.
+func (dpCl *DataPlaneClientSet) ListAllSecrets(ctx context.Context, filter *dataplanev1.ListSecretsFilter) ([]*dataplanev1.Secret, error) {
+	fetchPage := func(ctx context.Context, pageToken string) ([]*dataplanev1.Secret, string, error) {
+		req := connect.NewRequest(&dataplanev1.ListSecretsRequest{
+			Filter:    filter,
+			PageToken: pageToken,
+			PageSize:  100,
+		})
+		resp, err := dpCl.Secret.ListSecrets(ctx, req)
+		if err != nil {
+			return nil, "", err
+		}
+		return resp.Msg.GetSecrets(), resp.Msg.GetNextPageToken(), nil
+	}
+	return Paginate(ctx, maxPages, fetchPage)
+}
+
 // ListAllShadowLinkTopics returns all the ShadowTopics for a given shadow link
 // using the pagination feature to traverse all pages of the list.
 func (dpCl *DataPlaneClientSet) ListAllShadowLinkTopics(ctx context.Context, shadowLinkName string) ([]*dataplanev1.ShadowTopic, error) {

src/go/rpk/pkg/publicapi/dataplane_test.go

@@ -0,0 +1,134 @@
+// Copyright 2026 Redpanda Data, Inc.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.md
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0
+
+package publicapi
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"buf.build/gen/go/redpandadata/dataplane/connectrpc/go/redpanda/api/dataplane/v1/dataplanev1connect"
+	dataplanev1 "buf.build/gen/go/redpandadata/dataplane/protocolbuffers/go/redpanda/api/dataplane/v1"
+	"connectrpc.com/connect"
+	"github.com/stretchr/testify/require"
+)
+
+// pagedSecretService serves totalSecrets secrets, honoring page_size and
+// page_token the way the Public API does, so that a client which ignores
+// pagination only ever sees the first page.
+type pagedSecretService struct {
+	dataplanev1connect.UnimplementedSecretServiceHandler
+
+	totalSecrets int
+	gotFilters   []*dataplanev1.ListSecretsFilter
+	gotPageSizes []int32
+}
+
+func (s *pagedSecretService) ListSecrets(_ context.Context, req *connect.Request[dataplanev1.ListSecretsRequest]) (*connect.Response[dataplanev1.ListSecretsResponse], error) {
+	s.gotFilters = append(s.gotFilters, req.Msg.GetFilter())
+	s.gotPageSizes = append(s.gotPageSizes, req.Msg.GetPageSize())
+
+	pageSize := int(req.Msg.GetPageSize())
+	if pageSize <= 0 {
+		pageSize = 100
+	}
+
+	// The page token is the index of the first secret in the page.
+	var start int
+	if token := req.Msg.GetPageToken(); token != "" {
+		if _, err := fmt.Sscanf(token, "%d", &start); err != nil {
+			return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("bad page token %q", token))
+		}
+	}
+
+	end := min(start+pageSize, s.totalSecrets)
+	secrets := make([]*dataplanev1.Secret, 0, end-start)
+	for i := start; i < end; i++ {
+		secrets = append(secrets, dataplanev1.Secret_builder{Id: fmt.Sprintf("SECRET_%d", i)}.Build())
+	}
+
+	var nextPageToken string
+	if end < s.totalSecrets {
+		nextPageToken = fmt.Sprintf("%d", end)
+	}
+	return connect.NewResponse(dataplanev1.ListSecretsResponse_builder{
+		Secrets:       secrets,
+		NextPageToken: nextPageToken,
+	}.Build()), nil
+}
+
+func newTestSecretClientSet(t *testing.T, svc *pagedSecretService) *DataPlaneClientSet {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.Handle(dataplanev1connect.NewSecretServiceHandler(svc))
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+
+	cl, err := NewDataPlaneClientSet(srv.URL, "test-token")
+	require.NoError(t, err)
+	return cl
+}
+
+func TestListAllSecrets(t *testing.T) {
+	// 250 secrets span three pages at a page size of 100. A client that does
+	// not follow next_page_token would stop at 100.
+	for _, tt := range []struct {
+		name         string
+		totalSecrets int
+		expRequests  int
+	}{
+		{name: "empty", totalSecrets: 0, expRequests: 1},
+		{name: "single partial page", totalSecrets: 42, expRequests: 1},
+		{name: "exactly one page", totalSecrets: 100, expRequests: 1},
+		{name: "beyond the first page", totalSecrets: 101, expRequests: 2},
+		{name: "several pages", totalSecrets: 250, expRequests: 3},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			svc := &pagedSecretService{totalSecrets: tt.totalSecrets}
+			cl := newTestSecretClientSet(t, svc)
+
+			secrets, err := cl.ListAllSecrets(context.Background(), nil)
+			require.NoError(t, err)
+			require.Len(t, secrets, tt.totalSecrets)
+			require.Len(t, svc.gotPageSizes, tt.expRequests)
+
+			// Every secret is returned exactly once, in order.
+			for i, secret := range secrets {
+				require.Equal(t, fmt.Sprintf("SECRET_%d", i), secret.GetId())
+			}
+			// A page size must be requested, otherwise we are at the mercy of
+			// the server default.
+			for _, pageSize := range svc.gotPageSizes {
+				require.NotZero(t, pageSize)
+			}
+		})
+	}
+}
+
+func TestListAllSecretsForwardsFilterToEveryPage(t *testing.T) {
+	svc := &pagedSecretService{totalSecrets: 250}
+	cl := newTestSecretClientSet(t, svc)
+
+	filter := dataplanev1.ListSecretsFilter_builder{
+		NameContains: "MY_SECRET",
+		Scopes:       []dataplanev1.Scope{dataplanev1.Scope_SCOPE_REDPANDA_CLUSTER},
+	}.Build()
+
+	_, err := cl.ListAllSecrets(context.Background(), filter)
+	require.NoError(t, err)
+
+	require.Len(t, svc.gotFilters, 3)
+	for _, got := range svc.gotFilters {
+		require.Equal(t, "MY_SECRET", got.GetNameContains())
+		require.Equal(t, []dataplanev1.Scope{dataplanev1.Scope_SCOPE_REDPANDA_CLUSTER}, got.GetScopes())
+	}
+}