diff --git a/README.md b/README.md index 35522cca9..8fa337762 100644 --- a/README.md +++ b/README.md @@ -52,3 +52,26 @@ You can obtain `longhornctl` either through downloading a prebuilt binary or by > - `longhornctl`: A command-line interface for remote Longhorn operations, designed to be run outside the Kubernetes cluster. It executes `longhornctl-local` for operations within the cluster. > - `longhornctl-local`: A command-line interface to be used within a DaemonSet pod inside the Kubernetes cluster, handling in-cluster and host operations. +### V2-only prerequisites + +The default `--data-engine-mode=v1-capable` retains iSCSI requirements. An explicitly selected `v2-only` mode omits iSCSI packages, modules and service operations, and requires SPDK checks and setup even without `--enable-spdk`. + +Use matching CLI binaries and a CLI image containing this capability. The remote command passes an explicit flag to the local executable so an older image fails instead of silently running V1 checks. + +```bash +longhornctl install preflight --data-engine-mode=v2-only --huge-page-size=2048 --allow-pci=none +longhornctl check preflight --data-engine-mode=v2-only --huge-page-size=2048 +``` + +Installation success is prerequisite setup, followed by a separate host qualification check. A required reboot remains unfinished installation; reboot and rerun installation before checking. Restart kubelet when necessary for its Node capacity to reflect the hugepage allocation. V2 check failures return a nonzero exit status after printing the complete report. + +V2 installation supports the known package-manager distributions. COS, Talos, unknown operating systems, unqualified NUMA layouts and host-provided SPDK startup hooks are rejected; configure immutable hosts through their own declarative prerequisites. The supported setup profile uses 2MiB pages on one existing NUMA node (node 0 by default). `HUGENODE` accepts one nonnegative integer. The requested memory, page size, PCI selection and driver must agree with any supplied `--spdk-options`; unknown, duplicate or conflicting options are rejected. Setup uses a bounded, cleared environment and verifies the actual allocation afterward. + +An empty PCI selection is fenced as `none`. Explicit device selection requires unique full PCI addresses; do not select devices still in use. Installer `--driver-override` accepts `vfio-pci` or `uio_pci_generic`; checker `--userspace-driver` uses the kernel module names `vfio_pci` or `uio_pci_generic`. + +V2 remote runs use separate resource names and collect complete reports from the identified DaemonSet and completed container instances. Cleanup deletes only objects created by that invocation, with UID preconditions. This mode neither converts existing volumes nor enables the manager's separate V2-only installation capability. Retire V1 data before removing its host prerequisites. + +Remote V2 installation records its invocation under `/var/lib/longhorn-cli/preflight-attempts` before changing host prerequisites. A Kubernetes retry or replacement Pod reports an error instead of repeating that invocation's host writes. The small attempt marker remains for audit and replay protection; assess any partial installation or required reboot before explicitly starting a new command. Failed and reboot-pending attempts are transported as error reports and the remote command exits nonzero. + +Replay protection requires a private installer-owned directory on a verified ext, XFS, Btrfs or F2FS filesystem. Volatile and unverified stores are refused. Directory entries and the marker are synced before host changes start. + diff --git a/cmd/local/subcmd/check.go b/cmd/local/subcmd/check.go index 689398744..3a4304ff7 100644 --- a/cmd/local/subcmd/check.go +++ b/cmd/local/subcmd/check.go @@ -8,9 +8,10 @@ import ( "github.com/spf13/cobra" "github.com/longhorn/cli/pkg/consts" - local "github.com/longhorn/cli/pkg/local/preflight" "github.com/longhorn/cli/pkg/types" "github.com/longhorn/cli/pkg/utils" + + local "github.com/longhorn/cli/pkg/local/preflight" ) func NewCmdCheck(globalOpts *types.GlobalCmdOptions) *cobra.Command { @@ -35,7 +36,11 @@ func newCmdCheckPreflight(globalOpts *types.GlobalCmdOptions) *cobra.Command { Long: `This command verifies your Kubernetes cluster environment to ensure it meets Longhorn's requirements. It performs a series of checks that can help identify potential issues that may prevent Longhorn from functioning correctly.`, PreRun: func(cmd *cobra.Command, args []string) { + if err := validateLocalDataEngineEnvironment(cmd, localChecker.DataEngineMode); err != nil { + utils.CheckErr(err) + } localChecker.LogLevel = globalOpts.LogLevel + localChecker.CollectionOnly = os.Getenv(consts.EnvPreflightCollectionOnly) == "true" if err := localChecker.Init(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to initialize preflight checker")) @@ -43,6 +48,10 @@ func newCmdCheckPreflight(globalOpts *types.GlobalCmdOptions) *cobra.Command { }, Run: func(cmd *cobra.Command, args []string) { + if localChecker.DataEngineMode == string(types.DataEngineModeV2Only) { + utils.CheckErr(runAndOutputV2Preflight(&localChecker)) + return + } if err := localChecker.Run(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to run preflight checker")) } @@ -51,6 +60,9 @@ func newCmdCheckPreflight(globalOpts *types.GlobalCmdOptions) *cobra.Command { }, PostRun: func(cmd *cobra.Command, args []string) { + if localChecker.DataEngineMode == string(types.DataEngineModeV2Only) { + return + } if err := localChecker.Output(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to output preflight checker collection")) } @@ -61,6 +73,7 @@ func newCmdCheckPreflight(globalOpts *types.GlobalCmdOptions) *cobra.Command { utils.SetGlobalOptionsLocal(cmd, globalOpts) + bindLocalDataEngineMode(cmd, &localChecker.DataEngineMode) cmd.Flags().StringVarP(&localChecker.OutputFilePath, consts.CmdOptOutputFile, "o", os.Getenv(consts.EnvOutputFilePath), "Output the result to a file, default to stdout.") cmd.Flags().BoolVar(&localChecker.EnableSpdk, consts.CmdOptEnableSpdk, utils.ConvertStringToTypeOrDefault(os.Getenv(consts.EnvEnableSpdk), false), "Enable checking of SPDK required packages, modules, and setup.") cmd.Flags().IntVar(&localChecker.HugePageSize, consts.CmdOptHugePageSize, utils.ConvertStringToTypeOrDefault(os.Getenv(consts.EnvHugePageSize), 2048), "Specify the huge page size in MiB for SPDK.") diff --git a/cmd/local/subcmd/check_v2.go b/cmd/local/subcmd/check_v2.go new file mode 100644 index 000000000..1fa832f65 --- /dev/null +++ b/cmd/local/subcmd/check_v2.go @@ -0,0 +1,15 @@ +package subcmd + +import "errors" + +type preflightReportRunner interface { + Run() error + Output() error +} + +// Write the complete report before returning the qualification verdict. In the +// remote transport phase the checker itself suppresses only that final verdict. +func runAndOutputV2Preflight(checker preflightReportRunner) error { + runErr := checker.Run() + return errors.Join(runErr, checker.Output()) +} diff --git a/cmd/local/subcmd/check_v2_test.go b/cmd/local/subcmd/check_v2_test.go new file mode 100644 index 000000000..9895ae125 --- /dev/null +++ b/cmd/local/subcmd/check_v2_test.go @@ -0,0 +1,28 @@ +package subcmd + +import ( + "errors" + "testing" +) + +type failedPreflightReport struct { + runErr, outputErr error + wrote bool +} + +func (report *failedPreflightReport) Run() error { return report.runErr } +func (report *failedPreflightReport) Output() error { + report.wrote = true + return report.outputErr +} + +func TestV2PreflightFailureStillOutputsReport(t *testing.T) { + qualificationErr, outputErr := errors.New("host qualification failed"), errors.New("report write failed") + for _, writeErr := range []error{nil, outputErr} { + report := &failedPreflightReport{runErr: qualificationErr, outputErr: writeErr} + err := runAndOutputV2Preflight(report) + if !report.wrote || !errors.Is(err, qualificationErr) || (writeErr != nil && !errors.Is(err, writeErr)) { + t.Fatalf("missing report or failure: wrote=%v error=%v", report.wrote, err) + } + } +} diff --git a/cmd/local/subcmd/data_engine_mode.go b/cmd/local/subcmd/data_engine_mode.go new file mode 100644 index 000000000..67d74ac68 --- /dev/null +++ b/cmd/local/subcmd/data_engine_mode.go @@ -0,0 +1,35 @@ +package subcmd + +import ( + "fmt" + "os" + "strconv" + + "github.com/spf13/cobra" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" +) + +func bindLocalDataEngineMode(cmd *cobra.Command, target *string) { + value := os.Getenv(consts.EnvDataEngineMode) + if value == "" { + value = string(types.DataEngineModeV1Capable) + } + cmd.Flags().StringVar(target, consts.CmdOptDataEngineMode, value, "Installation capability: v1-capable or v2-only. V2-only omits iSCSI and requires SPDK qualification.") +} + +func validateLocalDataEngineEnvironment(cmd *cobra.Command, value string) error { + mode, err := types.ParseDataEngineMode(value) + if err != nil { + return err + } + if mode == types.DataEngineModeV2Only && !cmd.Flags().Changed(consts.CmdOptHugePageSize) { + if raw, present := os.LookupEnv(consts.EnvHugePageSize); present { + if _, err := strconv.Atoi(raw); err != nil { + return fmt.Errorf("v2-only hugepage environment must contain an explicit integer") + } + } + } + return nil +} diff --git a/cmd/local/subcmd/install.go b/cmd/local/subcmd/install.go index 5fd4fe7c8..d10f01545 100644 --- a/cmd/local/subcmd/install.go +++ b/cmd/local/subcmd/install.go @@ -4,15 +4,15 @@ import ( "fmt" "os" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/pkg/errors" - "github.com/longhorn/cli/pkg/consts" - local "github.com/longhorn/cli/pkg/local/preflight" "github.com/longhorn/cli/pkg/types" "github.com/longhorn/cli/pkg/utils" + + local "github.com/longhorn/cli/pkg/local/preflight" ) func NewCmdInstall(globalOpts *types.GlobalCmdOptions) *cobra.Command { @@ -38,7 +38,11 @@ func newCmdInstallPreflight(globalOpts *types.GlobalCmdOptions) *cobra.Command { These dependencies ensure your Kubernetes cluster meets the requirements for successful Longhorn operation.`, PreRun: func(cmd *cobra.Command, args []string) { + if err := validateLocalDataEngineEnvironment(cmd, localInstaller.DataEngineMode); err != nil { + utils.CheckErr(err) + } localInstaller.LogLevel = globalOpts.LogLevel + localInstaller.AttemptID = os.Getenv(consts.EnvPreflightAttemptID) if err := localInstaller.Init(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to initialize preflight installer")) @@ -46,6 +50,15 @@ These dependencies ensure your Kubernetes cluster meets the requirements for suc }, Run: func(cmd *cobra.Command, args []string) { + if localInstaller.DataEngineMode == string(types.DataEngineModeV2Only) { + if os.Getenv(consts.EnvPreflightCollectionOnly) == "true" { + utils.CheckErr(localInstaller.RunForCollection()) + utils.CheckErr(localInstaller.Output()) + return + } + utils.CheckErr(runAndOutputV2Preflight(&localInstaller)) + return + } if err := localInstaller.Run(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to run preflight installer")) } @@ -54,6 +67,9 @@ These dependencies ensure your Kubernetes cluster meets the requirements for suc }, PostRun: func(cmd *cobra.Command, args []string) { + if localInstaller.DataEngineMode == string(types.DataEngineModeV2Only) { + return + } if err := localInstaller.Output(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to output preflight checker collection")) } @@ -64,6 +80,7 @@ These dependencies ensure your Kubernetes cluster meets the requirements for suc utils.SetGlobalOptionsLocal(cmd, globalOpts) + bindLocalDataEngineMode(cmd, &localInstaller.DataEngineMode) cmd.Flags().StringVarP(&localInstaller.OutputFilePath, consts.CmdOptOutputFile, "o", os.Getenv(consts.EnvOutputFilePath), "Output the result to a file, default to stdout.") cmd.Flags().BoolVar(&localInstaller.UpdatePackages, consts.CmdOptUpdatePackages, utils.ConvertStringToTypeOrDefault(os.Getenv(consts.EnvUpdatePackageList), true), "Update packages before installing required dependencies.") cmd.Flags().BoolVar(&localInstaller.EnableSpdk, consts.CmdOptEnableSpdk, utils.ConvertStringToTypeOrDefault(os.Getenv(consts.EnvEnableSpdk), false), "Enable installation of SPDK required packages, modules, and setup.") diff --git a/cmd/remote/subcmd/check.go b/cmd/remote/subcmd/check.go index 51235251a..c733dcaee 100644 --- a/cmd/remote/subcmd/check.go +++ b/cmd/remote/subcmd/check.go @@ -1,6 +1,8 @@ package subcmd import ( + stderrors "errors" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -58,6 +60,9 @@ INFO[2024-07-16T17:17:42+08:00] Cleaning up preflight checker INFO[2024-07-16T17:17:42+08:00] Completed preflight checker`, PreRun: func(cmd *cobra.Command, args []string) { + if err := preflightChecker.ValidateDataEngineMode(); err != nil { + utils.CheckErr(err) + } preflightChecker.Image = globalOpts.Image preflightChecker.ImageRegistry = globalOpts.ImageRegistry preflightChecker.ImagePullSecret = globalOpts.ImagePullSecret @@ -72,14 +77,23 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker`, } logrus.Info("Cleaning up preflight checker") - if err := preflightChecker.Cleanup(); err != nil { - utils.CheckErr(errors.Wrapf(err, "Failed to cleanup preflight checker")) + if preflightChecker.DataEngineMode != string(types.DataEngineModeV2Only) { + if err := preflightChecker.Cleanup(); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to cleanup preflight checker")) + } } }, Run: func(cmd *cobra.Command, args []string) { logrus.Info("Running preflight checker") output, err := preflightChecker.Run() + if preflightChecker.DataEngineMode == string(types.DataEngineModeV2Only) { + if output != "" { + logrus.Infof("Retrieved preflight checker result:\n%v", output) + } + utils.CheckErr(stderrors.Join(err, preflightChecker.Cleanup())) + return + } if err != nil { utils.CheckErr(errors.Wrap(err, "Failed to run preflight checker")) } @@ -88,6 +102,9 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker`, }, PostRun: func(cmd *cobra.Command, args []string) { + if preflightChecker.DataEngineMode == string(types.DataEngineModeV2Only) { + return + } logrus.Info("Cleaning up preflight checker") if err := preflightChecker.Cleanup(); err != nil { utils.CheckErr(errors.Wrapf(err, "Failed to cleanup preflight checker")) @@ -99,6 +116,7 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker`, utils.SetGlobalOptionsRemote(cmd, globalOpts) + cmd.Flags().StringVar(&preflightChecker.DataEngineMode, consts.CmdOptDataEngineMode, string(types.DataEngineModeV1Capable), "Installation capability: v1-capable or v2-only. V2-only omits iSCSI and requires SPDK qualification.") cmd.Flags().BoolVar(&preflightChecker.EnableSpdk, consts.CmdOptEnableSpdk, false, "Enable checking of SPDK required packages, modules, and setup.") cmd.Flags().IntVar(&preflightChecker.HugePageSize, consts.CmdOptHugePageSize, 2048, "Specify the huge page size in MiB for SPDK.") cmd.Flags().StringVar(&preflightChecker.UserspaceDriver, consts.CmdOptUserspaceDriver, "", "Userspace I/O driver for SPDK.") diff --git a/cmd/remote/subcmd/install.go b/cmd/remote/subcmd/install.go index 7e84f4b30..69e4812a7 100644 --- a/cmd/remote/subcmd/install.go +++ b/cmd/remote/subcmd/install.go @@ -3,6 +3,8 @@ package subcmd import ( "fmt" + stderrors "errors" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -64,6 +66,9 @@ If a reboot is required, the following message will be displayed: - Need to reboot the system and execute longhornctl install preflight again`, PreRun: func(cmd *cobra.Command, args []string) { + if err := preflightInstaller.ValidateDataEngineMode(); err != nil { + utils.CheckErr(err) + } preflightInstaller.Image = globalOpts.Image preflightInstaller.ImageRegistry = globalOpts.ImageRegistry preflightInstaller.ImagePullSecret = globalOpts.ImagePullSecret @@ -79,14 +84,24 @@ If a reboot is required, the following message will be displayed: } logrus.Info("Cleaning up preflight installer") - if err := preflightInstaller.Cleanup(); err != nil { - utils.CheckErr(errors.Wrapf(err, "Failed to cleanup preflight installer")) + if preflightInstaller.DataEngineMode != string(types.DataEngineModeV2Only) { + if err := preflightInstaller.Cleanup(); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to cleanup preflight installer")) + } } }, Run: func(cmd *cobra.Command, args []string) { logrus.Info("Running preflight installer") output, err := preflightInstaller.Run() + if preflightInstaller.DataEngineMode == string(types.DataEngineModeV2Only) { + if output != "" { + logrus.Infof("Retrieved preflight installer result:\n%v", output) + } + utils.CheckErr(stderrors.Join(err, preflightInstaller.Cleanup())) + logrus.Info("Completed V2 prerequisite setup; run check preflight --data-engine-mode=v2-only to qualify every host") + return + } if err != nil { utils.CheckErr(errors.Wrap(err, "Failed to run preflight installer")) } @@ -95,6 +110,9 @@ If a reboot is required, the following message will be displayed: }, PostRun: func(cmd *cobra.Command, args []string) { + if preflightInstaller.DataEngineMode == string(types.DataEngineModeV2Only) { + return + } logrus.Info("Cleaning up preflight installer") if preflightInstaller.OperatingSystem != string(consts.OperatingSystemContainerOptimizedOS) { if err := preflightInstaller.Cleanup(); err != nil { @@ -110,6 +128,7 @@ If a reboot is required, the following message will be displayed: utils.SetGlobalOptionsRemote(cmd, globalOpts) + cmd.Flags().StringVar(&preflightInstaller.DataEngineMode, consts.CmdOptDataEngineMode, string(types.DataEngineModeV1Capable), "Installation capability: v1-capable or v2-only. V2-only omits iSCSI and requires SPDK setup.") cmd.Flags().StringVar(&preflightInstaller.OperatingSystem, consts.CmdOptOperatingSystem, "", "Specify the operating system (\"\", cos). Leave this empty to use the package manager for installation.") cmd.Flags().BoolVar(&preflightInstaller.UpdatePackages, consts.CmdOptUpdatePackages, true, "Update packages before installing required dependencies.") cmd.Flags().BoolVar(&preflightInstaller.EnableSpdk, consts.CmdOptEnableSpdk, false, "Enable installation of SPDK required packages, modules, and setup.") diff --git a/deploy/check.yaml b/deploy/check.yaml index e7273d20c..2ad98dfcd 100644 --- a/deploy/check.yaml +++ b/deploy/check.yaml @@ -25,6 +25,8 @@ spec: securityContext: privileged: true env: + - name: DATA_ENGINE_MODE + value: "v1-capable" - name: ENABLE_SPDK value: "true" - name: HUGEMEM diff --git a/deploy/install.yaml b/deploy/install.yaml index 981541a1e..8873426a8 100644 --- a/deploy/install.yaml +++ b/deploy/install.yaml @@ -25,6 +25,8 @@ spec: securityContext: privileged: true env: + - name: DATA_ENGINE_MODE + value: "v1-capable" - name: UPDATE_PACKAGE_LIST value: "true" - name: ENABLE_SPDK diff --git a/docs/longhornctl.md b/docs/longhornctl.md index 64ff2a976..5df62253f 100644 --- a/docs/longhornctl.md +++ b/docs/longhornctl.md @@ -10,7 +10,7 @@ A CLI tool for troubleshooting and managing Longhorn operations. ``` -h, --help help for longhornctl - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -33,4 +33,4 @@ A CLI tool for troubleshooting and managing Longhorn operations. * [longhornctl trim](longhornctl_trim.md) - Longhorn trimming operations * [longhornctl version](longhornctl_version.md) - Print longhornctl version -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_check.md b/docs/longhornctl_check.md index d481efb59..6d4dd5ecd 100644 --- a/docs/longhornctl_check.md +++ b/docs/longhornctl_check.md @@ -6,7 +6,7 @@ Longhorn checking operations ``` -h, --help help for check - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Longhorn checking operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl check preflight](longhornctl_check_preflight.md) - Run a preflight check for Longhorn -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_check_preflight.md b/docs/longhornctl_check_preflight.md index 7bd2d2d66..87617c274 100644 --- a/docs/longhornctl_check_preflight.md +++ b/docs/longhornctl_check_preflight.md @@ -43,10 +43,11 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker ### Options ``` + --data-engine-mode string Installation capability: v1-capable or v2-only. V2-only omits iSCSI and requires SPDK qualification. (default "v1-capable") --enable-spdk Enable checking of SPDK required packages, modules, and setup. -h, --help help for preflight --huge-page-size int Specify the huge page size in MiB for SPDK. (default 2048) - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -66,4 +67,4 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker * [longhornctl check](longhornctl_check.md) - Longhorn checking operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_checksum.md b/docs/longhornctl_checksum.md index 109f8b37b..4bdf44b04 100644 --- a/docs/longhornctl_checksum.md +++ b/docs/longhornctl_checksum.md @@ -6,7 +6,7 @@ Snapshot checksum operations ``` -h, --help help for checksum - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Snapshot checksum operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl checksum volume](longhornctl_checksum_volume.md) - Trigger on-demand snapshot checksum calculation for a volume -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_checksum_volume.md b/docs/longhornctl_checksum_volume.md index 531cc3524..d0602fcd4 100644 --- a/docs/longhornctl_checksum_volume.md +++ b/docs/longhornctl_checksum_volume.md @@ -27,7 +27,7 @@ $ longhornctl checksum volume --name=v1 --namespace=longhorn-system ``` --all Apply to all volumes -h, --help help for volume - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -48,4 +48,4 @@ $ longhornctl checksum volume --name=v1 --namespace=longhorn-system * [longhornctl checksum](longhornctl_checksum.md) - Snapshot checksum operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_diagnose.md b/docs/longhornctl_diagnose.md index ddfc240b5..acde8bc36 100644 --- a/docs/longhornctl_diagnose.md +++ b/docs/longhornctl_diagnose.md @@ -49,4 +49,4 @@ $ longhornctl diagnose -o json 2>/dev/null | jq '.[] | select(.status == "fail") * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 29-Sep-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_doc.md b/docs/longhornctl_doc.md index 5ae9d6ba6..17c52d6f2 100644 --- a/docs/longhornctl_doc.md +++ b/docs/longhornctl_doc.md @@ -19,7 +19,7 @@ longhornctl doc [output directory] [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -33,4 +33,4 @@ longhornctl doc [output directory] [flags] * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_export.md b/docs/longhornctl_export.md index 43fd4bee4..f9ffaf4e9 100644 --- a/docs/longhornctl_export.md +++ b/docs/longhornctl_export.md @@ -6,7 +6,7 @@ Export Longhorn resources ``` -h, --help help for export - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Export Longhorn resources * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl export replica](longhornctl_export_replica.md) - Export data from a Longhorn replica -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_export_replica.md b/docs/longhornctl_export_replica.md index 8f155ee73..cc692b0c6 100644 --- a/docs/longhornctl_export_replica.md +++ b/docs/longhornctl_export_replica.md @@ -48,9 +48,9 @@ lost+found ``` --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") - --engine-image string Engine image to use to create volume from the replica. (default "longhornio/longhorn-engine:v1.13.0-dev") + --engine-image string Engine image to use to create volume from the replica. (default "longhornio/longhorn-engine:v1.14.0-dev") -h, --help help for replica - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -72,4 +72,4 @@ lost+found * [longhornctl export](longhornctl_export.md) - Export Longhorn resources * [longhornctl export replica stop](longhornctl_export_replica_stop.md) - Stop the replica export process -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_export_replica_stop.md b/docs/longhornctl_export_replica_stop.md index 0a4688f3f..eaf017629 100644 --- a/docs/longhornctl_export_replica_stop.md +++ b/docs/longhornctl_export_replica_stop.md @@ -22,7 +22,7 @@ INFO[2024-07-16T17:29:14+08:00] Successfully stopped exporting replica ``` -h, --help help for stop - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -41,4 +41,4 @@ INFO[2024-07-16T17:29:14+08:00] Successfully stopped exporting replica * [longhornctl export replica](longhornctl_export_replica.md) - Export data from a Longhorn replica -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_get.md b/docs/longhornctl_get.md index 4aaf8dce3..dbf7d93cf 100644 --- a/docs/longhornctl_get.md +++ b/docs/longhornctl_get.md @@ -6,7 +6,7 @@ Longhorn information gathering operations ``` -h, --help help for get - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Longhorn information gathering operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl get replica](longhornctl_get_replica.md) - Retrieve Longhorn replica information -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_get_replica.md b/docs/longhornctl_get_replica.md index ac36c15bf..e078bd85a 100644 --- a/docs/longhornctl_get_replica.md +++ b/docs/longhornctl_get_replica.md @@ -49,7 +49,7 @@ INFO[2024-07-16T17:23:51+08:00] Completed replica getter ``` --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") -h, --help help for replica - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -70,4 +70,4 @@ INFO[2024-07-16T17:23:51+08:00] Completed replica getter * [longhornctl get](longhornctl_get.md) - Longhorn information gathering operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_global-options.md b/docs/longhornctl_global-options.md index 49c891b90..0cb9d9ccc 100644 --- a/docs/longhornctl_global-options.md +++ b/docs/longhornctl_global-options.md @@ -19,7 +19,7 @@ longhornctl global-options [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -33,4 +33,4 @@ longhornctl global-options [flags] * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_install.md b/docs/longhornctl_install.md index e57924769..14b4938c6 100644 --- a/docs/longhornctl_install.md +++ b/docs/longhornctl_install.md @@ -6,7 +6,7 @@ Longhorn installation operations ``` -h, --help help for install - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Longhorn installation operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl install preflight](longhornctl_install_preflight.md) - Install Longhorn preflight -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_install_preflight.md b/docs/longhornctl_install_preflight.md index 34a488417..d818b19e9 100644 --- a/docs/longhornctl_install_preflight.md +++ b/docs/longhornctl_install_preflight.md @@ -47,11 +47,12 @@ If a reboot is required, the following message will be displayed: ``` --allow-pci string Specify a comma-separated (,) list of allowed PCI devices. By default, all PCI devices are blocked by a non-valid address. (default "none") + --data-engine-mode string Installation capability: v1-capable or v2-only. V2-only omits iSCSI and requires SPDK setup. (default "v1-capable") --driver-override string Userspace driver for device bindings. Override default driver for PCI devices. --enable-spdk Enable installation of SPDK required packages, modules, and setup. -h, --help help for preflight --huge-page-size int Specify the huge page size in MiB for SPDK. (default 2048) - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -76,4 +77,4 @@ If a reboot is required, the following message will be displayed: * [longhornctl install](longhornctl_install.md) - Longhorn installation operations * [longhornctl install preflight stop](longhornctl_install_preflight_stop.md) - Stop Longhorn preflight installer -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_install_preflight_stop.md b/docs/longhornctl_install_preflight_stop.md index 7bb5433d6..2ae95cf7a 100644 --- a/docs/longhornctl_install_preflight_stop.md +++ b/docs/longhornctl_install_preflight_stop.md @@ -22,7 +22,7 @@ INFO[2024-07-16T17:21:32+08:00] Successfully stopped preflight installer ``` -h, --help help for stop - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -42,4 +42,4 @@ INFO[2024-07-16T17:21:32+08:00] Successfully stopped preflight installer * [longhornctl install preflight](longhornctl_install_preflight.md) - Install Longhorn preflight -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_trim.md b/docs/longhornctl_trim.md index acb1d6145..beb126787 100644 --- a/docs/longhornctl_trim.md +++ b/docs/longhornctl_trim.md @@ -6,7 +6,7 @@ Longhorn trimming operations ``` -h, --help help for trim - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Longhorn trimming operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl trim volume](longhornctl_trim_volume.md) - Trim a Longhorn volume -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_trim_volume.md b/docs/longhornctl_trim_volume.md index 0990cc383..7b0bd3939 100644 --- a/docs/longhornctl_trim_volume.md +++ b/docs/longhornctl_trim_volume.md @@ -31,7 +31,7 @@ INFO[2024-07-16T17:32:01+08:00] Completed volume trimmer vo ``` -h, --help help for volume - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -51,4 +51,4 @@ INFO[2024-07-16T17:32:01+08:00] Completed volume trimmer vo * [longhornctl trim](longhornctl_trim.md) - Longhorn trimming operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_version.md b/docs/longhornctl_version.md index 4e97b8640..db01103ae 100644 --- a/docs/longhornctl_version.md +++ b/docs/longhornctl_version.md @@ -15,7 +15,7 @@ longhornctl version [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -29,4 +29,4 @@ longhornctl version [flags] * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/pkg/consts/cmd.go b/pkg/consts/cmd.go index 94b3175a8..1846d3b7b 100644 --- a/pkg/consts/cmd.go +++ b/pkg/consts/cmd.go @@ -47,6 +47,7 @@ const ( CmdOptNodeSelector = "node-selector" CmdOptTolerations = "tolerations" CmdOptAll = "all" + CmdOptDataEngineMode = "data-engine-mode" // SPDK options CmdOptAllowPci = "allow-pci" diff --git a/pkg/consts/environment.go b/pkg/consts/environment.go index b982f6283..304732a1f 100644 --- a/pkg/consts/environment.go +++ b/pkg/consts/environment.go @@ -7,10 +7,13 @@ const ( ) const ( - EnvCurrentNodeID = "CURRENT_NODE_ID" - EnvKubeConfigPath = "KUBECONFIG" - EnvLogLevel = "LOG_LEVEL" - EnvOutputFilePath = "OUTPUT_FILE_PATH" + EnvCurrentNodeID = "CURRENT_NODE_ID" + EnvKubeConfigPath = "KUBECONFIG" + EnvLogLevel = "LOG_LEVEL" + EnvOutputFilePath = "OUTPUT_FILE_PATH" + EnvDataEngineMode = "DATA_ENGINE_MODE" + EnvPreflightCollectionOnly = "PREFLIGHT_COLLECTION_ONLY" + EnvPreflightAttemptID = "PREFLIGHT_ATTEMPT_ID" EnvLonghornDataDirectory = "LONGHORN_DATA_DIRECTORY" EnvLonghornNamespace = "LONGHORN_NAMESPACE" diff --git a/pkg/local/preflight/checker.go b/pkg/local/preflight/checker.go index ca366fbeb..092d05b01 100644 --- a/pkg/local/preflight/checker.go +++ b/pkg/local/preflight/checker.go @@ -18,9 +18,7 @@ import ( kubeclient "k8s.io/client-go/kubernetes" commonkube "github.com/longhorn/go-common-libs/kubernetes" - commonnfs "github.com/longhorn/go-common-libs/nfs" commonns "github.com/longhorn/go-common-libs/ns" - commonsys "github.com/longhorn/go-common-libs/sys" commontypes "github.com/longhorn/go-common-libs/types" lhmgrutil "github.com/longhorn/longhorn-manager/util" @@ -41,7 +39,12 @@ type Checker struct { OutputFilePath string - kubeClient *kubeclient.Clientset + kubeClient kubeclient.Interface + nfsFacts checkerNFSFacts + + // CollectionOnly is the internal in-Pod transport phase. It preserves the + // full report for the following reporter; the remote CLI owns its verdict. + CollectionOnly bool osRelease string packageManager pkgmgr.PackageManager @@ -57,6 +60,9 @@ type Checker struct { // Init initializes the Checker. func (local *Checker) Init() error { + if err := local.ValidateDataEngineMode(); err != nil { + return err + } local.collection.Log = &types.LogCollection{} config, err := commonkube.GetInClusterConfig() @@ -74,9 +80,17 @@ func (local *Checker) Init() error { return errors.Wrap(err, "failed to get OS release") } local.osRelease = osRelease + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + if err := validateV2HostOS(osRelease); err != nil { + return err + } + } local.logger = logrus.WithField("os", local.osRelease) if local.osRelease == fmt.Sprint(consts.OperatingSystemContainerOptimizedOS) { + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + return fmt.Errorf("V2-only preflight is not qualified for Container-Optimized OS") + } return nil } @@ -100,10 +114,13 @@ func (local *Checker) Init() error { if err != nil { return err } + local.packageManager = packageManager + return local.configureDependencies(packageManagerType) +} +func (local *Checker) configureDependencies(packageManagerType pkgmgr.PackageManagerType) error { switch packageManagerType { case pkgmgr.PackageManagerApt: - local.packageManager = packageManager local.packages = []string{ "nfs-common", "open-iscsi", "cryptsetup", "dmsetup", } @@ -121,7 +138,6 @@ func (local *Checker) Init() error { } case pkgmgr.PackageManagerYum: - local.packageManager = packageManager local.packages = []string{ "nfs-utils", "iscsi-initiator-utils", "cryptsetup", "device-mapper", } @@ -139,7 +155,6 @@ func (local *Checker) Init() error { } case pkgmgr.PackageManagerZypper, pkgmgr.PackageManagerTransactionalUpdate, pkgmgr.PackageManagerBinary: - local.packageManager = packageManager local.packages = []string{ "nfs-client", "open-iscsi", "cryptsetup", "device-mapper", } @@ -157,7 +172,6 @@ func (local *Checker) Init() error { } case pkgmgr.PackageManagerPacman: - local.packageManager = packageManager local.packages = []string{ "nfs-utils", "open-iscsi", "cryptsetup", "device-mapper", } @@ -175,7 +189,11 @@ func (local *Checker) Init() error { } default: - return errors.Errorf("operating system (%v) package manager (%s) is not supported", osRelease, packageManagerType) + return errors.Errorf("operating system (%v) package manager (%s) is not supported", local.osRelease, packageManagerType) + } + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + local.packages = withoutISCSIStrings(local.packages) + local.modules = withoutISCSIStrings(local.modules) } return nil @@ -183,6 +201,22 @@ func (local *Checker) Init() error { // Run executes the preflight checks. func (local *Checker) Run() error { + if err := local.ValidateDataEngineMode(); err != nil { + return err + } + mode, err := types.ParseDataEngineMode(local.DataEngineMode) + if err != nil { + return err + } + if mode == types.DataEngineModeV2Only { + if err := validateV2HostOS(local.osRelease); err != nil { + return err + } + if local.osRelease == fmt.Sprint(consts.OperatingSystemContainerOptimizedOS) || local.packageManager == nil || local.kubeClient == nil || local.collection.Log == nil { + return fmt.Errorf("V2-only preflight requires a qualified initialized package-manager host") + } + local.collection.DataEngineMode = string(mode) + } checkTasks := []func() error{ local.checkKubeDNS, } @@ -194,15 +228,17 @@ func (local *Checker) Run() error { local.checkContainerOptimizedOS, ) default: + if mode.RequiresISCSI() { + checkTasks = append(checkTasks, local.checkIscsidService) + } checkTasks = append(checkTasks, - local.checkIscsidService, local.checkMultipathService, local.checkNFSv4Support, func() error { return local.checkPackagesInstalled(false) }, func() error { return local.checkModulesLoaded(false) }, ) - if local.EnableSpdk { + if mode.NeedsSPDK(local.EnableSpdk) { instructionSets := map[string][]string{ "amd64": {"sse4_2"}, "arm64": {"asimd"}, @@ -227,9 +263,32 @@ func (local *Checker) Run() error { } } + if mode == types.DataEngineModeV2Only && !local.CollectionOnly { + return local.QualificationError() + } return nil } +func (local *Checker) QualificationError() error { + if local.DataEngineMode != string(types.DataEngineModeV2Only) { + return nil + } + if local.collection.DataEngineMode != string(types.DataEngineModeV2Only) || local.collection.Log == nil || len(local.collection.Log.Error) != 0 || len(local.collection.Log.Info) == 0 { + return fmt.Errorf("V2-only host preflight is not qualified; inspect the complete report") + } + return nil +} + +func withoutISCSIStrings(dependencies []string) []string { + filtered := make([]string, 0, len(dependencies)) + for _, dependency := range dependencies { + if dependency != "open-iscsi" && dependency != "iscsi-initiator-utils" && dependency != "iscsi_tcp" { + filtered = append(filtered, dependency) + } + } + return filtered +} + // Output converts the collection to JSON and output to stdout or the output file. func (local *Checker) Output() error { local.logger.Tracef("Outputting preflight checks results") @@ -360,6 +419,9 @@ func (local *Checker) checkIscsidService() error { // checkHugePages checks if HugePages is enabled. func (local *Checker) checkHugePages() error { + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + return local.checkV2HugePages() + } logrus.Info("Checking if HugePages is enabled") topic := formatTopic(consts.PreflightCheckTopicHugePages) @@ -613,16 +675,20 @@ func (local *Checker) checkNFSv4Support() error { // check kernel capability var isKernelSupport = false + facts := local.nfsFacts + if facts == nil { + facts = hostCheckerNFSFacts{} + } - kernelVersion, err := utils.GetKernelVersion() + kernelVersion, err := facts.KernelVersion() if err != nil { return wrapInternalError(topic, fmt.Errorf("failed to detect kernel version: %v", err)) } hostBootDir := filepath.Join(consts.VolumeMountHostDirectory, commontypes.SysBootDirectory) - kernelConfigMap, err := commonsys.GetBootKernelConfigMap(hostBootDir, kernelVersion) + kernelConfigMap, err := facts.BootKernelConfig(hostBootDir, kernelVersion) if err != nil { // Fallback attempt: Read from procfs if the first attempt failed - kernelConfigMap, err = commonsys.GetProcKernelConfigMap("/proc") + kernelConfigMap, err = facts.ProcKernelConfig("/proc") if err != nil { return wrapInternalError(topic, fmt.Errorf("failed to read kernel config: %v", err)) } @@ -639,7 +705,7 @@ func (local *Checker) checkNFSv4Support() error { break } else if configVal == "m" { // Check if the module is loaded - moduleLoaded, err := utils.IsModuleLoaded(module) + moduleLoaded, err := facts.ModuleLoaded(module) if err != nil { logrus.Debugf("Failed to check if module %s is loaded: %v", module, err) continue @@ -662,7 +728,7 @@ func (local *Checker) checkNFSv4Support() error { var isSupportedNFSVersion bool hostEtcDir := filepath.Join(consts.VolumeMountHostDirectory, commontypes.SysEtcDirectory) - nfsMajor, nfsMinor, err := commonnfs.GetSystemDefaultNFSVersion(hostEtcDir) + nfsMajor, nfsMinor, err := facts.DefaultNFSVersion(hostEtcDir) if err == nil { isSupportedNFSVersion = nfsMajor == 4 && (nfsMinor == 0 || nfsMinor == 1 || nfsMinor == 2) } else if errors.Is(err, commontypes.ErrNotConfigured) { diff --git a/pkg/local/preflight/checker_data_engine_mode_test.go b/pkg/local/preflight/checker_data_engine_mode_test.go new file mode 100644 index 000000000..c80ae3fb2 --- /dev/null +++ b/pkg/local/preflight/checker_data_engine_mode_test.go @@ -0,0 +1,177 @@ +package preflight + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/mock" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/local/preflight/packagemanager/mocks" + "github.com/longhorn/cli/pkg/types" + + pkgmgr "github.com/longhorn/cli/pkg/local/preflight/packagemanager" + remote "github.com/longhorn/cli/pkg/remote/preflight" +) + +type fixtureNFSFacts struct{ reads int } + +func (facts *fixtureNFSFacts) KernelVersion() (string, error) { + facts.reads++ + return "fixture", nil +} +func (facts *fixtureNFSFacts) BootKernelConfig(string, string) (map[string]string, error) { + facts.reads++ + return map[string]string{"CONFIG_NFS_V4": "y"}, nil +} +func (*fixtureNFSFacts) ProcKernelConfig(string) (map[string]string, error) { + return nil, nil +} +func (*fixtureNFSFacts) ModuleLoaded(string) (bool, error) { return true, nil } +func (facts *fixtureNFSFacts) DefaultNFSVersion(string) (int, int, error) { + facts.reads++ + return 4, 1, nil +} + +func TestV2OnlyCheckerForcesSPDKWithoutAnyISCSIProbe(t *testing.T) { + for _, missingNVMe := range []bool{false, true} { + for _, collectionOnly := range []bool{false, true} { + t.Run(strings.Join([]string{map[bool]string{false: "healthy", true: "missing-nvme"}[missingNVMe], map[bool]string{false: "direct", true: "collection"}[collectionOnly]}, "/"), func(t *testing.T) { + t.Setenv(consts.EnvCurrentNodeID, "fixture-node") + manager := new(mocks.MockPackageManager) + manager.On("GetServiceStatus", "multipathd.service").Return("", nil).Once() + for _, name := range []string{"nfs-common", "cryptsetup", "dmsetup"} { + manager.On("CheckPackageInstalled", name).Return("", nil).Once() + } + for _, name := range []string{"nfs", "dm_crypt", "nvme_tcp", "uio_pci_generic", "vfio_pci", "ublk_drv"} { + var moduleError error + if missingNVMe && name == "nvme_tcp" { + moduleError = pkgmgr.ErrPackageNotInstalled + } + manager.On("CheckModLoaded", name).Return(moduleError).Once() + } + manager.On("Modprobe", "ublk_drv", []string{"--dry-run"}).Return("", nil).Once() + manager.On("Execute", []string{}, "cat", []string{"/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"}, v2SPDKReadTimeout).Return("1024\n", nil).Once() + manager.On("Execute", []string{}, "sh", mock.Anything, mock.Anything).Return("/sys/kernel/iommu_groups/0\n", nil).Once() + instruction := "sse4_2" + if runtime.GOARCH == "arm64" { + instruction = "asimd" + } + manager.On("Execute", []string{}, "grep", mock.MatchedBy(func(args []string) bool { + return strings.Contains(strings.Join(args, " "), instruction) + }), mock.Anything).Return("", nil).Once() + facts := &fixtureNFSFacts{} + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + response.Header().Set("Content-Type", "application/json") + var object interface{} + switch request.URL.Path { + case "/api/v1/nodes/fixture-node": + object = &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "fixture-node"}, Status: corev1.NodeStatus{Capacity: corev1.ResourceList{corev1.ResourceName("hugepages-2Mi"): resource.MustParse("2Gi")}}} + case "/apis/apps/v1/namespaces/kube-system/deployments": + object = &appsv1.DeploymentList{Items: []appsv1.Deployment{{ObjectMeta: metav1.ObjectMeta{Name: "dns", Namespace: "kube-system", Labels: map[string]string{"k8s-app": "kube-dns"}}, Spec: appsv1.DeploymentSpec{Replicas: ptr.To(int32(2))}, Status: appsv1.DeploymentStatus{ReadyReplicas: 2}}}} + default: + t.Errorf("unexpected API read %s", request.URL.Path) + http.NotFound(response, request) + return + } + if err := json.NewEncoder(response).Encode(object); err != nil { + t.Error(err) + } + })) + t.Cleanup(server.Close) + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + checker := &Checker{CheckerCmdOptions: remote.CheckerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}, CollectionOnly: collectionOnly, + logger: logrus.NewEntry(logrus.New()), osRelease: "ubuntu", packageManager: manager, nfsFacts: facts, + collection: types.NodeCollection{Log: &types.LogCollection{}}, OutputFilePath: filepath.Join(t.TempDir(), "report.json"), + kubeClient: client} + if err := checker.configureDependencies(pkgmgr.PackageManagerApt); err != nil { + t.Fatal(err) + } + err = checker.Run() + if (err != nil) != (missingNVMe && !collectionOnly) { + t.Fatalf("qualification/transport verdict changed: %v", err) + } + if err := checker.Output(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(checker.OutputFilePath) + if err != nil { + t.Fatal(err) + } + var report map[string]json.RawMessage + if err := json.Unmarshal(data, &report); err != nil || string(report["dataEngineMode"]) != `"v2-only"` { + t.Fatal("persisted report is not bound to V2-only checks") + } + if (len(checker.collection.Log.Error) != 0) != missingNVMe || facts.reads != 3 { + t.Fatal("mandatory NVMe or native NFS checks were skipped") + } + if !strings.Contains(strings.Join(checker.collection.Log.Info, " "), "cryptsetup") || !strings.Contains(strings.Join(checker.collection.Log.Warn, " "), "multipathd") { + t.Fatal("encryption or multipath checks disappeared") + } + manager.AssertExpectations(t) + }) + } + } +} + +func TestV2OnlyCheckerDependencyProfilesRemoveOnlyISCSI(t *testing.T) { + for _, profile := range []pkgmgr.PackageManagerType{pkgmgr.PackageManagerApt, pkgmgr.PackageManagerYum, pkgmgr.PackageManagerZypper, pkgmgr.PackageManagerTransactionalUpdate, pkgmgr.PackageManagerBinary, pkgmgr.PackageManagerPacman} { + checker := &Checker{CheckerCmdOptions: remote.CheckerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}} + if err := checker.configureDependencies(profile); err != nil { + t.Fatal(err) + } + for _, name := range append(append([]string{}, checker.packages...), checker.modules...) { + if name == "open-iscsi" || name == "iscsi-initiator-utils" || name == "iscsi_tcp" { + t.Fatalf("profile %s retains iSCSI dependency %s", profile, name) + } + } + if len(checker.packages) != 3 || len(checker.modules) != 2 || len(checker.services) != 1 || len(checker.spdkDepModules) != 3 { + t.Fatalf("profile %s removed dependencies beyond iSCSI", profile) + } + } +} + +func TestV2HugepagesDoNotTrustKernelDefaultSizeOrCachedCapacity(t *testing.T) { + manager := new(mocks.MockPackageManager) + manager.On("Execute", []string{}, "grep", []string{"HugePages_Total", "/proc/meminfo"}, mock.Anything).Return("HugePages_Total: 1024\n", nil).Maybe() + manager.On("Execute", []string{}, "cat", []string{"/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"}, mock.Anything).Return("0\n", nil).Maybe() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path != "/api/v1/nodes/fixture-node" { + t.Errorf("unexpected read %s", r.URL.Path) + } + _ = json.NewEncoder(w).Encode(&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "fixture-node"}, Status: corev1.NodeStatus{Capacity: corev1.ResourceList{corev1.ResourceName("hugepages-2Mi"): resource.MustParse("2Gi")}}}) + })) + t.Cleanup(server.Close) + t.Setenv(consts.EnvCurrentNodeID, "fixture-node") + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + checker := Checker{CheckerCmdOptions: remote.CheckerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}, packageManager: manager, kubeClient: client, collection: types.NodeCollection{Log: &types.LogCollection{}}} + if err := checker.checkHugePages(); err != nil { + t.Fatal(err) + } + if len(checker.collection.Log.Error) == 0 { + t.Fatal("kernel default-size pages and stale Node capacity qualified missing actual 2MiB pages") + } +} diff --git a/pkg/local/preflight/checker_hugepages_v2.go b/pkg/local/preflight/checker_hugepages_v2.go new file mode 100644 index 000000000..6b6f43d8d --- /dev/null +++ b/pkg/local/preflight/checker_hugepages_v2.go @@ -0,0 +1,25 @@ +package preflight + +import ( + "fmt" + "strconv" + "strings" +) + +func (local *Checker) checkV2HugePages() error { + output, err := local.packageManager.Execute([]string{}, "cat", []string{"/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"}, v2SPDKReadTimeout) + if err != nil { + return fmt.Errorf("failed to read actual V2 2MiB allocation: %w", err) + } + value := strings.TrimSpace(output) + pages, err := strconv.ParseUint(value, 10, 64) + if err != nil || value == "" || strconv.FormatUint(pages, 10) != value { + return fmt.Errorf("actual V2 2MiB allocation is unresolved") + } + if pages < uint64(local.HugePageSize/2) { + local.collection.Log.Error = append(local.collection.Log.Error, "Actual host 2MiB hugepages are insufficient for V2") + return nil + } + local.collection.Log.Info = append(local.collection.Log.Info, "Verified actual host 2MiB hugepage allocation for V2") + return local.checkHugePagesCapacity() +} diff --git a/pkg/local/preflight/checker_nfs_facts.go b/pkg/local/preflight/checker_nfs_facts.go new file mode 100644 index 000000000..f1799410f --- /dev/null +++ b/pkg/local/preflight/checker_nfs_facts.go @@ -0,0 +1,34 @@ +package preflight + +import ( + commonnfs "github.com/longhorn/go-common-libs/nfs" + commonsys "github.com/longhorn/go-common-libs/sys" + + "github.com/longhorn/cli/pkg/utils" +) + +type checkerNFSFacts interface { + KernelVersion() (string, error) + BootKernelConfig(directory, version string) (map[string]string, error) + ProcKernelConfig(directory string) (map[string]string, error) + ModuleLoaded(name string) (bool, error) + DefaultNFSVersion(directory string) (int, int, error) +} + +type hostCheckerNFSFacts struct{} + +func (hostCheckerNFSFacts) KernelVersion() (string, error) { return utils.GetKernelVersion() } + +func (hostCheckerNFSFacts) BootKernelConfig(directory, version string) (map[string]string, error) { + return commonsys.GetBootKernelConfigMap(directory, version) +} + +func (hostCheckerNFSFacts) ProcKernelConfig(directory string) (map[string]string, error) { + return commonsys.GetProcKernelConfigMap(directory) +} + +func (hostCheckerNFSFacts) ModuleLoaded(name string) (bool, error) { return utils.IsModuleLoaded(name) } + +func (hostCheckerNFSFacts) DefaultNFSVersion(directory string) (int, int, error) { + return commonnfs.GetSystemDefaultNFSVersion(directory) +} diff --git a/pkg/local/preflight/data_engine_host.go b/pkg/local/preflight/data_engine_host.go new file mode 100644 index 000000000..d6e25c126 --- /dev/null +++ b/pkg/local/preflight/data_engine_host.go @@ -0,0 +1,12 @@ +package preflight + +import "fmt" + +func validateV2HostOS(osRelease string) error { + switch osRelease { + case "sles", "suse", "opensuse", "opensuse-leap", "sl-micro", "ubuntu", "debian", "rhel", "ol", "rocky", "centos", "fedora", "amzn", "opencloudos", "arch": + return nil + default: + return fmt.Errorf("V2-only host qualification is unsupported for this operating system; use its declarative host prerequisites") + } +} diff --git a/pkg/local/preflight/installer.go b/pkg/local/preflight/installer.go index 746a8321c..a64c4ba9f 100644 --- a/pkg/local/preflight/installer.go +++ b/pkg/local/preflight/installer.go @@ -39,6 +39,7 @@ type Installer struct { logger *logrus.Entry OutputFilePath string + AttemptID string kubeClient *kubeclient.Clientset @@ -70,6 +71,9 @@ func requiredPackages(required bool, names ...string) []Package { // Init initializes the Installer. func (local *Installer) Init() error { + if err := local.ValidateDataEngineMode(); err != nil { + return err + } local.collection.Log = &types.LogCollection{} config, err := commonkube.GetInClusterConfig() @@ -87,6 +91,11 @@ func (local *Installer) Init() error { return errors.Wrap(err, "failed to get OS release") } local.osRelease = osRelease + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + if err := validateV2HostOS(osRelease); err != nil { + return err + } + } local.logger = logrus.WithField("os", local.osRelease) packageManagerType, err := utils.GetPackageManagerType(osRelease) @@ -115,10 +124,13 @@ func (local *Installer) Init() error { if err != nil { return err } + local.packageManager = pkgMgr + return local.configureDependencies(packageManagerType, kernelRelease) +} +func (local *Installer) configureDependencies(packageManagerType pkgmgr.PackageManagerType, kernelRelease string) error { switch packageManagerType { case pkgmgr.PackageManagerApt: - local.packageManager = pkgMgr local.packages = requiredPackages(true, "nfs-common", "open-iscsi", "cryptsetup") local.modules = requiredPackages(true, "nfs", "dm_crypt") local.services = requiredPackages(true, "iscsid") @@ -126,43 +138,60 @@ func (local *Installer) Init() error { // The installation is kept for older ubuntu versions to ensure the module is present, and it won't cause issue on newer versions as the package manager will skip installation if the package is not found. local.spdkDepPackages = requiredPackages(false, "linux-modules-extra-"+kernelRelease) local.spdkDepModules = requiredPackages(true, "nvme_tcp", "uio_pci_generic", "vfio_pci") - return nil case pkgmgr.PackageManagerYum: - local.packageManager = pkgMgr local.packages = requiredPackages(true, "nfs-utils", "iscsi-initiator-utils", "cryptsetup") local.modules = requiredPackages(true, "nfs", "iscsi_tcp", "dm_crypt") local.services = requiredPackages(true, "iscsid") local.spdkDepPackages = requiredPackages(true) local.spdkDepModules = requiredPackages(true, "nvme_tcp", "uio_pci_generic", "vfio_pci") - return nil case pkgmgr.PackageManagerZypper, pkgmgr.PackageManagerTransactionalUpdate, pkgmgr.PackageManagerBinary: - local.packageManager = pkgMgr local.packages = requiredPackages(true, "nfs-client", "open-iscsi", "cryptsetup") local.modules = requiredPackages(true, "nfs", "iscsi_tcp", "dm_crypt") local.services = requiredPackages(true, "iscsid") local.spdkDepPackages = requiredPackages(true) local.spdkDepModules = requiredPackages(true, "nvme_tcp", "uio_pci_generic", "vfio_pci") - return nil case pkgmgr.PackageManagerPacman: - local.packageManager = pkgMgr local.packages = requiredPackages(true, "nfs-utils", "open-iscsi", "cryptsetup") local.modules = requiredPackages(true, "nfs", "iscsi_tcp", "dm_crypt") local.services = requiredPackages(true, "iscsid") local.spdkDepPackages = requiredPackages(true) local.spdkDepModules = requiredPackages(true, "nvme_tcp", "uio_pci_generic", "vfio_pci") - return nil default: - return errors.Errorf("Operating system (%v) package manager (%s) is not supported", osRelease, packageManagerType) + return errors.Errorf("Operating system (%v) package manager (%s) is not supported", local.osRelease, packageManagerType) } + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + local.packages = withoutISCSIPackages(local.packages) + local.modules = withoutISCSIPackages(local.modules) + local.services = withoutISCSIPackages(local.services) + } + return nil } func (local *Installer) Run() error { + if err := local.ValidateDataEngineMode(); err != nil { + return err + } + mode, err := types.ParseDataEngineMode(local.DataEngineMode) + if err != nil { + return err + } + if mode == types.DataEngineModeV2Only { + if err := validateV2HostOS(local.osRelease); err != nil { + return err + } + if local.packageManager == nil || local.collection.Log == nil || local.kubeClient == nil { + return errors.New("V2-only installation requires an initialized qualified package-manager host") + } + if err := local.qualifyV2SPDKHost(); err != nil { + return err + } + local.collection.DataEngineMode = string(mode) + } var rebootRequired bool - var err error if local.UpdatePackages { if err := local.updatePackageList(); err != nil { @@ -170,7 +199,7 @@ func (local *Installer) Run() error { } } - rebootRequired, err = local.checkAndinstallPackages(local.EnableSpdk) + rebootRequired, err = local.checkAndinstallPackages(mode.NeedsSPDK(local.EnableSpdk)) if err != nil { return err } @@ -178,6 +207,9 @@ func (local *Installer) Run() error { if rebootRequired { logrus.Warn("Need to reboot the system and execute longhornctl install preflight again") local.collection.Log.Warn = append(local.collection.Log.Warn, "Need to reboot the system and execute longhornctl install preflight again") + if mode == types.DataEngineModeV2Only { + return errors.New("V2-only installation is pending a host reboot; rerun installation and qualification afterward") + } return nil } @@ -189,7 +221,7 @@ func (local *Installer) Run() error { return err } - if local.EnableSpdk { + if mode.NeedsSPDK(local.EnableSpdk) { // Load ublk_drv module if supported by the kernel if _, err := local.packageManager.Modprobe("ublk_drv", "--dry-run"); err != nil { local.collection.Log.Warn = append(local.collection.Log.Warn, @@ -328,6 +360,9 @@ func (local *Installer) updatePackageList() error { // configureSPDKEnv configures SPDK environment. func (local *Installer) configureSPDKEnv() error { + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + return local.configureV2SPDKEnv() + } // Blindly remove the SPDK source code directory if it exists. spdkPath := filepath.Join(consts.VolumeMountHostDirectory, consts.SpdkPath) if err := os.RemoveAll(spdkPath); err != nil { @@ -354,6 +389,16 @@ func (local *Installer) configureSPDKEnv() error { return nil } +func withoutISCSIPackages(dependencies []Package) []Package { + filtered := make([]Package, 0, len(dependencies)) + for _, dependency := range dependencies { + if dependency.Name != "open-iscsi" && dependency.Name != "iscsi-initiator-utils" && dependency.Name != "iscsi_tcp" && dependency.Name != "iscsid" { + filtered = append(filtered, dependency) + } + } + return filtered +} + // getEnvsForConfiguringSPDKEnv parses the comma-separated --spdk-options string // into a list of KEY=VALUE environment variables. SPDK's scripts/setup.sh reads // its tunables (e.g. HUGEMEM, HUGENODE, PERSIST_HUGE, PCI_ALLOWED) from the diff --git a/pkg/local/preflight/installer_collection_v2.go b/pkg/local/preflight/installer_collection_v2.go new file mode 100644 index 000000000..e96ed34b5 --- /dev/null +++ b/pkg/local/preflight/installer_collection_v2.go @@ -0,0 +1,115 @@ +package preflight + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "syscall" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" +) + +var v2InstallAttemptID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// RunForCollection transports a failed or reboot-pending installation as a +// complete error report. The remote collector, not the init-container exit +// code, owns the verdict. A persistent per-invocation host marker prevents +// Kubernetes init retries or replacement Pods from repeating host writes. +func (local *Installer) RunForCollection() error { + if err := local.ValidateDataEngineMode(); err != nil { + return err + } + if local.DataEngineMode != string(types.DataEngineModeV2Only) || local.collection.Log == nil || local.packageManager == nil || local.kubeClient == nil { + return fmt.Errorf("V2 collection requires an initialized installer") + } + if local.OutputFilePath != filepath.Join(consts.VolumeMountSharedDirectory, consts.FileNameOutputJSON) || len(local.AttemptID) > 63 || !strings.HasPrefix(local.AttemptID, consts.AppNamePreflightInstaller+"-") || !v2InstallAttemptID.MatchString(local.AttemptID) { + return fmt.Errorf("V2 installer collection requires its identified invocation and shared output") + } + local.collection.DataEngineMode = string(types.DataEngineModeV2Only) + if err := local.qualifyV2SPDKHost(); err != nil { + local.recordV2InstallFailure(err) + return nil + } + directory := filepath.Join(consts.VolumeMountHostDirectory, "var/lib/longhorn-cli/preflight-attempts") + if err := qualifyV2AttemptFilesystem(directory); err != nil { + local.recordV2InstallFailure(err) + return nil + } + return runV2InstallAttempt(directory, local.AttemptID, local.Run, local.recordV2InstallFailure) +} + +func (local *Installer) recordV2InstallFailure(err error) { + local.collection.Log.Info = append(local.collection.Log.Info, "V2 prerequisite setup returned a failure report; readiness is not established") + local.collection.Log.Error = append(local.collection.Log.Error, err.Error()) +} + +func runV2InstallAttempt(directory, attemptID string, run func() error, recordFailure func(error)) error { + if err := claimV2InstallAttempt(directory, attemptID); err != nil { + recordFailure(err) + return nil + } + if err := run(); err != nil { + recordFailure(err) + } + return nil +} + +func claimV2InstallAttempt(directory, attemptID string) error { + if err := prepareV2AttemptDirectory(directory); err != nil { + return err + } + file, err := os.OpenFile(filepath.Join(directory, attemptID), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("V2 installation was already attempted on this host; automatic replay is refused; inspect the prior failure or reboot state before a new invocation") + } + if err != nil { + return err + } + if err := errors.Join(file.Sync(), file.Close()); err != nil { + return err + } + return syncV2AttemptDirectory(directory) +} + +func syncV2AttemptDirectory(directory string) error { + dir, err := os.Open(directory) + if err != nil { + return err + } + return errors.Join(dir.Sync(), dir.Close()) +} + +func prepareV2AttemptDirectory(directory string) error { + // The runtime path has two private levels below the existing /var/lib. + // Persist each child's entry in its parent, including when a concurrent + // invocation created it, before the host-changing callback can run. + root := filepath.Dir(filepath.Dir(directory)) + info, err := os.Lstat(root) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("V2 attempt root is not an identified directory") + } + for _, path := range []string{filepath.Dir(directory), directory} { + if err := os.Mkdir(path, 0700); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + info, err := os.Lstat(path) + if err != nil { + return err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0077 != 0 || !ok || int(stat.Uid) != os.Geteuid() { + return fmt.Errorf("V2 attempt directory is not private and owned by this installer") + } + if err := errors.Join(syncV2AttemptDirectory(path), syncV2AttemptDirectory(filepath.Dir(path))); err != nil { + return err + } + } + return nil +} diff --git a/pkg/local/preflight/installer_collection_v2_test.go b/pkg/local/preflight/installer_collection_v2_test.go new file mode 100644 index 000000000..63d5377aa --- /dev/null +++ b/pkg/local/preflight/installer_collection_v2_test.go @@ -0,0 +1,72 @@ +package preflight + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/longhorn/cli/pkg/types" +) + +func TestV2InstallerTransportReportsFailureWithoutAutomaticReplay(t *testing.T) { + directory := filepath.Join(t.TempDir(), "private-tool", "persistent-attempts") + calls := 0 + installer := Installer{collection: types.NodeCollection{Log: &types.LogCollection{}}} + run := func() error { calls++; return errors.New("installation pending a host reboot") } + if err := runV2InstallAttempt(directory, "invocation-1", run, installer.recordV2InstallFailure); err != nil { + t.Fatal(err) + } + if calls != 1 || len(installer.collection.Log.Error) != 1 || !strings.Contains(installer.collection.Log.Error[0], "host reboot") { + t.Fatal("failed installation did not produce its failure report") + } + // A new container/Pod uses the same persistent host marker. It must be + // able to report failure without calling the mutating installer again. + replacement := Installer{collection: types.NodeCollection{Log: &types.LogCollection{}}} + if err := runV2InstallAttempt(directory, "invocation-1", run, replacement.recordV2InstallFailure); err != nil { + t.Fatal(err) + } + if calls != 1 || len(replacement.collection.Log.Error) != 1 || !strings.Contains(replacement.collection.Log.Error[0], "automatic replay is refused") { + t.Fatal("container replacement replayed host writes or lost the refusal report") + } + if err := runV2InstallAttempt(directory, "explicit-new-invocation", run, replacement.recordV2InstallFailure); err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatal("an explicit new invocation could not run") + } +} + +func TestV2InstallerCannotMutateWithoutItsAttemptRecord(t *testing.T) { + directory := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(directory, []byte("existing"), 0600); err != nil { + t.Fatal(err) + } + runs, failures := 0, 0 + if err := runV2InstallAttempt(directory, "invocation", func() error { runs++; return nil }, func(error) { failures++ }); err != nil { + t.Fatal(err) + } + if runs != 0 || failures != 1 { + t.Fatal("host mutation ran without a persistent attempt record") + } +} + +func TestV2AttemptStoreRejectsSymlinksBeforeHostWrites(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "tool")); err != nil { + t.Fatal(err) + } + runs, failures := 0, 0 + if err := runV2InstallAttempt(filepath.Join(root, "tool", "attempts"), "invocation", func() error { runs++; return nil }, func(error) { failures++ }); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(outside) + if err != nil { + t.Fatal(err) + } + if runs != 0 || failures != 1 || len(entries) != 0 { + t.Fatal("unidentified attempt directory allowed host writes or changed its symlink target") + } +} diff --git a/pkg/local/preflight/installer_data_engine_mode_test.go b/pkg/local/preflight/installer_data_engine_mode_test.go new file mode 100644 index 000000000..ba3fee76b --- /dev/null +++ b/pkg/local/preflight/installer_data_engine_mode_test.go @@ -0,0 +1,36 @@ +package preflight + +import ( + "strings" + "testing" + + pkgmgr "github.com/longhorn/cli/pkg/local/preflight/packagemanager" + remote "github.com/longhorn/cli/pkg/remote/preflight" +) + +func TestV2InstallerRejectsInvalidCapabilityBeforeInitialization(t *testing.T) { + installer := Installer{InstallerCmdOptions: remote.InstallerCmdOptions{DataEngineMode: "unknown"}} + if err := installer.Init(); err == nil || !strings.Contains(err.Error(), "data-engine-mode") { + t.Fatalf("invalid mode reached host or connection initialization: %v", err) + } +} + +func TestV2InstallerProfilesRemoveOnlyISCSI(t *testing.T) { + for _, profile := range []pkgmgr.PackageManagerType{pkgmgr.PackageManagerApt, pkgmgr.PackageManagerYum, pkgmgr.PackageManagerZypper, pkgmgr.PackageManagerTransactionalUpdate, pkgmgr.PackageManagerBinary, pkgmgr.PackageManagerPacman} { + installer := Installer{InstallerCmdOptions: remote.InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}} + if err := installer.configureDependencies(profile, "fixture-kernel"); err != nil { + t.Fatal(err) + } + for _, dependency := range append(append(append([]Package{}, installer.packages...), installer.modules...), installer.services...) { + if dependency.Name == "open-iscsi" || dependency.Name == "iscsi-initiator-utils" || dependency.Name == "iscsi_tcp" || dependency.Name == "iscsid" { + t.Fatalf("V2 profile retains %s", dependency.Name) + } + if !dependency.Required { + t.Fatalf("required dependency became optional: %s", dependency.Name) + } + } + if len(installer.packages) != 2 || len(installer.modules) != 2 || len(installer.services) != 0 || len(installer.spdkDepModules) != 3 { + t.Fatalf("V2 profile removed non-iSCSI dependencies: %s", profile) + } + } +} diff --git a/pkg/local/preflight/installer_journal_filesystem_linux.go b/pkg/local/preflight/installer_journal_filesystem_linux.go new file mode 100644 index 000000000..e074a4c06 --- /dev/null +++ b/pkg/local/preflight/installer_journal_filesystem_linux.go @@ -0,0 +1,39 @@ +//go:build linux + +package preflight + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +func qualifyV2AttemptFilesystem(directory string) error { + path := directory + for { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) && filepath.Dir(path) != path { + path = filepath.Dir(path) + continue + } + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("V2 attempt store is not an identified directory") + } + var status unix.Statfs_t + if err := unix.Statfs(path, &status); err != nil { + return err + } + switch status.Type { + case unix.EXT4_SUPER_MAGIC, unix.XFS_SUPER_MAGIC, unix.BTRFS_SUPER_MAGIC, unix.F2FS_SUPER_MAGIC: + return nil + default: + return fmt.Errorf("V2 installer replay protection requires a qualified persistent ext, XFS, Btrfs or F2FS attempt store") + } + } +} diff --git a/pkg/local/preflight/installer_journal_filesystem_linux_test.go b/pkg/local/preflight/installer_journal_filesystem_linux_test.go new file mode 100644 index 000000000..365e0c403 --- /dev/null +++ b/pkg/local/preflight/installer_journal_filesystem_linux_test.go @@ -0,0 +1,11 @@ +//go:build linux + +package preflight + +import "testing" + +func TestV2AttemptStoreRejectsVirtualFilesystems(t *testing.T) { + if err := qualifyV2AttemptFilesystem("/proc/longhorn-cli/nonexistent-attempts"); err == nil { + t.Fatal("virtual attempt store incorrectly promised durable replay protection") + } +} diff --git a/pkg/local/preflight/installer_journal_filesystem_other.go b/pkg/local/preflight/installer_journal_filesystem_other.go new file mode 100644 index 000000000..a718fbe1b --- /dev/null +++ b/pkg/local/preflight/installer_journal_filesystem_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package preflight + +import "fmt" + +func qualifyV2AttemptFilesystem(string) error { + return fmt.Errorf("V2 installer replay protection requires a qualified Linux host") +} diff --git a/pkg/local/preflight/installer_spdk_v2.go b/pkg/local/preflight/installer_spdk_v2.go new file mode 100644 index 000000000..85fcd0f5d --- /dev/null +++ b/pkg/local/preflight/installer_spdk_v2.go @@ -0,0 +1,104 @@ +package preflight + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + cp "github.com/otiai10/copy" + + "github.com/longhorn/cli/pkg/consts" +) + +const v2SPDKReadTimeout = 15 * time.Second +const v2SPDKSetupTimeout = 3 * time.Minute + +var v2HostReadEnvironment = []string{"-i", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "LC_ALL=C", "/bin/bash", "--noprofile", "--norc"} + +// The pinned setup script accepts eval-like NUMA grammar and optionally sources +// host-provided scripts. V2 accepts one canonical node and rejects those hooks. +func (local *Installer) qualifyV2SPDKHost() error { + _, node, err := local.V2SPDKSetupEnvironment() + if err != nil { + return err + } + args := append(append([]string{}, v2HostReadEnvironment...), "-c", `set -eu +test -x /usr/bin/env +test -x /bin/bash +test ! -e /bin/wpdk_common.sh && test ! -L /bin/wpdk_common.sh +test ! -e /etc/opt/spdk-pkgdep/paths/export.sh && test ! -L /etc/opt/spdk-pkgdep/paths/export.sh +test -d "/sys/devices/system/node/node$1/hugepages/hugepages-2048kB" +test -r "/sys/devices/system/node/node$1/hugepages/hugepages-2048kB/nr_hugepages" +test -r "/sys/devices/system/node/node$1/hugepages/hugepages-2048kB/free_hugepages" +`, "--", node) + if _, err := local.packageManager.Execute(nil, "/usr/bin/env", args, v2SPDKReadTimeout); err != nil { + return fmt.Errorf("V2 SPDK host prerequisites are unqualified: %w", err) + } + _, err = local.readV2NodeHugepages(node) + return err +} + +func (local *Installer) readV2NodeHugepages(node string) (uint64, error) { + path := "/sys/devices/system/node/node" + node + "/hugepages/hugepages-2048kB/nr_hugepages" + args := append(append([]string{}, v2HostReadEnvironment...), "-c", `cat -- "$1"`, "--", path) + output, err := local.packageManager.Execute(nil, "/usr/bin/env", args, v2SPDKReadTimeout) + if err != nil { + return 0, fmt.Errorf("failed to read actual V2 2MiB allocation: %w", err) + } + value := strings.TrimSpace(output) + pages, err := strconv.ParseUint(value, 10, 64) + if err != nil || value == "" || strconv.FormatUint(pages, 10) != value { + return 0, fmt.Errorf("actual V2 2MiB allocation is unresolved") + } + return pages, nil +} + +func (local *Installer) configureV2SPDKEnv() (resultErr error) { + environment, node, err := local.V2SPDKSetupEnvironment() + if err != nil { + return err + } + if err := local.qualifyV2SPDKHost(); err != nil { + return err + } + // Copy only into this invocation's new directory. Never remove or reuse a + // shared host SPDK tree belonging to another invocation or application. + hostDirectory, err := os.MkdirTemp(filepath.Join(consts.VolumeMountHostDirectory, "tmp"), "longhorn-spdk-v2-") + if err != nil { + return err + } + defer func() { resultErr = errors.Join(resultErr, os.RemoveAll(hostDirectory)) }() + if err := cp.Copy("/spdk", hostDirectory); err != nil { + return err + } + if err := local.qualifyV2SPDKHost(); err != nil { + return err + } + relative, err := filepath.Rel(consts.VolumeMountHostDirectory, hostDirectory) + if err != nil || strings.HasPrefix(relative, "..") || filepath.IsAbs(relative) { + return fmt.Errorf("V2 setup host directory is not identified") + } + setupPath := filepath.Join("/", relative, "scripts/setup.sh") + return local.executeV2SPDKSetup(setupPath, environment, node) +} + +func (local *Installer) executeV2SPDKSetup(setupPath string, environment []string, node string) error { + args := append([]string{"-i"}, environment...) + args = append(args, "/bin/bash", "--noprofile", "--norc", "--", setupPath, "config") + if _, err := local.packageManager.Execute(nil, "/usr/bin/env", args, v2SPDKSetupTimeout); err != nil { + return fmt.Errorf("V2 SPDK setup failed: %w", err) + } + pages, err := local.readV2NodeHugepages(node) + if err != nil { + return err + } + if pages < uint64(local.HugePageSize/2) { + return fmt.Errorf("V2 SPDK setup did not allocate the requested 2MiB hugepages on the selected NUMA node") + } + local.collection.Log.Info = append(local.collection.Log.Info, "Configured V2 SPDK environment and verified actual 2MiB hugepage allocation; run V2 preflight qualification after any required kubelet restart") + return nil +} diff --git a/pkg/local/preflight/installer_spdk_v2_test.go b/pkg/local/preflight/installer_spdk_v2_test.go new file mode 100644 index 000000000..e9eebe413 --- /dev/null +++ b/pkg/local/preflight/installer_spdk_v2_test.go @@ -0,0 +1,70 @@ +package preflight + +import ( + "errors" + "reflect" + "testing" + + "github.com/stretchr/testify/mock" + + "github.com/longhorn/cli/pkg/local/preflight/packagemanager/mocks" + "github.com/longhorn/cli/pkg/types" + + remote "github.com/longhorn/cli/pkg/remote/preflight" +) + +func TestV2SPDKSetupClearsInheritedOptionsAndVerifiesActualAllocation(t *testing.T) { + t.Setenv("BASH_ENV", "/unqualified/startup") + t.Setenv("NRHUGE", "0") + t.Setenv("PCI_ALLOWED", "all") + for _, test := range []struct { + name, pages string + setupError bool + valid bool + }{ + {"complete", "1024", false, true}, + {"setup-failed", "", true, false}, + {"partial-allocation", "1000", false, false}, + {"unreadable-allocation", "unknown", false, false}, + } { + t.Run(test.name, func(t *testing.T) { + manager := new(mocks.MockPackageManager) + installer := Installer{InstallerCmdOptions: remote.InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048, AllowPci: "none"}, packageManager: manager, collection: types.NodeCollection{Log: &types.LogCollection{}}} + environment, node, err := installer.V2SPDKSetupEnvironment() + if err != nil { + t.Fatal(err) + } + args := append([]string{"-i"}, environment...) + args = append(args, "/bin/bash", "--noprofile", "--norc", "--", "/tmp/owned/scripts/setup.sh", "config") + var setupErr error + if test.setupError { + setupErr = errors.New("native setup failed") + } + manager.On("Execute", []string(nil), "/usr/bin/env", mock.MatchedBy(func(actual []string) bool { return reflect.DeepEqual(actual, args) }), v2SPDKSetupTimeout).Return("", setupErr).Once() + if !test.setupError { + manager.On("Execute", []string(nil), "/usr/bin/env", mock.Anything, v2SPDKReadTimeout).Return(test.pages+"\n", nil).Once() + } + err = installer.executeV2SPDKSetup("/tmp/owned/scripts/setup.sh", environment, node) + if (err == nil) != test.valid { + t.Fatalf("incorrect setup verdict: %v", err) + } + if test.setupError && !errors.Is(err, setupErr) { + t.Fatal("native setup failure was swallowed") + } + if (len(installer.collection.Log.Info) > 0) != test.valid { + t.Fatal("setup reported success before actual allocation was verified") + } + manager.AssertExpectations(t) + }) + } +} + +func TestV2SPDKHostQualificationRejectsUnsupportedHooksBeforeWrites(t *testing.T) { + manager := new(mocks.MockPackageManager) + installer := Installer{InstallerCmdOptions: remote.InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048, AllowPci: "none"}, packageManager: manager} + manager.On("Execute", []string(nil), "/usr/bin/env", mock.Anything, v2SPDKReadTimeout).Return("", errors.New("unqualified host hook or NUMA interface")).Once() + if err := installer.qualifyV2SPDKHost(); err == nil { + t.Fatal("unsupported host profile qualified") + } + manager.AssertExpectations(t) +} diff --git a/pkg/remote/preflight/checker.go b/pkg/remote/preflight/checker.go index 1b82e5ac0..ca587a1fd 100644 --- a/pkg/remote/preflight/checker.go +++ b/pkg/remote/preflight/checker.go @@ -1,9 +1,15 @@ package preflight import ( + "context" + "crypto/rand" "encoding/json" "path/filepath" "reflect" + "strings" + "time" + + stderrors "errors" "github.com/pkg/errors" @@ -13,6 +19,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeclient "k8s.io/client-go/kubernetes" @@ -32,13 +39,16 @@ type Checker struct { kubeClient *kubeclient.Clientset - appName string // App name of the DaemonSet. + appName string // App name of the DaemonSet. + createdDaemonSet *appsv1.DaemonSet + createdRBAC *v2RBACReceipt } // CheckerCmdOptions holds the options for the command. type CheckerCmdOptions struct { types.GlobalCmdOptions + DataEngineMode string EnableSpdk bool HugePageSize int UserspaceDriver string @@ -46,6 +56,9 @@ type CheckerCmdOptions struct { // Init initializes the Checker. func (remote *Checker) Init() error { + if err := remote.ValidateDataEngineMode(); err != nil { + return err + } kubeClient, err := kubeutils.NewKubeClient("", remote.KubeConfigPath) if err != nil { return err @@ -54,11 +67,18 @@ func (remote *Checker) Init() error { remote.kubeClient = kubeClient remote.appName = consts.AppNamePreflightChecker + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + // A V2 run never reuses or pre-cleans another run's checker or reports. + remote.appName += "-" + strings.ToLower(rand.Text()) + } return nil } // Run creates the DaemonSet for the preflight check, and waits for it to complete. func (remote *Checker) Run() (string, error) { + if err := remote.ValidateDataEngineMode(); err != nil { + return "", err + } // Create RBAC to check: // - the node agent existence when the cluster is running on Container-Optimized OS (COS) // - replica count of the DNS deployment @@ -75,7 +95,14 @@ func (remote *Checker) Run() (string, error) { Verbs: []string{"get"}, }, } - err := kubeutils.CreateRbac(remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + var err error + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + remote.createdRBAC, err = createV2RBAC(ctx, remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + } else { + err = kubeutils.CreateRbac(remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + } if err != nil { return "", err } @@ -84,10 +111,23 @@ func (remote *Checker) Run() (string, error) { if err != nil { return "", err } - daemonSet, err := commonkube.CreateDaemonSet(remote.kubeClient, newDaemonSet) + var daemonSet *appsv1.DaemonSet + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + daemonSet, err = remote.kubeClient.AppsV1().DaemonSets(remote.Namespace).Create(ctx, newDaemonSet, metav1.CreateOptions{}) + } else { + daemonSet, err = commonkube.CreateDaemonSet(remote.kubeClient, newDaemonSet) + } if err != nil { return "", err } + remote.createdDaemonSet = daemonSet.DeepCopy() + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + if err := validateV2CreatedDaemonSet(newDaemonSet, daemonSet, consts.SubCmdCheck); err != nil { + return "", err + } + } err = kubeutils.MonitorDaemonSetContainer(remote.kubeClient, daemonSet, consts.ContainerNameInit, kubeutils.WaitForDaemonSetContainersExit, ptr.To(consts.ContainerConditionMaxTolerationMedium)) if err != nil { @@ -98,6 +138,9 @@ func (remote *Checker) Run() (string, error) { if err != nil { return "", err } + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + return collectV2Reports(context.Background(), remote.kubeClient, daemonSet, readV2PodReport(remote.kubeClient)) + } podCollections, err := kubeutils.GetDaemonSetPodCollections(remote.kubeClient, daemonSet, consts.ContainerNameOutput, false, false, nil) if err != nil { @@ -133,6 +176,22 @@ func (remote *Checker) Run() (string, error) { // Cleanup deletes the DaemonSet created for the preflight check. func (remote *Checker) Cleanup() error { var resultErr error + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + var daemonSetErr error + if remote.createdDaemonSet != nil { + uid := remote.createdDaemonSet.UID + if uid == "" { + daemonSetErr = errors.New("refusing V2 checker cleanup without its created UID") + } else if err := remote.kubeClient.AppsV1().DaemonSets(remote.createdDaemonSet.Namespace).Delete(ctx, remote.createdDaemonSet.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + daemonSetErr = err + } + } + rbacContext, rbacCancel := context.WithTimeout(context.Background(), time.Minute) + defer rbacCancel() + return stderrors.Join(daemonSetErr, cleanupV2RBAC(rbacContext, remote.kubeClient, remote.createdRBAC)) + } if err := commonkube.DeleteDaemonSet(remote.kubeClient, remote.Namespace, remote.appName); err != nil { resultErr = errors.Wrap(err, "failed to delete DaemonSet") @@ -152,6 +211,12 @@ func (remote *Checker) Cleanup() error { // newDaemonSet prepares a DaemonSet for the preflight check. func (remote *Checker) newDaemonSet() *appsv1.DaemonSet { outputFilePath := filepath.Join(consts.VolumeMountSharedDirectory, consts.FileNameOutputJSON) + command := []string{consts.CmdLonghornctlLocal, consts.SubCmdCheck, consts.SubCmdPreflight} + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + // Older local images must reject the explicit flag, rather than + // silently ignore an environment variable and report V1 checks. + command = append(command, "--"+consts.CmdOptDataEngineMode, remote.DataEngineMode) + } return &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: remote.appName, @@ -179,7 +244,7 @@ func (remote *Checker) newDaemonSet() *appsv1.DaemonSet { { Name: consts.ContainerNameInit, Image: utils.BuildImageName(remote.Image, remote.ImageRegistry), - Command: []string{consts.CmdLonghornctlLocal, consts.SubCmdCheck, consts.SubCmdPreflight}, + Command: command, Env: []corev1.EnvVar{ { Name: consts.EnvLogLevel, @@ -189,6 +254,14 @@ func (remote *Checker) newDaemonSet() *appsv1.DaemonSet { Name: consts.EnvOutputFilePath, Value: outputFilePath, }, + { + Name: consts.EnvDataEngineMode, + Value: remote.DataEngineMode, + }, + { + Name: consts.EnvPreflightCollectionOnly, + Value: commonutils.ConvertTypeToString(remote.DataEngineMode == string(types.DataEngineModeV2Only)), + }, { Name: consts.EnvEnableSpdk, Value: commonutils.ConvertTypeToString(remote.EnableSpdk), diff --git a/pkg/remote/preflight/checker_collection_v2.go b/pkg/remote/preflight/checker_collection_v2.go new file mode 100644 index 000000000..26b0aa0d1 --- /dev/null +++ b/pkg/remote/preflight/checker_collection_v2.go @@ -0,0 +1,489 @@ +package preflight + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "reflect" + "sort" + "strings" + "time" + + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/kustomize/kyaml/yaml" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" +) + +const maxV2ReportBytes = 256 * 1024 + +func decodeV2Report(data []byte) (*types.NodeCollection, error) { + if len(data) == 0 || len(data) > maxV2ReportBytes { + return nil, fmt.Errorf("missing or oversized V2 report") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + if err := uniqueJSONValue(decoder, 0); err != nil { + return nil, fmt.Errorf("invalid or ambiguous V2 report") + } + if _, err := decoder.Token(); err != io.EOF { + return nil, fmt.Errorf("V2 report contains trailing content") + } + decoder = json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return nil, fmt.Errorf("V2 report must be an object") + } + for name := range fields { + if name != "dataEngineMode" && name != "log" { + return nil, fmt.Errorf("V2 report has a noncanonical field") + } + } + var logs map[string]json.RawMessage + if err := json.Unmarshal(fields["log"], &logs); err != nil { + return nil, fmt.Errorf("V2 report log must be an object") + } + for name := range logs { + if name != "info" && name != "warn" && name != "error" { + return nil, fmt.Errorf("V2 report log has a noncanonical field") + } + } + var report types.NodeCollection + if err := decoder.Decode(&report); err != nil { + return nil, fmt.Errorf("V2 report has an invalid schema") + } + if report.DataEngineMode != string(types.DataEngineModeV2Only) || report.Log == nil || len(report.Log.Info) == 0 { + return nil, fmt.Errorf("V2 report is missing capability or check evidence") + } + return &report, nil +} + +func uniqueJSONValue(decoder *json.Decoder, depth int) error { + if depth > 32 { + return fmt.Errorf("JSON nesting exceeds limit") + } + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, ok := token.(json.Delim) + if !ok { + return nil + } + switch delimiter { + case '{': + keys := map[string]bool{} + for decoder.More() { + key, err := decoder.Token() + if err != nil { + return err + } + name, ok := key.(string) + if !ok || keys[name] { + return fmt.Errorf("duplicate JSON key") + } + keys[name] = true + if err := uniqueJSONValue(decoder, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := uniqueJSONValue(decoder, depth+1); err != nil { + return err + } + } + default: + return fmt.Errorf("unexpected JSON delimiter") + } + end, err := decoder.Token() + if err != nil || (delimiter == '{' && end != json.Delim('}')) || (delimiter == '[' && end != json.Delim(']')) { + return fmt.Errorf("incomplete JSON value") + } + return nil +} + +type v2ReportLogReader func(context.Context, string, string) ([]byte, error) + +type v2PodEvidence struct { + UID string + Name, Node string + Spec corev1.PodSpec + Owners []metav1.OwnerReference + Instances []corev1.ContainerStatus +} + +// The legacy collector intentionally tolerates missing logs. V2 qualification +// uses a separate complete inventory and brackets logs against exact instances. +func collectV2Reports(ctx context.Context, client kubeclient.Interface, expected *appsv1.DaemonSet, readLog v2ReportLogReader) (string, error) { + return collectV2OperationReports(ctx, client, expected, readLog, consts.SubCmdCheck) +} + +func collectV2OperationReports(ctx context.Context, client kubeclient.Interface, expected *appsv1.DaemonSet, readLog v2ReportLogReader, operation string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + if expected == nil || expected.UID == "" || expected.Generation <= 0 || expected.Spec.Selector == nil { + return "", fmt.Errorf("V2 collection requires a newly created identified DaemonSet") + } + beforeDS, beforePods, err := snapshotV2Checker(ctx, client, expected, operation) + if err != nil { + return "", err + } + nodes := map[string]*types.LogCollection{} + qualified := true + total := 0 + for _, pod := range beforePods { + data, err := readLog(ctx, expected.Namespace, pod.Name) + if err != nil { + return "", fmt.Errorf("failed to read a complete V2 report for node %s: %w", pod.Node, err) + } + total += len(data) + if total > 32*1024*1024 { + return "", fmt.Errorf("V2 report collection exceeds total limit") + } + report, err := decodeV2Report(data) + if err != nil { + return "", fmt.Errorf("invalid V2 report for node %s: %w", pod.Node, err) + } + nodes[pod.Node] = report.Log + qualified = qualified && len(report.Log.Error) == 0 + } + afterDS, afterPods, err := snapshotV2Checker(ctx, client, expected, operation) + if err != nil { + return "", err + } + if beforeDS.Status.DesiredNumberScheduled != afterDS.Status.DesiredNumberScheduled || !reflect.DeepEqual(beforePods, afterPods) { + return "", fmt.Errorf("V2 checker membership or execution changed while collecting reports") + } + output, err := yaml.Marshal(nodes) + if err != nil { + return "", err + } + if !qualified { + return string(output), fmt.Errorf("V2-only host preflight is not qualified; inspect the complete report") + } + return string(output), nil +} + +func snapshotV2Checker(ctx context.Context, client kubeclient.Interface, expected *appsv1.DaemonSet, operation string) (*appsv1.DaemonSet, []v2PodEvidence, error) { + ds, err := client.AppsV1().DaemonSets(expected.Namespace).Get(ctx, expected.Name, metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + if ds.UID != expected.UID || ds.Generation != expected.Generation || ds.DeletionTimestamp != nil || !reflect.DeepEqual(ds.Spec, expected.Spec) || ds.Status.ObservedGeneration < ds.Generation || ds.Status.DesiredNumberScheduled <= 0 || ds.Status.DesiredNumberScheduled > 4096 || ds.Status.CurrentNumberScheduled != ds.Status.DesiredNumberScheduled || ds.Status.UpdatedNumberScheduled != ds.Status.DesiredNumberScheduled || ds.Status.NumberMisscheduled != 0 { + return nil, nil, fmt.Errorf("V2 checker DaemonSet identity, specification or membership is unqualified") + } + selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector) + if err != nil || selector.Empty() { + return nil, nil, fmt.Errorf("V2 checker requires a specific selector") + } + evidence := []v2PodEvidence{} + seenNodes, seenUIDs, seenNames := map[string]bool{}, map[string]bool{}, map[string]bool{} + continuation, resourceVersion := "", "" + seenContinuations := map[string]bool{} + total := 0 + for page := 0; ; page++ { + if page >= 32 { + return nil, nil, fmt.Errorf("V2 Pod inventory exceeds page limit") + } + pods, err := client.CoreV1().Pods(ds.Namespace).List(ctx, metav1.ListOptions{Limit: 256, Continue: continuation}) + if err != nil { + return nil, nil, err + } + if pods.ResourceVersion == "" || (resourceVersion != "" && pods.ResourceVersion != resourceVersion) { + return nil, nil, fmt.Errorf("V2 Pod inventory is not a complete consistent snapshot") + } + resourceVersion = pods.ResourceVersion + total += len(pods.Items) + if total > 8192 { + return nil, nil, fmt.Errorf("V2 Pod inventory exceeds object limit") + } + for _, pod := range pods.Items { + owned, controllerCount := false, 0 + for _, owner := range pod.OwnerReferences { + if owner.Controller != nil && *owner.Controller { + controllerCount++ + owned = owner.UID == ds.UID && owner.Name == ds.Name && owner.Kind == "DaemonSet" && owner.APIVersion == "apps/v1" + } + } + matches := selector.Matches(labels.Set(pod.Labels)) + if !owned && !matches { + // A non-controlling reference must not hide an owned-looking Pod. + for _, owner := range pod.OwnerReferences { + if owner.UID == ds.UID { + return nil, nil, fmt.Errorf("V2 checker Pod has an ambiguous owner") + } + } + continue + } + if !owned || !matches || controllerCount != 1 || pod.UID == "" || pod.Name == "" || pod.Namespace != ds.Namespace || pod.Spec.NodeName == "" || pod.DeletionTimestamp != nil || seenNodes[pod.Spec.NodeName] || seenUIDs[string(pod.UID)] || seenNames[pod.Name] { + return nil, nil, fmt.Errorf("V2 checker Pod inventory has missing, foreign or duplicate execution") + } + instances, err := v2PodInstances(ds, &pod, operation) + if err != nil { + return nil, nil, err + } + seenNodes[pod.Spec.NodeName], seenUIDs[string(pod.UID)], seenNames[pod.Name] = true, true, true + evidence = append(evidence, v2PodEvidence{UID: string(pod.UID), Name: pod.Name, Node: pod.Spec.NodeName, Spec: pod.Spec, Owners: pod.OwnerReferences, Instances: instances}) + } + continuation = pods.Continue + if continuation == "" { + break + } + if seenContinuations[continuation] { + return nil, nil, fmt.Errorf("V2 Pod inventory repeated a continuation") + } + seenContinuations[continuation] = true + } + if len(evidence) != int(ds.Status.DesiredNumberScheduled) { + return nil, nil, fmt.Errorf("V2 checker is missing a report-producing Pod") + } + sort.Slice(evidence, func(i, j int) bool { return evidence[i].Node < evidence[j].Node }) + return ds, evidence, nil +} + +func v2PodInstances(ds *appsv1.DaemonSet, pod *corev1.Pod, operation string) ([]corev1.ContainerStatus, error) { + template := ds.Spec.Template.Spec + if !validV2CheckerTemplate(template, operation) || len(pod.Spec.InitContainers) != 2 || len(pod.Spec.EphemeralContainers) != 0 || len(pod.Spec.Containers) != len(template.Containers) || pod.Spec.ServiceAccountName != template.ServiceAccountName || pod.Spec.HostPID != template.HostPID { + return nil, fmt.Errorf("V2 checker Pod execution does not match its DaemonSet") + } + for _, volume := range template.Volumes { + found := false + for _, actual := range pod.Spec.Volumes { + if actual.Name == volume.Name { + found = reflect.DeepEqual(actual, volume) + } + } + if !found { + return nil, fmt.Errorf("V2 checker output or host volume changed") + } + } + serviceAccountVolume, valid := v2AdditionalVolumes(template.Volumes, pod.Spec.Volumes) + if !valid { + return nil, fmt.Errorf("V2 checker has an unrecognized additional volume") + } + instances := make([]corev1.ContainerStatus, 0, 2) + for index, expected := range template.InitContainers { + actual := pod.Spec.InitContainers[index] + if !sameV2ContainerExecution(expected, actual, serviceAccountVolume) { + return nil, fmt.Errorf("V2 checker or reporter execution changed") + } + var status *corev1.ContainerStatus + for _, candidate := range pod.Status.InitContainerStatuses { + if candidate.Name == expected.Name { + if status != nil { + return nil, fmt.Errorf("duplicate V2 checker container status") + } + copy := candidate + status = © + } + } + if status == nil || status.State.Terminated == nil || status.ImageID == "" || status.State.Terminated.ExitCode != 0 || status.State.Terminated.ContainerID == "" || status.State.Terminated.StartedAt.IsZero() || status.State.Terminated.FinishedAt.IsZero() || status.State.Terminated.FinishedAt.Before(&status.State.Terminated.StartedAt) || (status.ContainerID != "" && status.ContainerID != status.State.Terminated.ContainerID) { + return nil, fmt.Errorf("V2 checker or reporter did not complete successfully") + } + if index == 1 && status.State.Terminated.StartedAt.Before(&instances[0].State.Terminated.FinishedAt) { + return nil, fmt.Errorf("V2 reporter ran before its checker finished") + } + instances = append(instances, *status) + } + for index, expected := range template.Containers { + if !sameV2ContainerExecution(expected, pod.Spec.Containers[index], serviceAccountVolume) { + return nil, fmt.Errorf("V2 checker sidecar execution changed") + } + } + return instances, nil +} + +func validV2CheckerTemplate(template corev1.PodSpec, operation string) bool { + if operation != consts.SubCmdCheck && operation != consts.SubCmdInstall { + return false + } + if len(template.InitContainers) != 2 || template.InitContainers[0].Name != consts.ContainerNameInit || template.InitContainers[1].Name != consts.ContainerNameOutput { + return false + } + checker, reporter := template.InitContainers[0], template.InitContainers[1] + if !reflect.DeepEqual(checker.Command, []string{consts.CmdLonghornctlLocal, operation, consts.SubCmdPreflight, "--" + consts.CmdOptDataEngineMode, string(types.DataEngineModeV2Only)}) || len(checker.Args) != 0 || !reflect.DeepEqual(reporter.Command, []string{"cat", "/shared/output.json"}) || len(reporter.Args) != 0 { + return false + } + environment := map[string]corev1.EnvVar{} + for _, variable := range checker.Env { + if _, exists := environment[variable.Name]; exists { + return false + } + environment[variable.Name] = variable + } + mode, collection, node := environment[consts.EnvDataEngineMode], environment[consts.EnvPreflightCollectionOnly], environment[consts.EnvCurrentNodeID] + collectionValue := "true" + if operation == consts.SubCmdInstall { + attempt := environment[consts.EnvPreflightAttemptID] + if attempt.Value == "" || attempt.ValueFrom != nil { + return false + } + } + return mode.Value == string(types.DataEngineModeV2Only) && mode.ValueFrom == nil && collection.Value == collectionValue && collection.ValueFrom == nil && node.Value == "" && node.ValueFrom != nil && node.ValueFrom.FieldRef != nil && node.ValueFrom.FieldRef.FieldPath == "spec.nodeName" +} + +func sameV2ContainerExecution(expected, actual corev1.Container, serviceAccountVolume string) bool { + if !reflect.DeepEqual(expected.RestartPolicy, actual.RestartPolicy) || !reflect.DeepEqual(expected.StartupProbe, actual.StartupProbe) || !reflect.DeepEqual(expected.LivenessProbe, actual.LivenessProbe) || !reflect.DeepEqual(expected.ReadinessProbe, actual.ReadinessProbe) { + return false + } + if expected.Name != actual.Name || expected.Image != actual.Image || expected.WorkingDir != actual.WorkingDir || expected.Stdin != actual.Stdin || expected.TTY != actual.TTY || len(actual.VolumeDevices) != 0 || !reflect.DeepEqual(expected.Command, actual.Command) || !reflect.DeepEqual(expected.Args, actual.Args) || !reflect.DeepEqual(expected.EnvFrom, actual.EnvFrom) || !reflect.DeepEqual(expected.SecurityContext, actual.SecurityContext) || !reflect.DeepEqual(expected.Lifecycle, actual.Lifecycle) || len(expected.Env) != len(actual.Env) { + return false + } + seen := map[string]bool{} + for index, variable := range actual.Env { + if seen[variable.Name] { + return false + } + seen[variable.Name] = true + left, right := expected.Env[index].DeepCopy(), variable.DeepCopy() + for _, value := range []*corev1.EnvVar{left, right} { + if value.ValueFrom != nil && value.ValueFrom.FieldRef != nil && value.ValueFrom.FieldRef.APIVersion == "" { + value.ValueFrom.FieldRef.APIVersion = "v1" + } + } + if !reflect.DeepEqual(left, right) { + return false + } + } + for _, mount := range expected.VolumeMounts { + found := false + for _, actualMount := range actual.VolumeMounts { + if actualMount.Name == mount.Name { + found = reflect.DeepEqual(mount, actualMount) + } + } + if !found { + return false + } + } + for _, mount := range actual.VolumeMounts { + found := false + for _, expectedMount := range expected.VolumeMounts { + if reflect.DeepEqual(mount, expectedMount) { + found = true + } + } + if !found && (serviceAccountVolume == "" || !reflect.DeepEqual(mount, corev1.VolumeMount{Name: serviceAccountVolume, MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", ReadOnly: true})) { + return false + } + } + return true +} + +// Pod admission adds one API-token volume which is absent from the DaemonSet +// template. No other injected volume or mount may shadow the checker/report. +func v2AdditionalVolumes(expected, actual []corev1.Volume) (string, bool) { + names := map[string]bool{} + for _, volume := range expected { + names[volume.Name] = true + } + serviceAccountVolume := "" + seen := map[string]bool{} + for _, volume := range actual { + if seen[volume.Name] { + return "", false + } + seen[volume.Name] = true + if names[volume.Name] { + continue + } + projection := volume.Projected + if serviceAccountVolume != "" || !strings.HasPrefix(volume.Name, "kube-api-access-") || projection == nil || len(projection.Sources) != 3 || !reflect.DeepEqual(volume.VolumeSource, corev1.VolumeSource{Projected: projection}) { + return "", false + } + serviceAccountVolume = volume.Name + foundToken, foundCA, foundNamespace := false, false, false + for _, source := range projection.Sources { + switch { + case source.ServiceAccountToken != nil: + token := source.ServiceAccountToken + if foundToken || token.Path != "token" || !reflect.DeepEqual(source, corev1.VolumeProjection{ServiceAccountToken: token}) { + return "", false + } + foundToken = true + case source.ConfigMap != nil: + config := source.ConfigMap + if foundCA || config.Name != "kube-root-ca.crt" || len(config.Items) != 1 || config.Items[0].Key != "ca.crt" || config.Items[0].Path != "ca.crt" || !reflect.DeepEqual(source, corev1.VolumeProjection{ConfigMap: config}) { + return "", false + } + foundCA = true + case source.DownwardAPI != nil: + info := source.DownwardAPI + if foundNamespace || len(info.Items) != 1 || info.Items[0].Path != "namespace" || info.Items[0].FieldRef == nil || info.Items[0].FieldRef.FieldPath != "metadata.namespace" || info.Items[0].ResourceFieldRef != nil || !reflect.DeepEqual(source, corev1.VolumeProjection{DownwardAPI: info}) { + return "", false + } + foundNamespace = true + default: + return "", false + } + } + if !foundToken || !foundCA || !foundNamespace { + return "", false + } + } + return serviceAccountVolume, true +} + +func readV2PodReport(client kubeclient.Interface) v2ReportLogReader { + return func(ctx context.Context, namespace, pod string) ([]byte, error) { + limit := int64(maxV2ReportBytes + 1) + stream, err := client.CoreV1().Pods(namespace).GetLogs(pod, &corev1.PodLogOptions{Container: consts.ContainerNameOutput, LimitBytes: &limit}).Stream(ctx) + if err != nil { + return nil, err + } + data, readErr := io.ReadAll(io.LimitReader(stream, limit)) + closeErr := stream.Close() + if readErr != nil { + return nil, readErr + } + if closeErr != nil { + return nil, closeErr + } + return data, nil + } +} + +// Bind the admitted Create response to this invocation's requested inputs. +// Otherwise an admission rewrite could lower the requested hugepage check or +// replace the fresh output volume before either inventory sample is taken. +func validateV2CreatedDaemonSet(request, created *appsv1.DaemonSet, operation string) error { + if created == nil || created.UID == "" || created.Generation <= 0 || created.Name != request.Name || created.Namespace != request.Namespace { + return fmt.Errorf("V2 DaemonSet creation returned no identified receipt") + } + left, right := request.Spec.Template.Spec, created.Spec.Template.Spec + if !validV2CheckerTemplate(right, operation) || !reflect.DeepEqual(request.Spec.Selector, created.Spec.Selector) || !reflect.DeepEqual(left.NodeSelector, right.NodeSelector) || !reflect.DeepEqual(left.Affinity, right.Affinity) || !reflect.DeepEqual(left.Tolerations, right.Tolerations) || left.NodeName != right.NodeName || left.ServiceAccountName != right.ServiceAccountName || left.HostPID != right.HostPID || left.HostNetwork != right.HostNetwork || left.HostIPC != right.HostIPC || len(left.InitContainers) != len(right.InitContainers) || len(left.Containers) != len(right.Containers) || len(right.EphemeralContainers) != 0 || len(left.Volumes) != len(right.Volumes) { + return fmt.Errorf("V2 DaemonSet admission changed required execution or node coverage") + } + for index, container := range left.InitContainers { + if !sameV2ContainerExecution(container, right.InitContainers[index], "") { + return fmt.Errorf("V2 DaemonSet admission changed checker inputs") + } + } + for index, container := range left.Containers { + if !sameV2ContainerExecution(container, right.Containers[index], "") { + return fmt.Errorf("V2 DaemonSet admission changed companion execution") + } + } + for index, volume := range left.Volumes { + actual := right.Volumes[index].DeepCopy() + expected := volume.DeepCopy() + for _, item := range []*corev1.Volume{expected, actual} { + if item.HostPath != nil && item.HostPath.Type != nil && *item.HostPath.Type == corev1.HostPathUnset { + item.HostPath.Type = nil + } + } + if !reflect.DeepEqual(expected, actual) { + return fmt.Errorf("V2 DaemonSet admission changed the host or output volume") + } + } + return nil +} diff --git a/pkg/remote/preflight/checker_collection_v2_test.go b/pkg/remote/preflight/checker_collection_v2_test.go new file mode 100644 index 000000000..63c2e2fdf --- /dev/null +++ b/pkg/remote/preflight/checker_collection_v2_test.go @@ -0,0 +1,283 @@ +package preflight + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeclient "k8s.io/client-go/kubernetes" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" +) + +func TestV2ReportRejectsMissingAmbiguousAndPartialEvidence(t *testing.T) { + for _, input := range []string{ + ``, `{}`, `{"dataEngineMode":"v2-only","log":null}`, + `{"dataEngineMode":"v2-only","log":{}}`, + `{"dataEngineMode":"v1-capable","log":{"info":["ok"]}}`, + `{"dataEngineMode":"v2-only","dataEngineMode":"v2-only","log":{"info":["ok"]}}`, + `{"dataEngineMode":"v2-only","log":{"info":["ok"],"info":["ok"]}}`, + `{"dataEngineMode":"v2-only","log":{"info":["ok"],"unknown":true}}`, + `{"dataEngineMode":"v2-only","DataEngineMode":"v2-only","log":{"info":["ok"]}}`, + `{"dataEngineMode":"v2-only","log":{"Info":["ok"]}}`, + `{"dataEngineMode":"v2-only","log":{"info":["ok"]}} {}`, + `{"dataEngineMode":"v2-only","log":{"info":["ok"]}`, + strings.Repeat(" ", maxV2ReportBytes+1), + } { + if _, err := decodeV2Report([]byte(input)); err == nil { + t.Fatal("accepted missing, ambiguous, partial or unbounded V2 report") + } + } + for _, input := range []string{ + `{"dataEngineMode":"v2-only","log":{"info":["qualified"]}}`, + `{"dataEngineMode":"v2-only","log":{"info":["checks ran"],"error":["NVMe missing"]}}`, + } { + if _, err := decodeV2Report([]byte(input)); err != nil { + t.Fatalf("lost complete report: %v", err) + } + } +} + +type v2CollectionFixture struct { + mu sync.Mutex + ds *appsv1.DaemonSet + pods []corev1.Pod + paginate, failList bool +} + +func newV2CollectionFixture(t *testing.T) (*v2CollectionFixture, *kubeclient.Clientset, *appsv1.DaemonSet) { + t.Helper() + checker := Checker{CheckerCmdOptions: CheckerCmdOptions{DataEngineMode: string(types.DataEngineModeV2Only), GlobalCmdOptions: types.GlobalCmdOptions{Namespace: "test", Image: "fixture/cli:current"}, HugePageSize: 2048}, appName: "checker"} + ds := checker.newDaemonSet() + ds.UID, ds.Generation, ds.ResourceVersion = "ds-uid", 1, "1" + ds.Status = appsv1.DaemonSetStatus{ObservedGeneration: 1, DesiredNumberScheduled: 1, CurrentNumberScheduled: 1, UpdatedNumberScheduled: 1} + // The expected object is the API Create response, not the pre-serialization + // request: omitted empty slices have already become nil on that response. + encoded, err := json.Marshal(ds) + if err != nil { + t.Fatal(err) + } + var admitted appsv1.DaemonSet + if err := json.Unmarshal(encoded, &admitted); err != nil { + t.Fatal(err) + } + ds = &admitted + pod := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "checker-node", Namespace: "test", UID: "pod-uid", Labels: map[string]string{"app": "checker"}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "DaemonSet", Name: ds.Name, UID: ds.UID, Controller: ptr.To(true)}}}, Spec: *ds.Spec.Template.Spec.DeepCopy()} + pod.Spec.NodeName = "node-1" + for index, name := range []string{consts.ContainerNameInit, consts.ContainerNameOutput} { + start := metav1.NewTime(time.Unix(100+int64(index)*2, 0)) + end := metav1.NewTime(start.Add(time.Second)) + id := fmt.Sprintf("containerd://instance-%d", index) + pod.Status.InitContainerStatuses = append(pod.Status.InitContainerStatuses, corev1.ContainerStatus{Name: name, Image: ds.Spec.Template.Spec.InitContainers[index].Image, ImageID: "sha256:fixture", ContainerID: id, State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0, StartedAt: start, FinishedAt: end, ContainerID: id}}}) + } + fixture := &v2CollectionFixture{ds: ds.DeepCopy(), pods: []corev1.Pod{pod}} + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + fixture.mu.Lock() + defer fixture.mu.Unlock() + response.Header().Set("Content-Type", "application/json") + var object interface{} + switch request.URL.Path { + case "/apis/apps/v1/namespaces/test/daemonsets/checker": + object = fixture.ds + case "/api/v1/namespaces/test/pods": + if request.URL.Query().Get("labelSelector") != "" { + t.Error("V2 collector used an incomplete label-filtered inventory") + } + if fixture.failList { + http.Error(response, "inventory unavailable", http.StatusForbidden) + return + } + list := &corev1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "10"}, Items: fixture.pods} + if fixture.paginate && request.URL.Query().Get("continue") == "" { + list.Items = nil + list.Continue = "next" + } + object = list + default: + t.Errorf("unexpected API request %s", request.URL.Path) + http.NotFound(response, request) + return + } + if err := json.NewEncoder(response).Encode(object); err != nil { + t.Error(err) + } + })) + t.Cleanup(server.Close) + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + return fixture, client, ds +} + +func TestV2CollectorBindsCompleteCurrentExecution(t *testing.T) { + for _, test := range []struct { + name string + before, after func(*v2CollectionFixture) + logError bool + valid bool + }{ + {name: "complete", valid: true}, + {name: "native-service-account-projection", before: addV2FixtureServiceAccount, valid: true}, + {name: "complete-paginated", before: func(f *v2CollectionFixture) { f.paginate = true }, valid: true}, + {name: "missing-pod", before: func(f *v2CollectionFixture) { f.pods = nil }}, + {name: "lost-labels", before: func(f *v2CollectionFixture) { f.pods[0].Labels = nil }}, + {name: "foreign-controller", before: func(f *v2CollectionFixture) { f.pods[0].OwnerReferences[0].UID = "foreign" }}, + {name: "noncontroller-owner", before: func(f *v2CollectionFixture) { f.pods[0].OwnerReferences[0].Controller = ptr.To(false) }}, + {name: "terminating", before: func(f *v2CollectionFixture) { now := metav1.Now(); f.pods[0].DeletionTimestamp = &now }}, + {name: "duplicate-node", before: func(f *v2CollectionFixture) { + p := f.pods[0].DeepCopy() + p.Name = "duplicate" + p.UID = "other" + f.pods = append(f.pods, *p) + f.ds.Status.DesiredNumberScheduled = 2 + f.ds.Status.CurrentNumberScheduled = 2 + f.ds.Status.UpdatedNumberScheduled = 2 + }}, + {name: "unbound-executable", before: func(f *v2CollectionFixture) { f.pods[0].Spec.InitContainers[0].Command = []string{"old-checker"} }}, + {name: "restartable-installer", before: func(f *v2CollectionFixture) { + f.pods[0].Spec.InitContainers[0].RestartPolicy = ptr.To(corev1.ContainerRestartPolicyAlways) + }}, + {name: "injected-liveness-probe", before: func(f *v2CollectionFixture) { + f.pods[0].Spec.InitContainers[0].LivenessProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"false"}}}} + }}, + {name: "reporter-shadowed-output", before: func(f *v2CollectionFixture) { + f.pods[0].Spec.Volumes = append(f.pods[0].Spec.Volumes, corev1.Volume{Name: "stale", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "stale"}}}}) + f.pods[0].Spec.InitContainers[1].VolumeMounts = append(f.pods[0].Spec.InitContainers[1].VolumeMounts, corev1.VolumeMount{Name: "stale", MountPath: "/shared/output.json", SubPath: "report.json"}) + }}, + {name: "service-account-shadowed-output", before: func(f *v2CollectionFixture) { + addV2FixtureServiceAccount(f) + f.pods[0].Spec.InitContainers[1].VolumeMounts[1].MountPath = "/shared/output.json" + }}, + {name: "failed-checker", before: func(f *v2CollectionFixture) { f.pods[0].Status.InitContainerStatuses[0].State.Terminated.ExitCode = 1 }}, + {name: "missing-container-id", before: func(f *v2CollectionFixture) { + f.pods[0].Status.InitContainerStatuses[0].State.Terminated.ContainerID = "" + }}, + {name: "early-reporter", before: func(f *v2CollectionFixture) { + f.pods[0].Status.InitContainerStatuses[1].State.Terminated.StartedAt = metav1.NewTime(time.Unix(99, 0)) + }}, + {name: "list-error", before: func(f *v2CollectionFixture) { f.failList = true }}, + {name: "log-error", logError: true}, + {name: "replacement-pod", after: func(f *v2CollectionFixture) { f.pods[0].UID = k8stypes.UID("replacement") }}, + {name: "new-container-attempt", after: func(f *v2CollectionFixture) { f.pods[0].Status.InitContainerStatuses[0].RestartCount++ }}, + {name: "changed-spec", after: func(f *v2CollectionFixture) { f.ds.Spec.Template.Spec.InitContainers[0].Image = "other" }}, + {name: "inventory-disappears", after: func(f *v2CollectionFixture) { f.pods = nil }}, + } { + t.Run(test.name, func(t *testing.T) { + fixture, client, ds := newV2CollectionFixture(t) + if test.before != nil { + test.before(fixture) + } + output, err := collectV2Reports(t.Context(), client, ds, func(context.Context, string, string) ([]byte, error) { + if test.logError { + return nil, fmt.Errorf("logs unavailable") + } + if test.after != nil { + fixture.mu.Lock() + test.after(fixture) + fixture.mu.Unlock() + } + return []byte(`{"dataEngineMode":"v2-only","log":{"info":["checks completed"]}}`), nil + }) + if (err == nil) != test.valid || (test.valid && !strings.Contains(output, "node-1:")) || (!test.valid && output != "") { + t.Fatalf("incorrect execution qualification: output=%q err=%v", output, err) + } + }) + } +} + +func addV2FixtureServiceAccount(f *v2CollectionFixture) { + name := "kube-api-access-abcde" + f.pods[0].Spec.Volumes = append(f.pods[0].Spec.Volumes, corev1.Volume{Name: name, VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{DefaultMode: ptr.To(int32(420)), Sources: []corev1.VolumeProjection{ + {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Path: "token", ExpirationSeconds: ptr.To(int64(3607))}}, + {ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "kube-root-ca.crt"}, Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}}}, + {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}}, + }}}}) + for index := range f.pods[0].Spec.InitContainers { + f.pods[0].Spec.InitContainers[index].VolumeMounts = append(f.pods[0].Spec.InitContainers[index].VolumeMounts, corev1.VolumeMount{Name: name, ReadOnly: true, MountPath: "/var/run/secrets/kubernetes.io/serviceaccount"}) + } + for index := range f.pods[0].Spec.Containers { + f.pods[0].Spec.Containers[index].VolumeMounts = append(f.pods[0].Spec.Containers[index].VolumeMounts, corev1.VolumeMount{Name: name, ReadOnly: true, MountPath: "/var/run/secrets/kubernetes.io/serviceaccount"}) + } +} + +func TestV2LogReadUsesTheFixedBoundedReporter(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + if r.URL.Path != "/api/v1/namespaces/test/pods/checker/log" || query.Get("container") != consts.ContainerNameOutput || query.Get("limitBytes") != fmt.Sprint(maxV2ReportBytes+1) || query.Get("follow") == "true" || query.Get("previous") == "true" || query.Get("timestamps") == "true" || query.Get("tailLines") != "" { + t.Errorf("unqualified log request %s", r.URL) + } + _, _ = w.Write([]byte(strings.Repeat("x", maxV2ReportBytes+100))) + })) + t.Cleanup(server.Close) + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + data, err := readV2PodReport(client)(t.Context(), "test", "checker") + if err != nil || len(data) != maxV2ReportBytes+1 { + t.Fatalf("log stream not bounded: length=%d err=%v", len(data), err) + } + if _, err := decodeV2Report(data); err == nil { + t.Fatal("oversized log qualified") + } +} + +func TestV2CollectorReturnsFullReportWithFailedQualification(t *testing.T) { + _, client, ds := newV2CollectionFixture(t) + output, err := collectV2Reports(t.Context(), client, ds, func(context.Context, string, string) ([]byte, error) { + return []byte(`{"dataEngineMode":"v2-only","log":{"info":["checks completed"],"error":["NVMe missing"]}}`), nil + }) + if err == nil || !strings.Contains(output, "NVMe missing") || !strings.Contains(output, "node-1:") { + t.Fatalf("failed qualification lost its complete report: %q %v", output, err) + } +} + +func TestV2AdmissionCannotLowerRequestedChecksOrReplaceOutput(t *testing.T) { + _, _, created := newV2CollectionFixture(t) + request := created.DeepCopy() + request.Spec.Template.Spec.InitContainers[1].Env = []corev1.EnvVar{} + created.Spec.Template.Spec.Volumes[0].HostPath.Type = ptr.To(corev1.HostPathUnset) + if err := validateV2CreatedDaemonSet(request, created, consts.SubCmdCheck); err != nil { + t.Fatalf("native defaults were mistaken for an execution rewrite: %v", err) + } + for _, mutate := range []func(*appsv1.DaemonSet){ + func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.InitContainers[0].RestartPolicy = ptr.To(corev1.ContainerRestartPolicyAlways) + }, + func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.InitContainers[0].StartupProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"false"}}}} + }, + func(ds *appsv1.DaemonSet) { + for index, variable := range ds.Spec.Template.Spec.InitContainers[0].Env { + if variable.Name == consts.EnvHugePageSize { + ds.Spec.Template.Spec.InitContainers[0].Env[index].Value = "2" + } + } + }, + func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Volumes[1].VolumeSource = corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "stale-report"}}} + }, + func(ds *appsv1.DaemonSet) { ds.Spec.Template.Spec.NodeSelector = map[string]string{"node": "other"} }, + } { + changed := created.DeepCopy() + mutate(changed) + if err := validateV2CreatedDaemonSet(request, changed, consts.SubCmdCheck); err == nil { + t.Fatal("admission rewrote the requested qualification") + } + } +} diff --git a/pkg/remote/preflight/data_engine_mode.go b/pkg/remote/preflight/data_engine_mode.go new file mode 100644 index 000000000..8f698845c --- /dev/null +++ b/pkg/remote/preflight/data_engine_mode.go @@ -0,0 +1,149 @@ +package preflight + +import ( + "fmt" + "math" + "regexp" + "strconv" + "strings" + "time" + + "github.com/longhorn/cli/pkg/types" +) + +func validateV2Hugepages(size int) error { + if size <= 0 || size%2 != 0 || int64(size) > math.MaxInt64/(1024*1024) { + return fmt.Errorf("v2-only requires a positive, bounded amount of 2MiB hugepages") + } + return nil +} + +func validateV2Driver(driver string) error { + if driver != "" && driver != "vfio-pci" && driver != "uio_pci_generic" { + return fmt.Errorf("unsupported v2-only userspace driver") + } + return nil +} + +func (options *CheckerCmdOptions) ValidateDataEngineMode() error { + mode, err := types.ParseDataEngineMode(options.DataEngineMode) + if err != nil { + return err + } + options.DataEngineMode = string(mode) + if mode != types.DataEngineModeV2Only { + return nil + } + if err := validateV2Hugepages(options.HugePageSize); err != nil { + return err + } + if options.UserspaceDriver != "" && options.UserspaceDriver != "vfio_pci" && options.UserspaceDriver != "uio_pci_generic" { + return fmt.Errorf("unsupported v2-only userspace kernel module") + } + return nil +} + +func (options *InstallerCmdOptions) ValidateDataEngineMode() error { + mode, err := types.ParseDataEngineMode(options.DataEngineMode) + if err != nil { + return err + } + options.DataEngineMode = string(mode) + if mode != types.DataEngineModeV2Only { + return nil + } + _, _, err = options.V2SPDKSetupEnvironment() + return err +} + +var v2PCIAddress = regexp.MustCompile(`^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[01][0-9a-fA-F]\.[0-7]$`) + +func normalizeV2PCIAllowed(value string) (string, error) { + if value == "" || value == "none" { + return "none", nil + } + addresses := strings.FieldsFunc(value, func(r rune) bool { return r == ',' || r == ' ' || r == '\t' }) + if len(addresses) == 0 { + return "", fmt.Errorf("v2-only PCI selection is unresolved") + } + seen := map[string]bool{} + for index, address := range addresses { + address = strings.ToLower(address) + if !v2PCIAddress.MatchString(address) || seen[address] { + return "", fmt.Errorf("v2-only requires unique full PCI addresses or none") + } + seen[address], addresses[index] = true, address + } + return strings.Join(addresses, " "), nil +} + +// V2SPDKSetupEnvironment owns the pinned setup script's relevant inputs. +// It does not establish host qualification: callers must bind the script and +// host prerequisites before writes, then verify actual allocation afterward. +func (options InstallerCmdOptions) V2SPDKSetupEnvironment() ([]string, string, error) { + mode, err := types.ParseDataEngineMode(options.DataEngineMode) + if err != nil || mode != types.DataEngineModeV2Only { + return nil, "", fmt.Errorf("v2-only setup requires an explicit capability") + } + if options.OperatingSystem != "" { + return nil, "", fmt.Errorf("v2-only installation supports qualified package-manager hosts; COS and explicit OS overrides are unsupported") + } + if err := validateV2Hugepages(options.HugePageSize); err != nil { + return nil, "", err + } + if err := validateV2Driver(options.DriverOverride); err != nil { + return nil, "", err + } + if options.RestartKubelet { + window, err := time.ParseDuration(options.RestartKubeletWindow) + if err != nil || window < 0 { + return nil, "", fmt.Errorf("v2-only requires a valid nonnegative kubelet restart window") + } + } + pciAllowed, err := normalizeV2PCIAllowed(options.AllowPci) + if err != nil { + return nil, "", err + } + node, seen := "0", map[string]bool{} + if options.SpdkOptions != "" { + for _, option := range strings.Split(options.SpdkOptions, ",") { + name, value, assigned := strings.Cut(option, "=") + if !assigned || seen[name] { + return nil, "", fmt.Errorf("invalid or duplicate v2-only SPDK option") + } + seen[name] = true + switch name { + case "HUGEMEM": + if value != strconv.Itoa(options.HugePageSize) { + return nil, "", fmt.Errorf("SPDK memory override contradicts v2-only hugepage intent") + } + case "HUGEPGSZ": + if value != "2048" { + return nil, "", fmt.Errorf("v2-only setup requires 2MiB hugepages") + } + case "PCI_ALLOWED": + selected, err := normalizeV2PCIAllowed(value) + if err != nil || selected != pciAllowed { + return nil, "", fmt.Errorf("SPDK PCI override contradicts v2-only selection") + } + case "DRIVER_OVERRIDE": + if value != options.DriverOverride { + return nil, "", fmt.Errorf("SPDK driver override contradicts v2-only selection") + } + case "HUGENODE": + number, err := strconv.ParseUint(value, 10, 31) + if err != nil || strconv.FormatUint(number, 10) != value { + return nil, "", fmt.Errorf("v2-only setup requires one canonical NUMA node number") + } + node = value + default: + return nil, "", fmt.Errorf("unsupported SPDK option in v2-only setup") + } + } + } + return []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "LC_ALL=C", + "HUGEMEM=" + strconv.Itoa(options.HugePageSize), "HUGEPGSZ=2048", "HUGENODE=" + node, + "PCI_ALLOWED=" + pciAllowed, "DRIVER_OVERRIDE=" + options.DriverOverride, "TARGET_USER=root", + }, node, nil +} diff --git a/pkg/remote/preflight/data_engine_mode_test.go b/pkg/remote/preflight/data_engine_mode_test.go new file mode 100644 index 000000000..0bc0f66ae --- /dev/null +++ b/pkg/remote/preflight/data_engine_mode_test.go @@ -0,0 +1,158 @@ +package preflight + +import ( + "reflect" + "strconv" + "strings" + "testing" + + "github.com/longhorn/cli/pkg/consts" +) + +func TestCheckerModeRequiresResolvedV2Prerequisites(t *testing.T) { + for _, test := range []struct { + name, mode, driver string + hugepages int + valid bool + }{ + {"default", "", "", 0, true}, + {"v1", "v1-capable", "", 0, true}, + {"v2", "v2-only", "", 2048, true}, + {"v2-vfio-module", "v2-only", "vfio_pci", 2048, true}, + {"v2-uio-module", "v2-only", "uio_pci_generic", 2048, true}, + {"vfio-driver-is-not-module", "v2-only", "vfio-pci", 2048, false}, + {"invalid-mode", "v2", "", 2048, false}, + {"missing-pages", "v2-only", "", 0, false}, + {"negative-pages", "v2-only", "", -2, false}, + {"unaligned-pages", "v2-only", "", 1, false}, + {"invalid-driver", "v2-only", "unknown", 2048, false}, + } { + t.Run(test.name, func(t *testing.T) { + options := CheckerCmdOptions{DataEngineMode: test.mode, HugePageSize: test.hugepages, UserspaceDriver: test.driver} + if err := options.ValidateDataEngineMode(); (err == nil) != test.valid { + t.Fatalf("unexpected checker mode validation: %v", err) + } + if test.valid && options.DataEngineMode == "" { + t.Fatal("default capability was not normalized") + } + }) + } +} + +func TestV2SPDKEnvironmentBindsNativeMemoryPCIAndNUMAInputs(t *testing.T) { + options := InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048, AllowPci: "0000:00:01.0,0000:00:02.0", DriverOverride: "vfio-pci", SpdkOptions: "HUGENODE=1"} + environment, node, err := options.V2SPDKSetupEnvironment() + if err != nil || node != "1" { + t.Fatalf("unresolved setup profile: %v", err) + } + want := []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "LC_ALL=C", "HUGEMEM=2048", "HUGEPGSZ=2048", "HUGENODE=1", "PCI_ALLOWED=0000:00:01.0 0000:00:02.0", "DRIVER_OVERRIDE=vfio-pci", "TARGET_USER=root"} + if !reflect.DeepEqual(environment, want) { + t.Fatalf("setup environment does not bind the resolved options: %v", environment) + } + options.AllowPci, options.SpdkOptions = "", "" + environment, node, err = options.V2SPDKSetupEnvironment() + if err != nil || node != "0" || !strings.Contains(strings.Join(environment, "\n"), "PCI_ALLOWED=none\n") { + t.Fatal("empty PCI selection did not remain fenced, or native node-0 default changed") + } +} + +func TestRemoteCheckerBindsCapabilityToTheLocalExecutable(t *testing.T) { + for _, mode := range []string{"", "v1-capable", "v2-only"} { + for _, explicitSPDK := range []bool{false, true} { + checker := Checker{CheckerCmdOptions: CheckerCmdOptions{DataEngineMode: mode, HugePageSize: 2048, EnableSpdk: explicitSPDK}, appName: "checker"} + if err := checker.ValidateDataEngineMode(); err != nil { + t.Fatal(err) + } + pod := checker.newDaemonSet().Spec.Template.Spec + container := pod.InitContainers[0] + environment := map[string]string{} + for _, variable := range container.Env { + environment[variable.Name] = variable.Value + } + if environment["DATA_ENGINE_MODE"] != checker.DataEngineMode || environment["ENABLE_SPDK"] != strconv.FormatBool(explicitSPDK) { + t.Fatal("remote job lost capability or overwrote the explicit SPDK flag") + } + command := strings.Join(append(append([]string{}, container.Command...), container.Args...), " ") + if strings.Contains(command, "--data-engine-mode v2-only") != (mode == "v2-only") { + t.Fatal("V2-only job can silently run an older local executable, or default argv changed") + } + } + } +} + +func TestRemoteCheckerRejectsInvalidModeBeforeConnectionInitialization(t *testing.T) { + checker := Checker{CheckerCmdOptions: CheckerCmdOptions{DataEngineMode: "unknown"}} + if err := checker.Init(); err == nil || !strings.Contains(err.Error(), "data-engine-mode") { + t.Fatal("invalid capability reached connection initialization") + } +} + +func TestRemoteInstallerBindsV2ExecutableBeforeAnyMutation(t *testing.T) { + installer := Installer{InstallerCmdOptions: InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}, appName: "installer"} + if err := installer.ValidateDataEngineMode(); err != nil { + t.Fatal(err) + } + container := installer.newDaemonSetForPackageManager().Spec.Template.Spec.InitContainers[0] + if !strings.Contains(strings.Join(container.Command, " "), "--data-engine-mode v2-only") { + t.Fatal("V2 installer may silently execute an older V1-capable image") + } + if err := installer.InstallByContainerOptimizedOS(); err == nil { + t.Fatal("V2 installer reached unsupported COS mutations") + } +} + +func TestRemoteV2InstallerBindsItsReportAndAttemptProfile(t *testing.T) { + installer := Installer{InstallerCmdOptions: InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}, appName: consts.AppNamePreflightInstaller + "-fixture"} + if err := installer.ValidateDataEngineMode(); err != nil { + t.Fatal(err) + } + template := installer.newDaemonSetForPackageManager().Spec.Template.Spec + if !validV2CheckerTemplate(template, consts.SubCmdInstall) || validV2CheckerTemplate(template, consts.SubCmdCheck) { + t.Fatal("V2 installer transport lost its fixed executable, report, or attempt identity") + } +} + +func TestRemoteInstallerRejectsInvalidModeBeforeConnectionInitialization(t *testing.T) { + installer := Installer{InstallerCmdOptions: InstallerCmdOptions{DataEngineMode: "unknown"}} + if err := installer.Init(); err == nil || !strings.Contains(err.Error(), "data-engine-mode") { + t.Fatal("invalid mode reached remote installer initialization") + } +} + +func TestInstallerModeRejectsUnqualifiedOrConflictingSetupBeforeWrites(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*InstallerCmdOptions) + valid bool + }{ + {"compatible", func(*InstallerCmdOptions) {}, true}, + {"invalid-mode", func(o *InstallerCmdOptions) { o.DataEngineMode = "v2" }, false}, + {"cos", func(o *InstallerCmdOptions) { o.OperatingSystem = "cos" }, false}, + {"talos", func(o *InstallerCmdOptions) { o.OperatingSystem = "talos" }, false}, + {"zero-pages", func(o *InstallerCmdOptions) { o.HugePageSize = 0 }, false}, + {"unaligned-pages", func(o *InstallerCmdOptions) { o.HugePageSize = 3 }, false}, + {"bad-window", func(o *InstallerCmdOptions) { o.RestartKubelet, o.RestartKubeletWindow = true, "invalid" }, false}, + {"negative-window", func(o *InstallerCmdOptions) { o.RestartKubelet, o.RestartKubeletWindow = true, "-1m" }, false}, + {"invalid-driver", func(o *InstallerCmdOptions) { o.DriverOverride = "none" }, false}, + {"invalid-pci", func(o *InstallerCmdOptions) { o.AllowPci = "$(untrusted)" }, false}, + {"conflicting-pages", func(o *InstallerCmdOptions) { o.SpdkOptions = "HUGEMEM=0" }, false}, + {"conflicting-size", func(o *InstallerCmdOptions) { o.SpdkOptions = "HUGEPGSZ=1048576" }, false}, + {"override-page-count", func(o *InstallerCmdOptions) { o.SpdkOptions = "NRHUGE=0" }, false}, + {"unknown-option", func(o *InstallerCmdOptions) { o.SpdkOptions = "PERSIST_HUGE=yes" }, false}, + {"duplicate-option", func(o *InstallerCmdOptions) { o.SpdkOptions = "HUGEMEM=2048,HUGEMEM=2048" }, false}, + {"eval-node", func(o *InstallerCmdOptions) { o.SpdkOptions = "HUGENODE=$(untrusted)" }, false}, + {"resolved-options", func(o *InstallerCmdOptions) { o.SpdkOptions = "HUGEMEM=2048,HUGEPGSZ=2048,HUGENODE=0" }, true}, + } { + t.Run(test.name, func(t *testing.T) { + options := InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048, AllowPci: "none"} + test.mutate(&options) + if err := options.ValidateDataEngineMode(); (err == nil) != test.valid { + t.Fatalf("unexpected installer profile validation: %v", err) + } + }) + } + options := InstallerCmdOptions{SpdkOptions: "legacy-unparsed", OperatingSystem: "cos"} + if err := options.ValidateDataEngineMode(); err != nil || options.DataEngineMode != "v1-capable" { + t.Fatal("default installation behavior changed") + } +} diff --git a/pkg/remote/preflight/installer.go b/pkg/remote/preflight/installer.go index 1ba175422..8e1c897bc 100644 --- a/pkg/remote/preflight/installer.go +++ b/pkg/remote/preflight/installer.go @@ -1,11 +1,16 @@ package preflight import ( + "context" + "crypto/rand" "encoding/json" "path/filepath" "reflect" + "strings" "time" + stderrors "errors" + "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -15,6 +20,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeclient "k8s.io/client-go/kubernetes" @@ -34,13 +40,16 @@ type Installer struct { kubeClient *kubeclient.Clientset - appName string // App name of the DaemonSet. + appName string // App name of the DaemonSet. + createdDaemonSet *appsv1.DaemonSet + createdRBAC *v2RBACReceipt } // InstallerCmdOptions holds the options for the command. type InstallerCmdOptions struct { types.GlobalCmdOptions + DataEngineMode string OperatingSystem string UpdatePackages bool @@ -55,6 +64,13 @@ type InstallerCmdOptions struct { // Init initializes the Installer. func (remote *Installer) Init() error { + // Cleanup-only commands do not require host setup options. Validate the + // capability here and validate the complete installation profile in Run. + mode, err := types.ParseDataEngineMode(remote.DataEngineMode) + if err != nil { + return err + } + remote.DataEngineMode = string(mode) kubeClient, err := kubeutils.NewKubeClient("", remote.KubeConfigPath) if err != nil { return err @@ -68,6 +84,9 @@ func (remote *Installer) Init() error { default: remote.appName = consts.AppNamePreflightInstaller } + if mode == types.DataEngineModeV2Only { + remote.appName += "-" + strings.ToLower(rand.Text()) + } return nil } @@ -76,6 +95,9 @@ func (remote *Installer) Init() error { // It checks if the operating system is specified, and installs the dependencies accordingly. // If the operating system is not specified, it installs the dependencies with package manager. func (remote *Installer) Run() (string, error) { + if err := remote.ValidateDataEngineMode(); err != nil { + return "", err + } operatingSystem := consts.OperatingSystem(remote.OperatingSystem) switch operatingSystem { case consts.OperatingSystemContainerOptimizedOS: @@ -99,7 +121,14 @@ func (remote *Installer) Run() (string, error) { Verbs: []string{"get"}, }, } - err := kubeutils.CreateRbac(remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + var err error + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + remote.createdRBAC, err = createV2RBAC(ctx, remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + } else { + err = kubeutils.CreateRbac(remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + } if err != nil { return "", err } @@ -111,7 +140,7 @@ func (remote *Installer) Run() (string, error) { } output, err := remote.InstallByPackageManager() if err != nil { - return "", errors.Wrapf(err, "failed to install dependencies with package manager") + return output, errors.Wrapf(err, "failed to install dependencies with package manager") } logrus.Info("Installed dependencies with package manager") @@ -122,6 +151,22 @@ func (remote *Installer) Run() (string, error) { // Cleanup deletes the DaemonSet created for the preflight install when it's installed with package manager. func (remote *Installer) Cleanup() error { var resultErr error + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + var daemonSetErr error + if remote.createdDaemonSet != nil { + uid := remote.createdDaemonSet.UID + if uid == "" { + daemonSetErr = errors.New("refusing V2 installer cleanup without its created UID") + } else if err := remote.kubeClient.AppsV1().DaemonSets(remote.createdDaemonSet.Namespace).Delete(ctx, remote.createdDaemonSet.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + daemonSetErr = err + } + } + rbacContext, rbacCancel := context.WithTimeout(context.Background(), time.Minute) + defer rbacCancel() + return stderrors.Join(daemonSetErr, cleanupV2RBAC(rbacContext, remote.kubeClient, remote.createdRBAC)) + } if err := commonkube.DeleteDaemonSet(remote.kubeClient, remote.Namespace, remote.appName); err != nil { resultErr = errors.Wrap(err, "failed to delete DaemonSet") @@ -141,6 +186,12 @@ func (remote *Installer) Cleanup() error { // InstallByContainerOptimizedOS installs the dependencies on Container Optimized OS. // It creates a ConfigMap and a DaemonSet. Then it waits for the DaemonSet to be ready. func (remote *Installer) InstallByContainerOptimizedOS() error { + if err := remote.ValidateDataEngineMode(); err != nil { + return err + } + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + return errors.New("V2-only installation is not qualified for Container-Optimized OS") + } newConfigMap := remote.newConfigMapForContainerOptimizedOS() _, err := commonkube.CreateConfigMap(remote.kubeClient, newConfigMap) if err != nil { @@ -172,14 +223,30 @@ func (remote *Installer) InstallByContainerOptimizedOS() error { // - Successfully probed module dm_crypt // - Successfully started service iscsid func (remote *Installer) InstallByPackageManager() (string, error) { + if err := remote.ValidateDataEngineMode(); err != nil { + return "", err + } newDaemonSet, err := kubeutils.PrepareDaemonSet(remote.newDaemonSetForPackageManager(), remote.kubeClient, remote.NodeSelector, remote.ImagePullSecret, remote.Tolerations) if err != nil { return "", err } - daemonSet, err := commonkube.CreateDaemonSet(remote.kubeClient, newDaemonSet) + var daemonSet *appsv1.DaemonSet + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + daemonSet, err = remote.kubeClient.AppsV1().DaemonSets(remote.Namespace).Create(ctx, newDaemonSet, metav1.CreateOptions{}) + } else { + daemonSet, err = commonkube.CreateDaemonSet(remote.kubeClient, newDaemonSet) + } if err != nil { return "", err } + remote.createdDaemonSet = daemonSet.DeepCopy() + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + if err := validateV2CreatedDaemonSet(newDaemonSet, daemonSet, consts.SubCmdInstall); err != nil { + return "", err + } + } err = kubeutils.MonitorDaemonSetContainer(remote.kubeClient, daemonSet, consts.ContainerNameInit, kubeutils.WaitForDaemonSetContainersExit, ptr.To(consts.ContainerConditionMaxTolerationLong)) if err != nil { @@ -191,6 +258,9 @@ func (remote *Installer) InstallByPackageManager() (string, error) { return "", err } + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + return collectV2OperationReports(context.Background(), remote.kubeClient, daemonSet, readV2PodReport(remote.kubeClient), consts.SubCmdInstall) + } podCollections, err := kubeutils.GetDaemonSetPodCollections(remote.kubeClient, daemonSet, consts.ContainerNameOutput, false, false, nil) if err != nil { return "", err @@ -448,6 +518,10 @@ func (remote *Installer) newDaemonSetForContainerOptimizedOS() *appsv1.DaemonSet // newDaemonSetForPackageManager prepares a DaemonSet for installing dependencies with the package manager. func (remote *Installer) newDaemonSetForPackageManager() *appsv1.DaemonSet { + command := []string{consts.CmdLonghornctlLocal, consts.SubCmdInstall, consts.SubCmdPreflight} + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + command = append(command, "--"+consts.CmdOptDataEngineMode, remote.DataEngineMode) + } outputFilePath := filepath.Join(consts.VolumeMountSharedDirectory, consts.FileNameOutputJSON) return &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ @@ -479,8 +553,11 @@ func (remote *Installer) newDaemonSetForPackageManager() *appsv1.DaemonSet { { Name: consts.ContainerNameInit, Image: utils.BuildImageName(remote.Image, remote.ImageRegistry), - Command: []string{consts.CmdLonghornctlLocal, consts.SubCmdInstall, consts.SubCmdPreflight}, + Command: command, Env: []corev1.EnvVar{ + {Name: consts.EnvDataEngineMode, Value: remote.DataEngineMode}, + {Name: consts.EnvPreflightCollectionOnly, Value: commonutils.ConvertTypeToString(remote.DataEngineMode == string(types.DataEngineModeV2Only))}, + {Name: consts.EnvPreflightAttemptID, Value: remote.appName}, { Name: consts.EnvLogLevel, Value: remote.LogLevel, diff --git a/pkg/remote/preflight/rbac_v2.go b/pkg/remote/preflight/rbac_v2.go new file mode 100644 index 000000000..7e97b04ed --- /dev/null +++ b/pkg/remote/preflight/rbac_v2.go @@ -0,0 +1,76 @@ +package preflight + +import ( + "context" + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeclient "k8s.io/client-go/kubernetes" +) + +type v2RBACReceipt struct { + Namespace, Name string + ServiceAccount, ClusterRole, ClusterRoleBinding k8stypes.UID +} + +// Never adopt an AlreadyExists resource, or delete an object whose creation +// did not return its UID. Partial receipts retain ownership of partial creates. +func createV2RBAC(ctx context.Context, client kubeclient.Interface, namespace, name string, rules []rbacv1.PolicyRule) (*v2RBACReceipt, error) { + receipt := &v2RBACReceipt{Namespace: namespace, Name: name} + account, err := client.CoreV1().ServiceAccounts(namespace).Create(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}}, metav1.CreateOptions{}) + if err != nil { + return receipt, err + } + if account.UID == "" || account.Name != name || account.Namespace != namespace { + return receipt, fmt.Errorf("V2 checker service account creation returned no identified receipt") + } + receipt.ServiceAccount = account.UID + role, err := client.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: name}, Rules: rules}, metav1.CreateOptions{}) + if err != nil { + return receipt, err + } + if role.UID == "" || role.Name != name { + return receipt, fmt.Errorf("V2 checker role creation returned no identified receipt") + } + receipt.ClusterRole = role.UID + binding, err := client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: name}, RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: name}, Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: name, Namespace: namespace}}}, metav1.CreateOptions{}) + if err != nil { + return receipt, err + } + if binding.UID == "" || binding.Name != name { + return receipt, fmt.Errorf("V2 checker binding creation returned no identified receipt") + } + receipt.ClusterRoleBinding = binding.UID + return receipt, nil +} + +func cleanupV2RBAC(ctx context.Context, client kubeclient.Interface, receipt *v2RBACReceipt) error { + if receipt == nil { + return nil + } + var result error + remove := func(uid k8stypes.UID, deleteObject func(metav1.DeleteOptions) error) { + if uid == "" { + return + } + err := deleteObject(metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + if err != nil && !apierrors.IsNotFound(err) { + result = errors.Join(result, err) + } + } + remove(receipt.ClusterRoleBinding, func(options metav1.DeleteOptions) error { + return client.RbacV1().ClusterRoleBindings().Delete(ctx, receipt.Name, options) + }) + remove(receipt.ClusterRole, func(options metav1.DeleteOptions) error { + return client.RbacV1().ClusterRoles().Delete(ctx, receipt.Name, options) + }) + remove(receipt.ServiceAccount, func(options metav1.DeleteOptions) error { + return client.CoreV1().ServiceAccounts(receipt.Namespace).Delete(ctx, receipt.Name, options) + }) + return result +} diff --git a/pkg/remote/preflight/rbac_v2_test.go b/pkg/remote/preflight/rbac_v2_test.go new file mode 100644 index 000000000..9b9c33d7f --- /dev/null +++ b/pkg/remote/preflight/rbac_v2_test.go @@ -0,0 +1,148 @@ +package preflight + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "k8s.io/client-go/rest" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeclient "k8s.io/client-go/kubernetes" +) + +func TestV2RBACNeverAdoptsOrDeletesUnownedObjects(t *testing.T) { + for _, test := range []struct { + name, failResource string + replacement bool + created int + }{ + {"complete", "", false, 3}, + {"account-exists", "serviceaccounts", false, 0}, + {"role-exists", "clusterroles", false, 1}, + {"binding-exists", "clusterrolebindings", false, 2}, + {"replacement-conflict", "", true, 3}, + } { + t.Run(test.name, func(t *testing.T) { + var mu sync.Mutex + deletes := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("Content-Type", "application/json") + parts := strings.Split(r.URL.Path, "/") + resource := parts[len(parts)-1] + if r.Method == http.MethodDelete { + resource = parts[len(parts)-2] + } + if r.Method == http.MethodPost && resource == test.failResource { + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(&metav1.Status{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, Status: "Failure", Reason: metav1.StatusReasonAlreadyExists, Code: 409}) + return + } + if r.Method == http.MethodDelete { + deletes++ + var options metav1.DeleteOptions + if err := json.NewDecoder(r.Body).Decode(&options); err != nil || options.Preconditions == nil || options.Preconditions.UID == nil || string(*options.Preconditions.UID) != "owned-"+resource { + t.Errorf("cleanup missing created UID: %+v %v", options, err) + } + if test.replacement && resource == "clusterroles" { + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(&metav1.Status{Status: "Failure", Reason: metav1.StatusReasonConflict, Code: 409}) + return + } + _ = json.NewEncoder(w).Encode(&metav1.Status{Status: "Success"}) + return + } + if r.Method != http.MethodPost { + t.Errorf("unexpected ownership API method: %s", r.Method) + http.Error(w, "unexpected", 500) + return + } + var object map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&object); err != nil { + t.Error(err) + return + } + metadata := object["metadata"].(map[string]interface{}) + metadata["uid"] = "owned-" + resource + _ = json.NewEncoder(w).Encode(object) + })) + t.Cleanup(server.Close) + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL, ContentConfig: rest.ContentConfig{ContentType: "application/json", AcceptContentTypes: "application/json"}}) + if err != nil { + t.Fatal(err) + } + receipt, err := createV2RBAC(t.Context(), client, "test", "owned", nil) + if (err != nil) != (test.failResource != "") { + t.Fatalf("strict creation adopted existing resources: %v", err) + } + err = cleanupV2RBAC(t.Context(), client, receipt) + if (err != nil) != test.replacement { + t.Fatalf("cleanup hid replacement conflict: %v", err) + } + if deletes != test.created { + t.Fatalf("deleted objects without successful creation receipts: got=%d want=%d", deletes, test.created) + } + }) + } +} + +func TestV2CleanupStillRemovesOwnedRBACAfterDaemonSetFailure(t *testing.T) { + for _, installer := range []bool{false, true} { + for _, missingUID := range []bool{false, true} { + t.Run(fmt.Sprintf("installer=%v/missingUID=%v", installer, missingUID), func(t *testing.T) { + var mu sync.Mutex + deletes := map[string]int{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("Content-Type", "application/json") + parts := strings.Split(r.URL.Path, "/") + resource := parts[len(parts)-2] + deletes[resource]++ + var options metav1.DeleteOptions + if r.Method != http.MethodDelete || json.NewDecoder(r.Body).Decode(&options) != nil || options.Preconditions == nil || options.Preconditions.UID == nil || *options.Preconditions.UID == "" { + t.Error("cleanup attempted an unidentified deletion") + } + if resource == "daemonsets" { + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(&metav1.Status{Status: "Failure", Reason: metav1.StatusReasonConflict, Code: 409}) + return + } + _ = json.NewEncoder(w).Encode(&metav1.Status{Status: "Success"}) + })) + t.Cleanup(server.Close) + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL, ContentConfig: rest.ContentConfig{ContentType: "application/json", AcceptContentTypes: "application/json"}}) + if err != nil { + t.Fatal(err) + } + uid := k8stypes.UID("owned-ds") + if missingUID { + uid = "" + } + ds := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "owned", Namespace: "test", UID: uid}} + receipt := &v2RBACReceipt{Name: "owned", Namespace: "test", ServiceAccount: "sa-uid", ClusterRole: "role-uid", ClusterRoleBinding: "binding-uid"} + if installer { + checker := Installer{InstallerCmdOptions: InstallerCmdOptions{DataEngineMode: "v2-only"}, kubeClient: client, createdDaemonSet: ds, createdRBAC: receipt} + err = checker.Cleanup() + } else { + checker := Checker{CheckerCmdOptions: CheckerCmdOptions{DataEngineMode: "v2-only"}, kubeClient: client, createdDaemonSet: ds, createdRBAC: receipt} + err = checker.Cleanup() + } + if err == nil || deletes["serviceaccounts"] != 1 || deletes["clusterroles"] != 1 || deletes["clusterrolebindings"] != 1 { + t.Fatalf("DaemonSet failure leaked owned RBAC: deleted=%v err=%v", deletes, err) + } + if missingUID && deletes["daemonsets"] != 0 { + t.Fatal("deleted DaemonSet without UID") + } + }) + } + } +} diff --git a/pkg/types/data_engine_mode.go b/pkg/types/data_engine_mode.go new file mode 100644 index 000000000..f0da0d1b2 --- /dev/null +++ b/pkg/types/data_engine_mode.go @@ -0,0 +1,27 @@ +package types + +import "fmt" + +type DataEngineMode string + +const ( + DataEngineModeV1Capable DataEngineMode = "v1-capable" + DataEngineModeV2Only DataEngineMode = "v2-only" +) + +func ParseDataEngineMode(value string) (DataEngineMode, error) { + switch DataEngineMode(value) { + case "", DataEngineModeV1Capable: + return DataEngineModeV1Capable, nil + case DataEngineModeV2Only: + return DataEngineModeV2Only, nil + default: + return "", fmt.Errorf("data-engine-mode must be v1-capable or v2-only") + } +} + +func (mode DataEngineMode) RequiresISCSI() bool { return mode != DataEngineModeV2Only } + +func (mode DataEngineMode) NeedsSPDK(explicit bool) bool { + return explicit || mode == DataEngineModeV2Only +} diff --git a/pkg/types/data_engine_mode_test.go b/pkg/types/data_engine_mode_test.go new file mode 100644 index 000000000..7b28eece1 --- /dev/null +++ b/pkg/types/data_engine_mode_test.go @@ -0,0 +1,31 @@ +package types + +import "testing" + +func TestDataEngineModePreservesDefaultAndRequiresExplicitV2(t *testing.T) { + for _, test := range []struct { + value string + want DataEngineMode + }{ + {"", DataEngineModeV1Capable}, {"v1-capable", DataEngineModeV1Capable}, {"v2-only", DataEngineModeV2Only}, + {"v2", ""}, {"V2-Only", ""}, {" v2-only", ""}, {"unknown", ""}, + } { + t.Run(test.value, func(t *testing.T) { + mode, err := ParseDataEngineMode(test.value) + if test.want == "" { + if err == nil { + t.Fatal("invalid capability silently selected a default") + } + return + } + if err != nil || mode != test.want { + t.Fatalf("mode=%q err=%v want=%q", mode, err, test.want) + } + for _, explicit := range []bool{false, true} { + if mode.RequiresISCSI() != (mode == DataEngineModeV1Capable) || mode.NeedsSPDK(explicit) != (explicit || mode == DataEngineModeV2Only) { + t.Fatal("iSCSI or additive SPDK capability changed") + } + } + }) + } +} diff --git a/pkg/types/nodes.go b/pkg/types/nodes.go index 854caab14..719b21ad3 100644 --- a/pkg/types/nodes.go +++ b/pkg/types/nodes.go @@ -2,5 +2,6 @@ package types // NodeCollection represents a collection of nodes. type NodeCollection struct { - Log *LogCollection `json:"log,omitempty" yaml:"log,omitempty"` + Log *LogCollection `json:"log,omitempty" yaml:"log,omitempty"` + DataEngineMode string `json:"dataEngineMode,omitempty" yaml:"dataEngineMode,omitempty"` } diff --git a/pkg/utils/kubernetes/node.go b/pkg/utils/kubernetes/node.go index 1611cdc8d..a8087c609 100644 --- a/pkg/utils/kubernetes/node.go +++ b/pkg/utils/kubernetes/node.go @@ -13,7 +13,7 @@ import ( ) // GetHugePagesCapacity returns hugepages-2Mi capacity of current node -func GetHugePagesCapacity(kubeClient *kubeclient.Clientset) (*resource.Quantity, error) { +func GetHugePagesCapacity(kubeClient kubeclient.Interface) (*resource.Quantity, error) { currentNodeID := os.Getenv(consts.EnvCurrentNodeID) node, err := kubeClient.CoreV1().Nodes().Get(context.TODO(), currentNodeID, metav1.GetOptions{}) if err != nil {