From 4fdf9989cc3b4e034780e653a594c8ee26fbc3d0 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 00:03:17 +0100 Subject: [PATCH 1/7] feat(types): define explicit preflight engine capabilities Issue longhorn/longhorn#14438 Signed-off-by: Nikolai Emil Damm Co-authored-by: Codex (GPT-6.1 Sol) --- pkg/types/data_engine_mode.go | 27 ++++++++++++++++++++++++++ pkg/types/data_engine_mode_test.go | 31 ++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 pkg/types/data_engine_mode.go create mode 100644 pkg/types/data_engine_mode_test.go diff --git a/pkg/types/data_engine_mode.go b/pkg/types/data_engine_mode.go new file mode 100644 index 000000000..f0da0d1b2 --- /dev/null +++ b/pkg/types/data_engine_mode.go @@ -0,0 +1,27 @@ +package types + +import "fmt" + +type DataEngineMode string + +const ( + DataEngineModeV1Capable DataEngineMode = "v1-capable" + DataEngineModeV2Only DataEngineMode = "v2-only" +) + +func ParseDataEngineMode(value string) (DataEngineMode, error) { + switch DataEngineMode(value) { + case "", DataEngineModeV1Capable: + return DataEngineModeV1Capable, nil + case DataEngineModeV2Only: + return DataEngineModeV2Only, nil + default: + return "", fmt.Errorf("data-engine-mode must be v1-capable or v2-only") + } +} + +func (mode DataEngineMode) RequiresISCSI() bool { return mode != DataEngineModeV2Only } + +func (mode DataEngineMode) NeedsSPDK(explicit bool) bool { + return explicit || mode == DataEngineModeV2Only +} diff --git a/pkg/types/data_engine_mode_test.go b/pkg/types/data_engine_mode_test.go new file mode 100644 index 000000000..7b28eece1 --- /dev/null +++ b/pkg/types/data_engine_mode_test.go @@ -0,0 +1,31 @@ +package types + +import "testing" + +func TestDataEngineModePreservesDefaultAndRequiresExplicitV2(t *testing.T) { + for _, test := range []struct { + value string + want DataEngineMode + }{ + {"", DataEngineModeV1Capable}, {"v1-capable", DataEngineModeV1Capable}, {"v2-only", DataEngineModeV2Only}, + {"v2", ""}, {"V2-Only", ""}, {" v2-only", ""}, {"unknown", ""}, + } { + t.Run(test.value, func(t *testing.T) { + mode, err := ParseDataEngineMode(test.value) + if test.want == "" { + if err == nil { + t.Fatal("invalid capability silently selected a default") + } + return + } + if err != nil || mode != test.want { + t.Fatalf("mode=%q err=%v want=%q", mode, err, test.want) + } + for _, explicit := range []bool{false, true} { + if mode.RequiresISCSI() != (mode == DataEngineModeV1Capable) || mode.NeedsSPDK(explicit) != (explicit || mode == DataEngineModeV2Only) { + t.Fatal("iSCSI or additive SPDK capability changed") + } + } + }) + } +} From 87eacf7317500b123a59d23abdf6e01e961df500 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 01:13:50 +0100 Subject: [PATCH 2/7] feat(preflight): qualify V2 hosts without iSCSI dependencies Issue longhorn/longhorn#14438 Signed-off-by: Nikolai Emil Damm Co-authored-by: Codex (GPT-6.1 Sol) --- README.md | 19 + cmd/local/subcmd/check.go | 12 + cmd/local/subcmd/check_v2.go | 15 + cmd/local/subcmd/check_v2_test.go | 28 ++ cmd/local/subcmd/data_engine_mode.go | 35 ++ cmd/local/subcmd/install.go | 11 + cmd/remote/subcmd/check.go | 21 +- cmd/remote/subcmd/install.go | 22 +- deploy/check.yaml | 2 + deploy/install.yaml | 2 + docs/longhornctl.md | 4 +- docs/longhornctl_check.md | 4 +- docs/longhornctl_check_preflight.md | 5 +- docs/longhornctl_checksum.md | 4 +- docs/longhornctl_checksum_volume.md | 4 +- docs/longhornctl_diagnose.md | 2 +- docs/longhornctl_doc.md | 4 +- docs/longhornctl_export.md | 4 +- docs/longhornctl_export_replica.md | 6 +- docs/longhornctl_export_replica_stop.md | 4 +- docs/longhornctl_get.md | 4 +- docs/longhornctl_get_replica.md | 4 +- docs/longhornctl_global-options.md | 4 +- docs/longhornctl_install.md | 4 +- docs/longhornctl_install_preflight.md | 5 +- docs/longhornctl_install_preflight_stop.md | 4 +- docs/longhornctl_trim.md | 4 +- docs/longhornctl_trim_volume.md | 4 +- docs/longhornctl_version.md | 4 +- pkg/consts/cmd.go | 1 + pkg/consts/environment.go | 10 +- pkg/local/preflight/checker.go | 96 +++- .../checker_data_engine_mode_test.go | 177 +++++++ pkg/local/preflight/checker_hugepages_v2.go | 25 + pkg/local/preflight/checker_nfs_facts.go | 34 ++ pkg/local/preflight/data_engine_host.go | 12 + pkg/local/preflight/installer.go | 68 ++- .../installer_data_engine_mode_test.go | 36 ++ pkg/local/preflight/installer_spdk_v2.go | 104 ++++ pkg/local/preflight/installer_spdk_v2_test.go | 69 +++ pkg/remote/preflight/checker.go | 77 ++- pkg/remote/preflight/checker_collection_v2.go | 475 ++++++++++++++++++ .../preflight/checker_collection_v2_test.go | 270 ++++++++++ pkg/remote/preflight/data_engine_mode.go | 149 ++++++ pkg/remote/preflight/data_engine_mode_test.go | 145 ++++++ pkg/remote/preflight/installer.go | 81 ++- pkg/remote/preflight/rbac_v2.go | 76 +++ pkg/remote/preflight/rbac_v2_test.go | 92 ++++ pkg/types/nodes.go | 3 +- pkg/utils/kubernetes/node.go | 2 +- 50 files changed, 2163 insertions(+), 84 deletions(-) create mode 100644 cmd/local/subcmd/check_v2.go create mode 100644 cmd/local/subcmd/check_v2_test.go create mode 100644 cmd/local/subcmd/data_engine_mode.go create mode 100644 pkg/local/preflight/checker_data_engine_mode_test.go create mode 100644 pkg/local/preflight/checker_hugepages_v2.go create mode 100644 pkg/local/preflight/checker_nfs_facts.go create mode 100644 pkg/local/preflight/data_engine_host.go create mode 100644 pkg/local/preflight/installer_data_engine_mode_test.go create mode 100644 pkg/local/preflight/installer_spdk_v2.go create mode 100644 pkg/local/preflight/installer_spdk_v2_test.go create mode 100644 pkg/remote/preflight/checker_collection_v2.go create mode 100644 pkg/remote/preflight/checker_collection_v2_test.go create mode 100644 pkg/remote/preflight/data_engine_mode.go create mode 100644 pkg/remote/preflight/data_engine_mode_test.go create mode 100644 pkg/remote/preflight/rbac_v2.go create mode 100644 pkg/remote/preflight/rbac_v2_test.go diff --git a/README.md b/README.md index 35522cca9..31c1659b9 100644 --- a/README.md +++ b/README.md @@ -52,3 +52,22 @@ You can obtain `longhornctl` either through downloading a prebuilt binary or by > - `longhornctl`: A command-line interface for remote Longhorn operations, designed to be run outside the Kubernetes cluster. It executes `longhornctl-local` for operations within the cluster. > - `longhornctl-local`: A command-line interface to be used within a DaemonSet pod inside the Kubernetes cluster, handling in-cluster and host operations. +### V2-only prerequisites + +The default `--data-engine-mode=v1-capable` retains iSCSI requirements. An explicitly selected `v2-only` mode omits iSCSI packages, modules and service operations, and requires SPDK checks and setup even without `--enable-spdk`. + +Use matching CLI binaries and a CLI image containing this capability. The remote command passes an explicit flag to the local executable so an older image fails instead of silently running V1 checks. + +```bash +longhornctl install preflight --data-engine-mode=v2-only --hugepage-size=2048 --allow-pci=none +longhornctl check preflight --data-engine-mode=v2-only --hugepage-size=2048 +``` + +Installation success is prerequisite setup, followed by a separate host qualification check. A required reboot remains unfinished installation; reboot and rerun installation before checking. Restart kubelet when necessary for its Node capacity to reflect the hugepage allocation. V2 check failures return a nonzero exit status after printing the complete report. + +V2 installation supports the known package-manager distributions. COS, Talos, unknown operating systems, unqualified NUMA layouts and host-provided SPDK startup hooks are rejected; configure immutable hosts through their own declarative prerequisites. The supported setup profile uses 2MiB pages on one existing NUMA node (node 0 by default). `HUGENODE` accepts one nonnegative integer. The requested memory, page size, PCI selection and driver must agree with any supplied `--spdk-options`; unknown, duplicate or conflicting options are rejected. Setup uses a bounded, cleared environment and verifies the actual allocation afterward. + +An empty PCI selection is fenced as `none`. Explicit device selection requires unique full PCI addresses; do not select devices still in use. Installer `--driver-override` accepts `vfio-pci` or `uio_pci_generic`; checker `--userspace-driver` uses the kernel module names `vfio_pci` or `uio_pci_generic`. + +V2 remote runs use separate resource names and collect complete reports from the identified DaemonSet and completed container instances. Cleanup deletes only objects created by that invocation, with UID preconditions. This mode neither converts existing volumes nor enables the manager's separate V2-only installation capability. Retire V1 data before removing its host prerequisites. + diff --git a/cmd/local/subcmd/check.go b/cmd/local/subcmd/check.go index 689398744..757f98914 100644 --- a/cmd/local/subcmd/check.go +++ b/cmd/local/subcmd/check.go @@ -35,7 +35,11 @@ func newCmdCheckPreflight(globalOpts *types.GlobalCmdOptions) *cobra.Command { Long: `This command verifies your Kubernetes cluster environment to ensure it meets Longhorn's requirements. It performs a series of checks that can help identify potential issues that may prevent Longhorn from functioning correctly.`, PreRun: func(cmd *cobra.Command, args []string) { + if err := validateLocalDataEngineEnvironment(cmd, localChecker.DataEngineMode); err != nil { + utils.CheckErr(err) + } localChecker.LogLevel = globalOpts.LogLevel + localChecker.CollectionOnly = os.Getenv(consts.EnvPreflightCollectionOnly) == "true" if err := localChecker.Init(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to initialize preflight checker")) @@ -43,6 +47,10 @@ func newCmdCheckPreflight(globalOpts *types.GlobalCmdOptions) *cobra.Command { }, Run: func(cmd *cobra.Command, args []string) { + if localChecker.DataEngineMode == string(types.DataEngineModeV2Only) { + utils.CheckErr(runAndOutputV2Preflight(&localChecker)) + return + } if err := localChecker.Run(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to run preflight checker")) } @@ -51,6 +59,9 @@ func newCmdCheckPreflight(globalOpts *types.GlobalCmdOptions) *cobra.Command { }, PostRun: func(cmd *cobra.Command, args []string) { + if localChecker.DataEngineMode == string(types.DataEngineModeV2Only) { + return + } if err := localChecker.Output(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to output preflight checker collection")) } @@ -61,6 +72,7 @@ func newCmdCheckPreflight(globalOpts *types.GlobalCmdOptions) *cobra.Command { utils.SetGlobalOptionsLocal(cmd, globalOpts) + bindLocalDataEngineMode(cmd, &localChecker.DataEngineMode) cmd.Flags().StringVarP(&localChecker.OutputFilePath, consts.CmdOptOutputFile, "o", os.Getenv(consts.EnvOutputFilePath), "Output the result to a file, default to stdout.") cmd.Flags().BoolVar(&localChecker.EnableSpdk, consts.CmdOptEnableSpdk, utils.ConvertStringToTypeOrDefault(os.Getenv(consts.EnvEnableSpdk), false), "Enable checking of SPDK required packages, modules, and setup.") cmd.Flags().IntVar(&localChecker.HugePageSize, consts.CmdOptHugePageSize, utils.ConvertStringToTypeOrDefault(os.Getenv(consts.EnvHugePageSize), 2048), "Specify the huge page size in MiB for SPDK.") diff --git a/cmd/local/subcmd/check_v2.go b/cmd/local/subcmd/check_v2.go new file mode 100644 index 000000000..1fa832f65 --- /dev/null +++ b/cmd/local/subcmd/check_v2.go @@ -0,0 +1,15 @@ +package subcmd + +import "errors" + +type preflightReportRunner interface { + Run() error + Output() error +} + +// Write the complete report before returning the qualification verdict. In the +// remote transport phase the checker itself suppresses only that final verdict. +func runAndOutputV2Preflight(checker preflightReportRunner) error { + runErr := checker.Run() + return errors.Join(runErr, checker.Output()) +} diff --git a/cmd/local/subcmd/check_v2_test.go b/cmd/local/subcmd/check_v2_test.go new file mode 100644 index 000000000..9895ae125 --- /dev/null +++ b/cmd/local/subcmd/check_v2_test.go @@ -0,0 +1,28 @@ +package subcmd + +import ( + "errors" + "testing" +) + +type failedPreflightReport struct { + runErr, outputErr error + wrote bool +} + +func (report *failedPreflightReport) Run() error { return report.runErr } +func (report *failedPreflightReport) Output() error { + report.wrote = true + return report.outputErr +} + +func TestV2PreflightFailureStillOutputsReport(t *testing.T) { + qualificationErr, outputErr := errors.New("host qualification failed"), errors.New("report write failed") + for _, writeErr := range []error{nil, outputErr} { + report := &failedPreflightReport{runErr: qualificationErr, outputErr: writeErr} + err := runAndOutputV2Preflight(report) + if !report.wrote || !errors.Is(err, qualificationErr) || (writeErr != nil && !errors.Is(err, writeErr)) { + t.Fatalf("missing report or failure: wrote=%v error=%v", report.wrote, err) + } + } +} diff --git a/cmd/local/subcmd/data_engine_mode.go b/cmd/local/subcmd/data_engine_mode.go new file mode 100644 index 000000000..67d74ac68 --- /dev/null +++ b/cmd/local/subcmd/data_engine_mode.go @@ -0,0 +1,35 @@ +package subcmd + +import ( + "fmt" + "os" + "strconv" + + "github.com/spf13/cobra" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" +) + +func bindLocalDataEngineMode(cmd *cobra.Command, target *string) { + value := os.Getenv(consts.EnvDataEngineMode) + if value == "" { + value = string(types.DataEngineModeV1Capable) + } + cmd.Flags().StringVar(target, consts.CmdOptDataEngineMode, value, "Installation capability: v1-capable or v2-only. V2-only omits iSCSI and requires SPDK qualification.") +} + +func validateLocalDataEngineEnvironment(cmd *cobra.Command, value string) error { + mode, err := types.ParseDataEngineMode(value) + if err != nil { + return err + } + if mode == types.DataEngineModeV2Only && !cmd.Flags().Changed(consts.CmdOptHugePageSize) { + if raw, present := os.LookupEnv(consts.EnvHugePageSize); present { + if _, err := strconv.Atoi(raw); err != nil { + return fmt.Errorf("v2-only hugepage environment must contain an explicit integer") + } + } + } + return nil +} diff --git a/cmd/local/subcmd/install.go b/cmd/local/subcmd/install.go index 5fd4fe7c8..8ad26846f 100644 --- a/cmd/local/subcmd/install.go +++ b/cmd/local/subcmd/install.go @@ -38,6 +38,9 @@ func newCmdInstallPreflight(globalOpts *types.GlobalCmdOptions) *cobra.Command { These dependencies ensure your Kubernetes cluster meets the requirements for successful Longhorn operation.`, PreRun: func(cmd *cobra.Command, args []string) { + if err := validateLocalDataEngineEnvironment(cmd, localInstaller.DataEngineMode); err != nil { + utils.CheckErr(err) + } localInstaller.LogLevel = globalOpts.LogLevel if err := localInstaller.Init(); err != nil { @@ -46,6 +49,10 @@ These dependencies ensure your Kubernetes cluster meets the requirements for suc }, Run: func(cmd *cobra.Command, args []string) { + if localInstaller.DataEngineMode == string(types.DataEngineModeV2Only) { + utils.CheckErr(runAndOutputV2Preflight(&localInstaller)) + return + } if err := localInstaller.Run(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to run preflight installer")) } @@ -54,6 +61,9 @@ These dependencies ensure your Kubernetes cluster meets the requirements for suc }, PostRun: func(cmd *cobra.Command, args []string) { + if localInstaller.DataEngineMode == string(types.DataEngineModeV2Only) { + return + } if err := localInstaller.Output(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to output preflight checker collection")) } @@ -64,6 +74,7 @@ These dependencies ensure your Kubernetes cluster meets the requirements for suc utils.SetGlobalOptionsLocal(cmd, globalOpts) + bindLocalDataEngineMode(cmd, &localInstaller.DataEngineMode) cmd.Flags().StringVarP(&localInstaller.OutputFilePath, consts.CmdOptOutputFile, "o", os.Getenv(consts.EnvOutputFilePath), "Output the result to a file, default to stdout.") cmd.Flags().BoolVar(&localInstaller.UpdatePackages, consts.CmdOptUpdatePackages, utils.ConvertStringToTypeOrDefault(os.Getenv(consts.EnvUpdatePackageList), true), "Update packages before installing required dependencies.") cmd.Flags().BoolVar(&localInstaller.EnableSpdk, consts.CmdOptEnableSpdk, utils.ConvertStringToTypeOrDefault(os.Getenv(consts.EnvEnableSpdk), false), "Enable installation of SPDK required packages, modules, and setup.") diff --git a/cmd/remote/subcmd/check.go b/cmd/remote/subcmd/check.go index 51235251a..92e6facfa 100644 --- a/cmd/remote/subcmd/check.go +++ b/cmd/remote/subcmd/check.go @@ -1,6 +1,7 @@ package subcmd import ( + stderrors "errors" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -58,6 +59,9 @@ INFO[2024-07-16T17:17:42+08:00] Cleaning up preflight checker INFO[2024-07-16T17:17:42+08:00] Completed preflight checker`, PreRun: func(cmd *cobra.Command, args []string) { + if err := preflightChecker.ValidateDataEngineMode(); err != nil { + utils.CheckErr(err) + } preflightChecker.Image = globalOpts.Image preflightChecker.ImageRegistry = globalOpts.ImageRegistry preflightChecker.ImagePullSecret = globalOpts.ImagePullSecret @@ -72,14 +76,23 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker`, } logrus.Info("Cleaning up preflight checker") - if err := preflightChecker.Cleanup(); err != nil { - utils.CheckErr(errors.Wrapf(err, "Failed to cleanup preflight checker")) + if preflightChecker.DataEngineMode != string(types.DataEngineModeV2Only) { + if err := preflightChecker.Cleanup(); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to cleanup preflight checker")) + } } }, Run: func(cmd *cobra.Command, args []string) { logrus.Info("Running preflight checker") output, err := preflightChecker.Run() + if preflightChecker.DataEngineMode == string(types.DataEngineModeV2Only) { + if output != "" { + logrus.Infof("Retrieved preflight checker result:\n%v", output) + } + utils.CheckErr(stderrors.Join(err, preflightChecker.Cleanup())) + return + } if err != nil { utils.CheckErr(errors.Wrap(err, "Failed to run preflight checker")) } @@ -88,6 +101,9 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker`, }, PostRun: func(cmd *cobra.Command, args []string) { + if preflightChecker.DataEngineMode == string(types.DataEngineModeV2Only) { + return + } logrus.Info("Cleaning up preflight checker") if err := preflightChecker.Cleanup(); err != nil { utils.CheckErr(errors.Wrapf(err, "Failed to cleanup preflight checker")) @@ -99,6 +115,7 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker`, utils.SetGlobalOptionsRemote(cmd, globalOpts) + cmd.Flags().StringVar(&preflightChecker.DataEngineMode, consts.CmdOptDataEngineMode, string(types.DataEngineModeV1Capable), "Installation capability: v1-capable or v2-only. V2-only omits iSCSI and requires SPDK qualification.") cmd.Flags().BoolVar(&preflightChecker.EnableSpdk, consts.CmdOptEnableSpdk, false, "Enable checking of SPDK required packages, modules, and setup.") cmd.Flags().IntVar(&preflightChecker.HugePageSize, consts.CmdOptHugePageSize, 2048, "Specify the huge page size in MiB for SPDK.") cmd.Flags().StringVar(&preflightChecker.UserspaceDriver, consts.CmdOptUserspaceDriver, "", "Userspace I/O driver for SPDK.") diff --git a/cmd/remote/subcmd/install.go b/cmd/remote/subcmd/install.go index 7e84f4b30..29eaa9b1a 100644 --- a/cmd/remote/subcmd/install.go +++ b/cmd/remote/subcmd/install.go @@ -1,6 +1,7 @@ package subcmd import ( + stderrors "errors" "fmt" "github.com/pkg/errors" @@ -64,6 +65,9 @@ If a reboot is required, the following message will be displayed: - Need to reboot the system and execute longhornctl install preflight again`, PreRun: func(cmd *cobra.Command, args []string) { + if err := preflightInstaller.ValidateDataEngineMode(); err != nil { + utils.CheckErr(err) + } preflightInstaller.Image = globalOpts.Image preflightInstaller.ImageRegistry = globalOpts.ImageRegistry preflightInstaller.ImagePullSecret = globalOpts.ImagePullSecret @@ -79,14 +83,24 @@ If a reboot is required, the following message will be displayed: } logrus.Info("Cleaning up preflight installer") - if err := preflightInstaller.Cleanup(); err != nil { - utils.CheckErr(errors.Wrapf(err, "Failed to cleanup preflight installer")) + if preflightInstaller.DataEngineMode != string(types.DataEngineModeV2Only) { + if err := preflightInstaller.Cleanup(); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to cleanup preflight installer")) + } } }, Run: func(cmd *cobra.Command, args []string) { logrus.Info("Running preflight installer") output, err := preflightInstaller.Run() + if preflightInstaller.DataEngineMode == string(types.DataEngineModeV2Only) { + if output != "" { + logrus.Infof("Retrieved preflight installer result:\n%v", output) + } + utils.CheckErr(stderrors.Join(err, preflightInstaller.Cleanup())) + logrus.Info("Completed V2 prerequisite setup; run check preflight --data-engine-mode=v2-only to qualify every host") + return + } if err != nil { utils.CheckErr(errors.Wrap(err, "Failed to run preflight installer")) } @@ -95,6 +109,9 @@ If a reboot is required, the following message will be displayed: }, PostRun: func(cmd *cobra.Command, args []string) { + if preflightInstaller.DataEngineMode == string(types.DataEngineModeV2Only) { + return + } logrus.Info("Cleaning up preflight installer") if preflightInstaller.OperatingSystem != string(consts.OperatingSystemContainerOptimizedOS) { if err := preflightInstaller.Cleanup(); err != nil { @@ -110,6 +127,7 @@ If a reboot is required, the following message will be displayed: utils.SetGlobalOptionsRemote(cmd, globalOpts) + cmd.Flags().StringVar(&preflightInstaller.DataEngineMode, consts.CmdOptDataEngineMode, string(types.DataEngineModeV1Capable), "Installation capability: v1-capable or v2-only. V2-only omits iSCSI and requires SPDK setup.") cmd.Flags().StringVar(&preflightInstaller.OperatingSystem, consts.CmdOptOperatingSystem, "", "Specify the operating system (\"\", cos). Leave this empty to use the package manager for installation.") cmd.Flags().BoolVar(&preflightInstaller.UpdatePackages, consts.CmdOptUpdatePackages, true, "Update packages before installing required dependencies.") cmd.Flags().BoolVar(&preflightInstaller.EnableSpdk, consts.CmdOptEnableSpdk, false, "Enable installation of SPDK required packages, modules, and setup.") diff --git a/deploy/check.yaml b/deploy/check.yaml index e7273d20c..2ad98dfcd 100644 --- a/deploy/check.yaml +++ b/deploy/check.yaml @@ -25,6 +25,8 @@ spec: securityContext: privileged: true env: + - name: DATA_ENGINE_MODE + value: "v1-capable" - name: ENABLE_SPDK value: "true" - name: HUGEMEM diff --git a/deploy/install.yaml b/deploy/install.yaml index 981541a1e..8873426a8 100644 --- a/deploy/install.yaml +++ b/deploy/install.yaml @@ -25,6 +25,8 @@ spec: securityContext: privileged: true env: + - name: DATA_ENGINE_MODE + value: "v1-capable" - name: UPDATE_PACKAGE_LIST value: "true" - name: ENABLE_SPDK diff --git a/docs/longhornctl.md b/docs/longhornctl.md index 64ff2a976..ef5689778 100644 --- a/docs/longhornctl.md +++ b/docs/longhornctl.md @@ -10,7 +10,7 @@ A CLI tool for troubleshooting and managing Longhorn operations. ``` -h, --help help for longhornctl - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -33,4 +33,4 @@ A CLI tool for troubleshooting and managing Longhorn operations. * [longhornctl trim](longhornctl_trim.md) - Longhorn trimming operations * [longhornctl version](longhornctl_version.md) - Print longhornctl version -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_check.md b/docs/longhornctl_check.md index d481efb59..4e0712379 100644 --- a/docs/longhornctl_check.md +++ b/docs/longhornctl_check.md @@ -6,7 +6,7 @@ Longhorn checking operations ``` -h, --help help for check - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Longhorn checking operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl check preflight](longhornctl_check_preflight.md) - Run a preflight check for Longhorn -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_check_preflight.md b/docs/longhornctl_check_preflight.md index 7bd2d2d66..1b29dce95 100644 --- a/docs/longhornctl_check_preflight.md +++ b/docs/longhornctl_check_preflight.md @@ -43,10 +43,11 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker ### Options ``` + --data-engine-mode string Installation capability: v1-capable or v2-only. V2-only omits iSCSI and requires SPDK qualification. (default "v1-capable") --enable-spdk Enable checking of SPDK required packages, modules, and setup. -h, --help help for preflight --huge-page-size int Specify the huge page size in MiB for SPDK. (default 2048) - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -66,4 +67,4 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker * [longhornctl check](longhornctl_check.md) - Longhorn checking operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_checksum.md b/docs/longhornctl_checksum.md index 109f8b37b..1d40b5711 100644 --- a/docs/longhornctl_checksum.md +++ b/docs/longhornctl_checksum.md @@ -6,7 +6,7 @@ Snapshot checksum operations ``` -h, --help help for checksum - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Snapshot checksum operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl checksum volume](longhornctl_checksum_volume.md) - Trigger on-demand snapshot checksum calculation for a volume -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_checksum_volume.md b/docs/longhornctl_checksum_volume.md index 531cc3524..29e746a91 100644 --- a/docs/longhornctl_checksum_volume.md +++ b/docs/longhornctl_checksum_volume.md @@ -27,7 +27,7 @@ $ longhornctl checksum volume --name=v1 --namespace=longhorn-system ``` --all Apply to all volumes -h, --help help for volume - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -48,4 +48,4 @@ $ longhornctl checksum volume --name=v1 --namespace=longhorn-system * [longhornctl checksum](longhornctl_checksum.md) - Snapshot checksum operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_diagnose.md b/docs/longhornctl_diagnose.md index ddfc240b5..acde8bc36 100644 --- a/docs/longhornctl_diagnose.md +++ b/docs/longhornctl_diagnose.md @@ -49,4 +49,4 @@ $ longhornctl diagnose -o json 2>/dev/null | jq '.[] | select(.status == "fail") * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 29-Sep-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_doc.md b/docs/longhornctl_doc.md index 5ae9d6ba6..6f0eb48ad 100644 --- a/docs/longhornctl_doc.md +++ b/docs/longhornctl_doc.md @@ -19,7 +19,7 @@ longhornctl doc [output directory] [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -33,4 +33,4 @@ longhornctl doc [output directory] [flags] * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_export.md b/docs/longhornctl_export.md index 43fd4bee4..bedb8cfe9 100644 --- a/docs/longhornctl_export.md +++ b/docs/longhornctl_export.md @@ -6,7 +6,7 @@ Export Longhorn resources ``` -h, --help help for export - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Export Longhorn resources * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl export replica](longhornctl_export_replica.md) - Export data from a Longhorn replica -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_export_replica.md b/docs/longhornctl_export_replica.md index 8f155ee73..7aa0a1f17 100644 --- a/docs/longhornctl_export_replica.md +++ b/docs/longhornctl_export_replica.md @@ -48,9 +48,9 @@ lost+found ``` --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") - --engine-image string Engine image to use to create volume from the replica. (default "longhornio/longhorn-engine:v1.13.0-dev") + --engine-image string Engine image to use to create volume from the replica. (default "longhornio/longhorn-engine:") -h, --help help for replica - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -72,4 +72,4 @@ lost+found * [longhornctl export](longhornctl_export.md) - Export Longhorn resources * [longhornctl export replica stop](longhornctl_export_replica_stop.md) - Stop the replica export process -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_export_replica_stop.md b/docs/longhornctl_export_replica_stop.md index 0a4688f3f..e6d214963 100644 --- a/docs/longhornctl_export_replica_stop.md +++ b/docs/longhornctl_export_replica_stop.md @@ -22,7 +22,7 @@ INFO[2024-07-16T17:29:14+08:00] Successfully stopped exporting replica ``` -h, --help help for stop - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -41,4 +41,4 @@ INFO[2024-07-16T17:29:14+08:00] Successfully stopped exporting replica * [longhornctl export replica](longhornctl_export_replica.md) - Export data from a Longhorn replica -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_get.md b/docs/longhornctl_get.md index 4aaf8dce3..df413b4b9 100644 --- a/docs/longhornctl_get.md +++ b/docs/longhornctl_get.md @@ -6,7 +6,7 @@ Longhorn information gathering operations ``` -h, --help help for get - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Longhorn information gathering operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl get replica](longhornctl_get_replica.md) - Retrieve Longhorn replica information -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_get_replica.md b/docs/longhornctl_get_replica.md index ac36c15bf..15834a59b 100644 --- a/docs/longhornctl_get_replica.md +++ b/docs/longhornctl_get_replica.md @@ -49,7 +49,7 @@ INFO[2024-07-16T17:23:51+08:00] Completed replica getter ``` --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") -h, --help help for replica - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -70,4 +70,4 @@ INFO[2024-07-16T17:23:51+08:00] Completed replica getter * [longhornctl get](longhornctl_get.md) - Longhorn information gathering operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_global-options.md b/docs/longhornctl_global-options.md index 49c891b90..04ca1f7aa 100644 --- a/docs/longhornctl_global-options.md +++ b/docs/longhornctl_global-options.md @@ -19,7 +19,7 @@ longhornctl global-options [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -33,4 +33,4 @@ longhornctl global-options [flags] * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_install.md b/docs/longhornctl_install.md index e57924769..d840acfc0 100644 --- a/docs/longhornctl_install.md +++ b/docs/longhornctl_install.md @@ -6,7 +6,7 @@ Longhorn installation operations ``` -h, --help help for install - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Longhorn installation operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl install preflight](longhornctl_install_preflight.md) - Install Longhorn preflight -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_install_preflight.md b/docs/longhornctl_install_preflight.md index 34a488417..f5fa254b6 100644 --- a/docs/longhornctl_install_preflight.md +++ b/docs/longhornctl_install_preflight.md @@ -47,11 +47,12 @@ If a reboot is required, the following message will be displayed: ``` --allow-pci string Specify a comma-separated (,) list of allowed PCI devices. By default, all PCI devices are blocked by a non-valid address. (default "none") + --data-engine-mode string Installation capability: v1-capable or v2-only. V2-only omits iSCSI and requires SPDK setup. (default "v1-capable") --driver-override string Userspace driver for device bindings. Override default driver for PCI devices. --enable-spdk Enable installation of SPDK required packages, modules, and setup. -h, --help help for preflight --huge-page-size int Specify the huge page size in MiB for SPDK. (default 2048) - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -76,4 +77,4 @@ If a reboot is required, the following message will be displayed: * [longhornctl install](longhornctl_install.md) - Longhorn installation operations * [longhornctl install preflight stop](longhornctl_install_preflight_stop.md) - Stop Longhorn preflight installer -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_install_preflight_stop.md b/docs/longhornctl_install_preflight_stop.md index 7bb5433d6..ed1bad377 100644 --- a/docs/longhornctl_install_preflight_stop.md +++ b/docs/longhornctl_install_preflight_stop.md @@ -22,7 +22,7 @@ INFO[2024-07-16T17:21:32+08:00] Successfully stopped preflight installer ``` -h, --help help for stop - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -42,4 +42,4 @@ INFO[2024-07-16T17:21:32+08:00] Successfully stopped preflight installer * [longhornctl install preflight](longhornctl_install_preflight.md) - Install Longhorn preflight -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_trim.md b/docs/longhornctl_trim.md index acb1d6145..9955e6b22 100644 --- a/docs/longhornctl_trim.md +++ b/docs/longhornctl_trim.md @@ -6,7 +6,7 @@ Longhorn trimming operations ``` -h, --help help for trim - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -26,4 +26,4 @@ Longhorn trimming operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl trim volume](longhornctl_trim_volume.md) - Trim a Longhorn volume -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_trim_volume.md b/docs/longhornctl_trim_volume.md index 0990cc383..d021c78d0 100644 --- a/docs/longhornctl_trim_volume.md +++ b/docs/longhornctl_trim_volume.md @@ -31,7 +31,7 @@ INFO[2024-07-16T17:32:01+08:00] Completed volume trimmer vo ``` -h, --help help for volume - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -51,4 +51,4 @@ INFO[2024-07-16T17:32:01+08:00] Completed volume trimmer vo * [longhornctl trim](longhornctl_trim.md) - Longhorn trimming operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/docs/longhornctl_version.md b/docs/longhornctl_version.md index 4e97b8640..f27d6d590 100644 --- a/docs/longhornctl_version.md +++ b/docs/longhornctl_version.md @@ -15,7 +15,7 @@ longhornctl version [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path @@ -29,4 +29,4 @@ longhornctl version [flags] * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 2-Nov-2026 diff --git a/pkg/consts/cmd.go b/pkg/consts/cmd.go index 94b3175a8..1846d3b7b 100644 --- a/pkg/consts/cmd.go +++ b/pkg/consts/cmd.go @@ -47,6 +47,7 @@ const ( CmdOptNodeSelector = "node-selector" CmdOptTolerations = "tolerations" CmdOptAll = "all" + CmdOptDataEngineMode = "data-engine-mode" // SPDK options CmdOptAllowPci = "allow-pci" diff --git a/pkg/consts/environment.go b/pkg/consts/environment.go index b982f6283..c0479364e 100644 --- a/pkg/consts/environment.go +++ b/pkg/consts/environment.go @@ -7,10 +7,12 @@ const ( ) const ( - EnvCurrentNodeID = "CURRENT_NODE_ID" - EnvKubeConfigPath = "KUBECONFIG" - EnvLogLevel = "LOG_LEVEL" - EnvOutputFilePath = "OUTPUT_FILE_PATH" + EnvCurrentNodeID = "CURRENT_NODE_ID" + EnvKubeConfigPath = "KUBECONFIG" + EnvLogLevel = "LOG_LEVEL" + EnvOutputFilePath = "OUTPUT_FILE_PATH" + EnvDataEngineMode = "DATA_ENGINE_MODE" + EnvPreflightCollectionOnly = "PREFLIGHT_COLLECTION_ONLY" EnvLonghornDataDirectory = "LONGHORN_DATA_DIRECTORY" EnvLonghornNamespace = "LONGHORN_NAMESPACE" diff --git a/pkg/local/preflight/checker.go b/pkg/local/preflight/checker.go index ca366fbeb..092d05b01 100644 --- a/pkg/local/preflight/checker.go +++ b/pkg/local/preflight/checker.go @@ -18,9 +18,7 @@ import ( kubeclient "k8s.io/client-go/kubernetes" commonkube "github.com/longhorn/go-common-libs/kubernetes" - commonnfs "github.com/longhorn/go-common-libs/nfs" commonns "github.com/longhorn/go-common-libs/ns" - commonsys "github.com/longhorn/go-common-libs/sys" commontypes "github.com/longhorn/go-common-libs/types" lhmgrutil "github.com/longhorn/longhorn-manager/util" @@ -41,7 +39,12 @@ type Checker struct { OutputFilePath string - kubeClient *kubeclient.Clientset + kubeClient kubeclient.Interface + nfsFacts checkerNFSFacts + + // CollectionOnly is the internal in-Pod transport phase. It preserves the + // full report for the following reporter; the remote CLI owns its verdict. + CollectionOnly bool osRelease string packageManager pkgmgr.PackageManager @@ -57,6 +60,9 @@ type Checker struct { // Init initializes the Checker. func (local *Checker) Init() error { + if err := local.ValidateDataEngineMode(); err != nil { + return err + } local.collection.Log = &types.LogCollection{} config, err := commonkube.GetInClusterConfig() @@ -74,9 +80,17 @@ func (local *Checker) Init() error { return errors.Wrap(err, "failed to get OS release") } local.osRelease = osRelease + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + if err := validateV2HostOS(osRelease); err != nil { + return err + } + } local.logger = logrus.WithField("os", local.osRelease) if local.osRelease == fmt.Sprint(consts.OperatingSystemContainerOptimizedOS) { + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + return fmt.Errorf("V2-only preflight is not qualified for Container-Optimized OS") + } return nil } @@ -100,10 +114,13 @@ func (local *Checker) Init() error { if err != nil { return err } + local.packageManager = packageManager + return local.configureDependencies(packageManagerType) +} +func (local *Checker) configureDependencies(packageManagerType pkgmgr.PackageManagerType) error { switch packageManagerType { case pkgmgr.PackageManagerApt: - local.packageManager = packageManager local.packages = []string{ "nfs-common", "open-iscsi", "cryptsetup", "dmsetup", } @@ -121,7 +138,6 @@ func (local *Checker) Init() error { } case pkgmgr.PackageManagerYum: - local.packageManager = packageManager local.packages = []string{ "nfs-utils", "iscsi-initiator-utils", "cryptsetup", "device-mapper", } @@ -139,7 +155,6 @@ func (local *Checker) Init() error { } case pkgmgr.PackageManagerZypper, pkgmgr.PackageManagerTransactionalUpdate, pkgmgr.PackageManagerBinary: - local.packageManager = packageManager local.packages = []string{ "nfs-client", "open-iscsi", "cryptsetup", "device-mapper", } @@ -157,7 +172,6 @@ func (local *Checker) Init() error { } case pkgmgr.PackageManagerPacman: - local.packageManager = packageManager local.packages = []string{ "nfs-utils", "open-iscsi", "cryptsetup", "device-mapper", } @@ -175,7 +189,11 @@ func (local *Checker) Init() error { } default: - return errors.Errorf("operating system (%v) package manager (%s) is not supported", osRelease, packageManagerType) + return errors.Errorf("operating system (%v) package manager (%s) is not supported", local.osRelease, packageManagerType) + } + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + local.packages = withoutISCSIStrings(local.packages) + local.modules = withoutISCSIStrings(local.modules) } return nil @@ -183,6 +201,22 @@ func (local *Checker) Init() error { // Run executes the preflight checks. func (local *Checker) Run() error { + if err := local.ValidateDataEngineMode(); err != nil { + return err + } + mode, err := types.ParseDataEngineMode(local.DataEngineMode) + if err != nil { + return err + } + if mode == types.DataEngineModeV2Only { + if err := validateV2HostOS(local.osRelease); err != nil { + return err + } + if local.osRelease == fmt.Sprint(consts.OperatingSystemContainerOptimizedOS) || local.packageManager == nil || local.kubeClient == nil || local.collection.Log == nil { + return fmt.Errorf("V2-only preflight requires a qualified initialized package-manager host") + } + local.collection.DataEngineMode = string(mode) + } checkTasks := []func() error{ local.checkKubeDNS, } @@ -194,15 +228,17 @@ func (local *Checker) Run() error { local.checkContainerOptimizedOS, ) default: + if mode.RequiresISCSI() { + checkTasks = append(checkTasks, local.checkIscsidService) + } checkTasks = append(checkTasks, - local.checkIscsidService, local.checkMultipathService, local.checkNFSv4Support, func() error { return local.checkPackagesInstalled(false) }, func() error { return local.checkModulesLoaded(false) }, ) - if local.EnableSpdk { + if mode.NeedsSPDK(local.EnableSpdk) { instructionSets := map[string][]string{ "amd64": {"sse4_2"}, "arm64": {"asimd"}, @@ -227,9 +263,32 @@ func (local *Checker) Run() error { } } + if mode == types.DataEngineModeV2Only && !local.CollectionOnly { + return local.QualificationError() + } return nil } +func (local *Checker) QualificationError() error { + if local.DataEngineMode != string(types.DataEngineModeV2Only) { + return nil + } + if local.collection.DataEngineMode != string(types.DataEngineModeV2Only) || local.collection.Log == nil || len(local.collection.Log.Error) != 0 || len(local.collection.Log.Info) == 0 { + return fmt.Errorf("V2-only host preflight is not qualified; inspect the complete report") + } + return nil +} + +func withoutISCSIStrings(dependencies []string) []string { + filtered := make([]string, 0, len(dependencies)) + for _, dependency := range dependencies { + if dependency != "open-iscsi" && dependency != "iscsi-initiator-utils" && dependency != "iscsi_tcp" { + filtered = append(filtered, dependency) + } + } + return filtered +} + // Output converts the collection to JSON and output to stdout or the output file. func (local *Checker) Output() error { local.logger.Tracef("Outputting preflight checks results") @@ -360,6 +419,9 @@ func (local *Checker) checkIscsidService() error { // checkHugePages checks if HugePages is enabled. func (local *Checker) checkHugePages() error { + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + return local.checkV2HugePages() + } logrus.Info("Checking if HugePages is enabled") topic := formatTopic(consts.PreflightCheckTopicHugePages) @@ -613,16 +675,20 @@ func (local *Checker) checkNFSv4Support() error { // check kernel capability var isKernelSupport = false + facts := local.nfsFacts + if facts == nil { + facts = hostCheckerNFSFacts{} + } - kernelVersion, err := utils.GetKernelVersion() + kernelVersion, err := facts.KernelVersion() if err != nil { return wrapInternalError(topic, fmt.Errorf("failed to detect kernel version: %v", err)) } hostBootDir := filepath.Join(consts.VolumeMountHostDirectory, commontypes.SysBootDirectory) - kernelConfigMap, err := commonsys.GetBootKernelConfigMap(hostBootDir, kernelVersion) + kernelConfigMap, err := facts.BootKernelConfig(hostBootDir, kernelVersion) if err != nil { // Fallback attempt: Read from procfs if the first attempt failed - kernelConfigMap, err = commonsys.GetProcKernelConfigMap("/proc") + kernelConfigMap, err = facts.ProcKernelConfig("/proc") if err != nil { return wrapInternalError(topic, fmt.Errorf("failed to read kernel config: %v", err)) } @@ -639,7 +705,7 @@ func (local *Checker) checkNFSv4Support() error { break } else if configVal == "m" { // Check if the module is loaded - moduleLoaded, err := utils.IsModuleLoaded(module) + moduleLoaded, err := facts.ModuleLoaded(module) if err != nil { logrus.Debugf("Failed to check if module %s is loaded: %v", module, err) continue @@ -662,7 +728,7 @@ func (local *Checker) checkNFSv4Support() error { var isSupportedNFSVersion bool hostEtcDir := filepath.Join(consts.VolumeMountHostDirectory, commontypes.SysEtcDirectory) - nfsMajor, nfsMinor, err := commonnfs.GetSystemDefaultNFSVersion(hostEtcDir) + nfsMajor, nfsMinor, err := facts.DefaultNFSVersion(hostEtcDir) if err == nil { isSupportedNFSVersion = nfsMajor == 4 && (nfsMinor == 0 || nfsMinor == 1 || nfsMinor == 2) } else if errors.Is(err, commontypes.ErrNotConfigured) { diff --git a/pkg/local/preflight/checker_data_engine_mode_test.go b/pkg/local/preflight/checker_data_engine_mode_test.go new file mode 100644 index 000000000..f1467de09 --- /dev/null +++ b/pkg/local/preflight/checker_data_engine_mode_test.go @@ -0,0 +1,177 @@ +package preflight + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/mock" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" + + pkgmgr "github.com/longhorn/cli/pkg/local/preflight/packagemanager" + "github.com/longhorn/cli/pkg/local/preflight/packagemanager/mocks" + remote "github.com/longhorn/cli/pkg/remote/preflight" +) + +type fixtureNFSFacts struct{ reads int } + +func (facts *fixtureNFSFacts) KernelVersion() (string, error) { + facts.reads++ + return "fixture", nil +} +func (facts *fixtureNFSFacts) BootKernelConfig(string, string) (map[string]string, error) { + facts.reads++ + return map[string]string{"CONFIG_NFS_V4": "y"}, nil +} +func (*fixtureNFSFacts) ProcKernelConfig(string) (map[string]string, error) { + return nil, nil +} +func (*fixtureNFSFacts) ModuleLoaded(string) (bool, error) { return true, nil } +func (facts *fixtureNFSFacts) DefaultNFSVersion(string) (int, int, error) { + facts.reads++ + return 4, 1, nil +} + +func TestV2OnlyCheckerForcesSPDKWithoutAnyISCSIProbe(t *testing.T) { + for _, missingNVMe := range []bool{false, true} { + for _, collectionOnly := range []bool{false, true} { + t.Run(strings.Join([]string{map[bool]string{false: "healthy", true: "missing-nvme"}[missingNVMe], map[bool]string{false: "direct", true: "collection"}[collectionOnly]}, "/"), func(t *testing.T) { + t.Setenv(consts.EnvCurrentNodeID, "fixture-node") + manager := new(mocks.MockPackageManager) + manager.On("GetServiceStatus", "multipathd.service").Return("", nil).Once() + for _, name := range []string{"nfs-common", "cryptsetup", "dmsetup"} { + manager.On("CheckPackageInstalled", name).Return("", nil).Once() + } + for _, name := range []string{"nfs", "dm_crypt", "nvme_tcp", "uio_pci_generic", "vfio_pci", "ublk_drv"} { + var moduleError error + if missingNVMe && name == "nvme_tcp" { + moduleError = pkgmgr.ErrPackageNotInstalled + } + manager.On("CheckModLoaded", name).Return(moduleError).Once() + } + manager.On("Modprobe", "ublk_drv", []string{"--dry-run"}).Return("", nil).Once() + manager.On("Execute", []string{}, "cat", []string{"/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"}, v2SPDKReadTimeout).Return("1024\n", nil).Once() + manager.On("Execute", []string{}, "sh", mock.Anything, mock.Anything).Return("/sys/kernel/iommu_groups/0\n", nil).Once() + instruction := "sse4_2" + if runtime.GOARCH == "arm64" { + instruction = "asimd" + } + manager.On("Execute", []string{}, "grep", mock.MatchedBy(func(args []string) bool { + return strings.Contains(strings.Join(args, " "), instruction) + }), mock.Anything).Return("", nil).Once() + facts := &fixtureNFSFacts{} + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + response.Header().Set("Content-Type", "application/json") + var object interface{} + switch request.URL.Path { + case "/api/v1/nodes/fixture-node": + object = &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "fixture-node"}, Status: corev1.NodeStatus{Capacity: corev1.ResourceList{corev1.ResourceName("hugepages-2Mi"): resource.MustParse("2Gi")}}} + case "/apis/apps/v1/namespaces/kube-system/deployments": + object = &appsv1.DeploymentList{Items: []appsv1.Deployment{{ObjectMeta: metav1.ObjectMeta{Name: "dns", Namespace: "kube-system", Labels: map[string]string{"k8s-app": "kube-dns"}}, Spec: appsv1.DeploymentSpec{Replicas: ptr.To(int32(2))}, Status: appsv1.DeploymentStatus{ReadyReplicas: 2}}}} + default: + t.Errorf("unexpected API read %s", request.URL.Path) + http.NotFound(response, request) + return + } + if err := json.NewEncoder(response).Encode(object); err != nil { + t.Error(err) + } + })) + t.Cleanup(server.Close) + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + checker := &Checker{CheckerCmdOptions: remote.CheckerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}, CollectionOnly: collectionOnly, + logger: logrus.NewEntry(logrus.New()), osRelease: "ubuntu", packageManager: manager, nfsFacts: facts, + collection: types.NodeCollection{Log: &types.LogCollection{}}, OutputFilePath: filepath.Join(t.TempDir(), "report.json"), + kubeClient: client} + if err := checker.configureDependencies(pkgmgr.PackageManagerApt); err != nil { + t.Fatal(err) + } + err = checker.Run() + if (err != nil) != (missingNVMe && !collectionOnly) { + t.Fatalf("qualification/transport verdict changed: %v", err) + } + if err := checker.Output(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(checker.OutputFilePath) + if err != nil { + t.Fatal(err) + } + var report map[string]json.RawMessage + if err := json.Unmarshal(data, &report); err != nil || string(report["dataEngineMode"]) != `"v2-only"` { + t.Fatal("persisted report is not bound to V2-only checks") + } + if (len(checker.collection.Log.Error) != 0) != missingNVMe || facts.reads != 3 { + t.Fatal("mandatory NVMe or native NFS checks were skipped") + } + if !strings.Contains(strings.Join(checker.collection.Log.Info, " "), "cryptsetup") || !strings.Contains(strings.Join(checker.collection.Log.Warn, " "), "multipathd") { + t.Fatal("encryption or multipath checks disappeared") + } + manager.AssertExpectations(t) + }) + } + } +} + +func TestV2OnlyCheckerDependencyProfilesRemoveOnlyISCSI(t *testing.T) { + for _, profile := range []pkgmgr.PackageManagerType{pkgmgr.PackageManagerApt, pkgmgr.PackageManagerYum, pkgmgr.PackageManagerZypper, pkgmgr.PackageManagerTransactionalUpdate, pkgmgr.PackageManagerBinary, pkgmgr.PackageManagerPacman} { + checker := &Checker{CheckerCmdOptions: remote.CheckerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}} + if err := checker.configureDependencies(profile); err != nil { + t.Fatal(err) + } + for _, name := range append(append([]string{}, checker.packages...), checker.modules...) { + if name == "open-iscsi" || name == "iscsi-initiator-utils" || name == "iscsi_tcp" { + t.Fatalf("profile %s retains iSCSI dependency %s", profile, name) + } + } + if len(checker.packages) != 3 || len(checker.modules) != 2 || len(checker.services) != 1 || len(checker.spdkDepModules) != 3 { + t.Fatalf("profile %s removed dependencies beyond iSCSI", profile) + } + } +} + +func TestV2HugepagesDoNotTrustKernelDefaultSizeOrCachedCapacity(t *testing.T) { + manager := new(mocks.MockPackageManager) + manager.On("Execute", []string{}, "grep", []string{"HugePages_Total", "/proc/meminfo"}, mock.Anything).Return("HugePages_Total: 1024\n", nil).Maybe() + manager.On("Execute", []string{}, "cat", []string{"/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"}, mock.Anything).Return("0\n", nil).Maybe() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path != "/api/v1/nodes/fixture-node" { + t.Errorf("unexpected read %s", r.URL.Path) + } + _ = json.NewEncoder(w).Encode(&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "fixture-node"}, Status: corev1.NodeStatus{Capacity: corev1.ResourceList{corev1.ResourceName("hugepages-2Mi"): resource.MustParse("2Gi")}}}) + })) + t.Cleanup(server.Close) + t.Setenv(consts.EnvCurrentNodeID, "fixture-node") + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + checker := Checker{CheckerCmdOptions: remote.CheckerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}, packageManager: manager, kubeClient: client, collection: types.NodeCollection{Log: &types.LogCollection{}}} + if err := checker.checkHugePages(); err != nil { + t.Fatal(err) + } + if len(checker.collection.Log.Error) == 0 { + t.Fatal("kernel default-size pages and stale Node capacity qualified missing actual 2MiB pages") + } +} diff --git a/pkg/local/preflight/checker_hugepages_v2.go b/pkg/local/preflight/checker_hugepages_v2.go new file mode 100644 index 000000000..6b6f43d8d --- /dev/null +++ b/pkg/local/preflight/checker_hugepages_v2.go @@ -0,0 +1,25 @@ +package preflight + +import ( + "fmt" + "strconv" + "strings" +) + +func (local *Checker) checkV2HugePages() error { + output, err := local.packageManager.Execute([]string{}, "cat", []string{"/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"}, v2SPDKReadTimeout) + if err != nil { + return fmt.Errorf("failed to read actual V2 2MiB allocation: %w", err) + } + value := strings.TrimSpace(output) + pages, err := strconv.ParseUint(value, 10, 64) + if err != nil || value == "" || strconv.FormatUint(pages, 10) != value { + return fmt.Errorf("actual V2 2MiB allocation is unresolved") + } + if pages < uint64(local.HugePageSize/2) { + local.collection.Log.Error = append(local.collection.Log.Error, "Actual host 2MiB hugepages are insufficient for V2") + return nil + } + local.collection.Log.Info = append(local.collection.Log.Info, "Verified actual host 2MiB hugepage allocation for V2") + return local.checkHugePagesCapacity() +} diff --git a/pkg/local/preflight/checker_nfs_facts.go b/pkg/local/preflight/checker_nfs_facts.go new file mode 100644 index 000000000..f1799410f --- /dev/null +++ b/pkg/local/preflight/checker_nfs_facts.go @@ -0,0 +1,34 @@ +package preflight + +import ( + commonnfs "github.com/longhorn/go-common-libs/nfs" + commonsys "github.com/longhorn/go-common-libs/sys" + + "github.com/longhorn/cli/pkg/utils" +) + +type checkerNFSFacts interface { + KernelVersion() (string, error) + BootKernelConfig(directory, version string) (map[string]string, error) + ProcKernelConfig(directory string) (map[string]string, error) + ModuleLoaded(name string) (bool, error) + DefaultNFSVersion(directory string) (int, int, error) +} + +type hostCheckerNFSFacts struct{} + +func (hostCheckerNFSFacts) KernelVersion() (string, error) { return utils.GetKernelVersion() } + +func (hostCheckerNFSFacts) BootKernelConfig(directory, version string) (map[string]string, error) { + return commonsys.GetBootKernelConfigMap(directory, version) +} + +func (hostCheckerNFSFacts) ProcKernelConfig(directory string) (map[string]string, error) { + return commonsys.GetProcKernelConfigMap(directory) +} + +func (hostCheckerNFSFacts) ModuleLoaded(name string) (bool, error) { return utils.IsModuleLoaded(name) } + +func (hostCheckerNFSFacts) DefaultNFSVersion(directory string) (int, int, error) { + return commonnfs.GetSystemDefaultNFSVersion(directory) +} diff --git a/pkg/local/preflight/data_engine_host.go b/pkg/local/preflight/data_engine_host.go new file mode 100644 index 000000000..d6e25c126 --- /dev/null +++ b/pkg/local/preflight/data_engine_host.go @@ -0,0 +1,12 @@ +package preflight + +import "fmt" + +func validateV2HostOS(osRelease string) error { + switch osRelease { + case "sles", "suse", "opensuse", "opensuse-leap", "sl-micro", "ubuntu", "debian", "rhel", "ol", "rocky", "centos", "fedora", "amzn", "opencloudos", "arch": + return nil + default: + return fmt.Errorf("V2-only host qualification is unsupported for this operating system; use its declarative host prerequisites") + } +} diff --git a/pkg/local/preflight/installer.go b/pkg/local/preflight/installer.go index 746a8321c..22e371cd9 100644 --- a/pkg/local/preflight/installer.go +++ b/pkg/local/preflight/installer.go @@ -70,6 +70,9 @@ func requiredPackages(required bool, names ...string) []Package { // Init initializes the Installer. func (local *Installer) Init() error { + if err := local.ValidateDataEngineMode(); err != nil { + return err + } local.collection.Log = &types.LogCollection{} config, err := commonkube.GetInClusterConfig() @@ -87,6 +90,11 @@ func (local *Installer) Init() error { return errors.Wrap(err, "failed to get OS release") } local.osRelease = osRelease + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + if err := validateV2HostOS(osRelease); err != nil { + return err + } + } local.logger = logrus.WithField("os", local.osRelease) packageManagerType, err := utils.GetPackageManagerType(osRelease) @@ -115,10 +123,13 @@ func (local *Installer) Init() error { if err != nil { return err } + local.packageManager = pkgMgr + return local.configureDependencies(packageManagerType, kernelRelease) +} +func (local *Installer) configureDependencies(packageManagerType pkgmgr.PackageManagerType, kernelRelease string) error { switch packageManagerType { case pkgmgr.PackageManagerApt: - local.packageManager = pkgMgr local.packages = requiredPackages(true, "nfs-common", "open-iscsi", "cryptsetup") local.modules = requiredPackages(true, "nfs", "dm_crypt") local.services = requiredPackages(true, "iscsid") @@ -126,43 +137,60 @@ func (local *Installer) Init() error { // The installation is kept for older ubuntu versions to ensure the module is present, and it won't cause issue on newer versions as the package manager will skip installation if the package is not found. local.spdkDepPackages = requiredPackages(false, "linux-modules-extra-"+kernelRelease) local.spdkDepModules = requiredPackages(true, "nvme_tcp", "uio_pci_generic", "vfio_pci") - return nil case pkgmgr.PackageManagerYum: - local.packageManager = pkgMgr local.packages = requiredPackages(true, "nfs-utils", "iscsi-initiator-utils", "cryptsetup") local.modules = requiredPackages(true, "nfs", "iscsi_tcp", "dm_crypt") local.services = requiredPackages(true, "iscsid") local.spdkDepPackages = requiredPackages(true) local.spdkDepModules = requiredPackages(true, "nvme_tcp", "uio_pci_generic", "vfio_pci") - return nil case pkgmgr.PackageManagerZypper, pkgmgr.PackageManagerTransactionalUpdate, pkgmgr.PackageManagerBinary: - local.packageManager = pkgMgr local.packages = requiredPackages(true, "nfs-client", "open-iscsi", "cryptsetup") local.modules = requiredPackages(true, "nfs", "iscsi_tcp", "dm_crypt") local.services = requiredPackages(true, "iscsid") local.spdkDepPackages = requiredPackages(true) local.spdkDepModules = requiredPackages(true, "nvme_tcp", "uio_pci_generic", "vfio_pci") - return nil case pkgmgr.PackageManagerPacman: - local.packageManager = pkgMgr local.packages = requiredPackages(true, "nfs-utils", "open-iscsi", "cryptsetup") local.modules = requiredPackages(true, "nfs", "iscsi_tcp", "dm_crypt") local.services = requiredPackages(true, "iscsid") local.spdkDepPackages = requiredPackages(true) local.spdkDepModules = requiredPackages(true, "nvme_tcp", "uio_pci_generic", "vfio_pci") - return nil default: - return errors.Errorf("Operating system (%v) package manager (%s) is not supported", osRelease, packageManagerType) + return errors.Errorf("Operating system (%v) package manager (%s) is not supported", local.osRelease, packageManagerType) } + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + local.packages = withoutISCSIPackages(local.packages) + local.modules = withoutISCSIPackages(local.modules) + local.services = withoutISCSIPackages(local.services) + } + return nil } func (local *Installer) Run() error { + if err := local.ValidateDataEngineMode(); err != nil { + return err + } + mode, err := types.ParseDataEngineMode(local.DataEngineMode) + if err != nil { + return err + } + if mode == types.DataEngineModeV2Only { + if err := validateV2HostOS(local.osRelease); err != nil { + return err + } + if local.packageManager == nil || local.collection.Log == nil || local.kubeClient == nil { + return errors.New("V2-only installation requires an initialized qualified package-manager host") + } + if err := local.qualifyV2SPDKHost(); err != nil { + return err + } + local.collection.DataEngineMode = string(mode) + } var rebootRequired bool - var err error if local.UpdatePackages { if err := local.updatePackageList(); err != nil { @@ -170,7 +198,7 @@ func (local *Installer) Run() error { } } - rebootRequired, err = local.checkAndinstallPackages(local.EnableSpdk) + rebootRequired, err = local.checkAndinstallPackages(mode.NeedsSPDK(local.EnableSpdk)) if err != nil { return err } @@ -178,6 +206,9 @@ func (local *Installer) Run() error { if rebootRequired { logrus.Warn("Need to reboot the system and execute longhornctl install preflight again") local.collection.Log.Warn = append(local.collection.Log.Warn, "Need to reboot the system and execute longhornctl install preflight again") + if mode == types.DataEngineModeV2Only { + return errors.New("V2-only installation is pending a host reboot; rerun installation and qualification afterward") + } return nil } @@ -189,7 +220,7 @@ func (local *Installer) Run() error { return err } - if local.EnableSpdk { + if mode.NeedsSPDK(local.EnableSpdk) { // Load ublk_drv module if supported by the kernel if _, err := local.packageManager.Modprobe("ublk_drv", "--dry-run"); err != nil { local.collection.Log.Warn = append(local.collection.Log.Warn, @@ -328,6 +359,9 @@ func (local *Installer) updatePackageList() error { // configureSPDKEnv configures SPDK environment. func (local *Installer) configureSPDKEnv() error { + if local.DataEngineMode == string(types.DataEngineModeV2Only) { + return local.configureV2SPDKEnv() + } // Blindly remove the SPDK source code directory if it exists. spdkPath := filepath.Join(consts.VolumeMountHostDirectory, consts.SpdkPath) if err := os.RemoveAll(spdkPath); err != nil { @@ -354,6 +388,16 @@ func (local *Installer) configureSPDKEnv() error { return nil } +func withoutISCSIPackages(dependencies []Package) []Package { + filtered := make([]Package, 0, len(dependencies)) + for _, dependency := range dependencies { + if dependency.Name != "open-iscsi" && dependency.Name != "iscsi-initiator-utils" && dependency.Name != "iscsi_tcp" && dependency.Name != "iscsid" { + filtered = append(filtered, dependency) + } + } + return filtered +} + // getEnvsForConfiguringSPDKEnv parses the comma-separated --spdk-options string // into a list of KEY=VALUE environment variables. SPDK's scripts/setup.sh reads // its tunables (e.g. HUGEMEM, HUGENODE, PERSIST_HUGE, PCI_ALLOWED) from the diff --git a/pkg/local/preflight/installer_data_engine_mode_test.go b/pkg/local/preflight/installer_data_engine_mode_test.go new file mode 100644 index 000000000..ba3fee76b --- /dev/null +++ b/pkg/local/preflight/installer_data_engine_mode_test.go @@ -0,0 +1,36 @@ +package preflight + +import ( + "strings" + "testing" + + pkgmgr "github.com/longhorn/cli/pkg/local/preflight/packagemanager" + remote "github.com/longhorn/cli/pkg/remote/preflight" +) + +func TestV2InstallerRejectsInvalidCapabilityBeforeInitialization(t *testing.T) { + installer := Installer{InstallerCmdOptions: remote.InstallerCmdOptions{DataEngineMode: "unknown"}} + if err := installer.Init(); err == nil || !strings.Contains(err.Error(), "data-engine-mode") { + t.Fatalf("invalid mode reached host or connection initialization: %v", err) + } +} + +func TestV2InstallerProfilesRemoveOnlyISCSI(t *testing.T) { + for _, profile := range []pkgmgr.PackageManagerType{pkgmgr.PackageManagerApt, pkgmgr.PackageManagerYum, pkgmgr.PackageManagerZypper, pkgmgr.PackageManagerTransactionalUpdate, pkgmgr.PackageManagerBinary, pkgmgr.PackageManagerPacman} { + installer := Installer{InstallerCmdOptions: remote.InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}} + if err := installer.configureDependencies(profile, "fixture-kernel"); err != nil { + t.Fatal(err) + } + for _, dependency := range append(append(append([]Package{}, installer.packages...), installer.modules...), installer.services...) { + if dependency.Name == "open-iscsi" || dependency.Name == "iscsi-initiator-utils" || dependency.Name == "iscsi_tcp" || dependency.Name == "iscsid" { + t.Fatalf("V2 profile retains %s", dependency.Name) + } + if !dependency.Required { + t.Fatalf("required dependency became optional: %s", dependency.Name) + } + } + if len(installer.packages) != 2 || len(installer.modules) != 2 || len(installer.services) != 0 || len(installer.spdkDepModules) != 3 { + t.Fatalf("V2 profile removed non-iSCSI dependencies: %s", profile) + } + } +} diff --git a/pkg/local/preflight/installer_spdk_v2.go b/pkg/local/preflight/installer_spdk_v2.go new file mode 100644 index 000000000..85fcd0f5d --- /dev/null +++ b/pkg/local/preflight/installer_spdk_v2.go @@ -0,0 +1,104 @@ +package preflight + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + cp "github.com/otiai10/copy" + + "github.com/longhorn/cli/pkg/consts" +) + +const v2SPDKReadTimeout = 15 * time.Second +const v2SPDKSetupTimeout = 3 * time.Minute + +var v2HostReadEnvironment = []string{"-i", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "LC_ALL=C", "/bin/bash", "--noprofile", "--norc"} + +// The pinned setup script accepts eval-like NUMA grammar and optionally sources +// host-provided scripts. V2 accepts one canonical node and rejects those hooks. +func (local *Installer) qualifyV2SPDKHost() error { + _, node, err := local.V2SPDKSetupEnvironment() + if err != nil { + return err + } + args := append(append([]string{}, v2HostReadEnvironment...), "-c", `set -eu +test -x /usr/bin/env +test -x /bin/bash +test ! -e /bin/wpdk_common.sh && test ! -L /bin/wpdk_common.sh +test ! -e /etc/opt/spdk-pkgdep/paths/export.sh && test ! -L /etc/opt/spdk-pkgdep/paths/export.sh +test -d "/sys/devices/system/node/node$1/hugepages/hugepages-2048kB" +test -r "/sys/devices/system/node/node$1/hugepages/hugepages-2048kB/nr_hugepages" +test -r "/sys/devices/system/node/node$1/hugepages/hugepages-2048kB/free_hugepages" +`, "--", node) + if _, err := local.packageManager.Execute(nil, "/usr/bin/env", args, v2SPDKReadTimeout); err != nil { + return fmt.Errorf("V2 SPDK host prerequisites are unqualified: %w", err) + } + _, err = local.readV2NodeHugepages(node) + return err +} + +func (local *Installer) readV2NodeHugepages(node string) (uint64, error) { + path := "/sys/devices/system/node/node" + node + "/hugepages/hugepages-2048kB/nr_hugepages" + args := append(append([]string{}, v2HostReadEnvironment...), "-c", `cat -- "$1"`, "--", path) + output, err := local.packageManager.Execute(nil, "/usr/bin/env", args, v2SPDKReadTimeout) + if err != nil { + return 0, fmt.Errorf("failed to read actual V2 2MiB allocation: %w", err) + } + value := strings.TrimSpace(output) + pages, err := strconv.ParseUint(value, 10, 64) + if err != nil || value == "" || strconv.FormatUint(pages, 10) != value { + return 0, fmt.Errorf("actual V2 2MiB allocation is unresolved") + } + return pages, nil +} + +func (local *Installer) configureV2SPDKEnv() (resultErr error) { + environment, node, err := local.V2SPDKSetupEnvironment() + if err != nil { + return err + } + if err := local.qualifyV2SPDKHost(); err != nil { + return err + } + // Copy only into this invocation's new directory. Never remove or reuse a + // shared host SPDK tree belonging to another invocation or application. + hostDirectory, err := os.MkdirTemp(filepath.Join(consts.VolumeMountHostDirectory, "tmp"), "longhorn-spdk-v2-") + if err != nil { + return err + } + defer func() { resultErr = errors.Join(resultErr, os.RemoveAll(hostDirectory)) }() + if err := cp.Copy("/spdk", hostDirectory); err != nil { + return err + } + if err := local.qualifyV2SPDKHost(); err != nil { + return err + } + relative, err := filepath.Rel(consts.VolumeMountHostDirectory, hostDirectory) + if err != nil || strings.HasPrefix(relative, "..") || filepath.IsAbs(relative) { + return fmt.Errorf("V2 setup host directory is not identified") + } + setupPath := filepath.Join("/", relative, "scripts/setup.sh") + return local.executeV2SPDKSetup(setupPath, environment, node) +} + +func (local *Installer) executeV2SPDKSetup(setupPath string, environment []string, node string) error { + args := append([]string{"-i"}, environment...) + args = append(args, "/bin/bash", "--noprofile", "--norc", "--", setupPath, "config") + if _, err := local.packageManager.Execute(nil, "/usr/bin/env", args, v2SPDKSetupTimeout); err != nil { + return fmt.Errorf("V2 SPDK setup failed: %w", err) + } + pages, err := local.readV2NodeHugepages(node) + if err != nil { + return err + } + if pages < uint64(local.HugePageSize/2) { + return fmt.Errorf("V2 SPDK setup did not allocate the requested 2MiB hugepages on the selected NUMA node") + } + local.collection.Log.Info = append(local.collection.Log.Info, "Configured V2 SPDK environment and verified actual 2MiB hugepage allocation; run V2 preflight qualification after any required kubelet restart") + return nil +} diff --git a/pkg/local/preflight/installer_spdk_v2_test.go b/pkg/local/preflight/installer_spdk_v2_test.go new file mode 100644 index 000000000..d1a835771 --- /dev/null +++ b/pkg/local/preflight/installer_spdk_v2_test.go @@ -0,0 +1,69 @@ +package preflight + +import ( + "errors" + "reflect" + "testing" + + "github.com/stretchr/testify/mock" + + "github.com/longhorn/cli/pkg/local/preflight/packagemanager/mocks" + remote "github.com/longhorn/cli/pkg/remote/preflight" + "github.com/longhorn/cli/pkg/types" +) + +func TestV2SPDKSetupClearsInheritedOptionsAndVerifiesActualAllocation(t *testing.T) { + t.Setenv("BASH_ENV", "/unqualified/startup") + t.Setenv("NRHUGE", "0") + t.Setenv("PCI_ALLOWED", "all") + for _, test := range []struct { + name, pages string + setupError bool + valid bool + }{ + {"complete", "1024", false, true}, + {"setup-failed", "", true, false}, + {"partial-allocation", "1000", false, false}, + {"unreadable-allocation", "unknown", false, false}, + } { + t.Run(test.name, func(t *testing.T) { + manager := new(mocks.MockPackageManager) + installer := Installer{InstallerCmdOptions: remote.InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048, AllowPci: "none"}, packageManager: manager, collection: types.NodeCollection{Log: &types.LogCollection{}}} + environment, node, err := installer.V2SPDKSetupEnvironment() + if err != nil { + t.Fatal(err) + } + args := append([]string{"-i"}, environment...) + args = append(args, "/bin/bash", "--noprofile", "--norc", "--", "/tmp/owned/scripts/setup.sh", "config") + var setupErr error + if test.setupError { + setupErr = errors.New("native setup failed") + } + manager.On("Execute", []string(nil), "/usr/bin/env", mock.MatchedBy(func(actual []string) bool { return reflect.DeepEqual(actual, args) }), v2SPDKSetupTimeout).Return("", setupErr).Once() + if !test.setupError { + manager.On("Execute", []string(nil), "/usr/bin/env", mock.Anything, v2SPDKReadTimeout).Return(test.pages+"\n", nil).Once() + } + err = installer.executeV2SPDKSetup("/tmp/owned/scripts/setup.sh", environment, node) + if (err == nil) != test.valid { + t.Fatalf("incorrect setup verdict: %v", err) + } + if test.setupError && !errors.Is(err, setupErr) { + t.Fatal("native setup failure was swallowed") + } + if (len(installer.collection.Log.Info) > 0) != test.valid { + t.Fatal("setup reported success before actual allocation was verified") + } + manager.AssertExpectations(t) + }) + } +} + +func TestV2SPDKHostQualificationRejectsUnsupportedHooksBeforeWrites(t *testing.T) { + manager := new(mocks.MockPackageManager) + installer := Installer{InstallerCmdOptions: remote.InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048, AllowPci: "none"}, packageManager: manager} + manager.On("Execute", []string(nil), "/usr/bin/env", mock.Anything, v2SPDKReadTimeout).Return("", errors.New("unqualified host hook or NUMA interface")).Once() + if err := installer.qualifyV2SPDKHost(); err == nil { + t.Fatal("unsupported host profile qualified") + } + manager.AssertExpectations(t) +} diff --git a/pkg/remote/preflight/checker.go b/pkg/remote/preflight/checker.go index 1b82e5ac0..712a90007 100644 --- a/pkg/remote/preflight/checker.go +++ b/pkg/remote/preflight/checker.go @@ -1,9 +1,13 @@ package preflight import ( + "context" + "crypto/rand" "encoding/json" "path/filepath" "reflect" + "strings" + "time" "github.com/pkg/errors" @@ -13,6 +17,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeclient "k8s.io/client-go/kubernetes" @@ -32,13 +37,16 @@ type Checker struct { kubeClient *kubeclient.Clientset - appName string // App name of the DaemonSet. + appName string // App name of the DaemonSet. + createdDaemonSet *appsv1.DaemonSet + createdRBAC *v2RBACReceipt } // CheckerCmdOptions holds the options for the command. type CheckerCmdOptions struct { types.GlobalCmdOptions + DataEngineMode string EnableSpdk bool HugePageSize int UserspaceDriver string @@ -46,6 +54,9 @@ type CheckerCmdOptions struct { // Init initializes the Checker. func (remote *Checker) Init() error { + if err := remote.ValidateDataEngineMode(); err != nil { + return err + } kubeClient, err := kubeutils.NewKubeClient("", remote.KubeConfigPath) if err != nil { return err @@ -54,11 +65,18 @@ func (remote *Checker) Init() error { remote.kubeClient = kubeClient remote.appName = consts.AppNamePreflightChecker + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + // A V2 run never reuses or pre-cleans another run's checker or reports. + remote.appName += "-" + strings.ToLower(rand.Text()) + } return nil } // Run creates the DaemonSet for the preflight check, and waits for it to complete. func (remote *Checker) Run() (string, error) { + if err := remote.ValidateDataEngineMode(); err != nil { + return "", err + } // Create RBAC to check: // - the node agent existence when the cluster is running on Container-Optimized OS (COS) // - replica count of the DNS deployment @@ -75,7 +93,14 @@ func (remote *Checker) Run() (string, error) { Verbs: []string{"get"}, }, } - err := kubeutils.CreateRbac(remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + var err error + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + remote.createdRBAC, err = createV2RBAC(ctx, remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + } else { + err = kubeutils.CreateRbac(remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + } if err != nil { return "", err } @@ -84,10 +109,23 @@ func (remote *Checker) Run() (string, error) { if err != nil { return "", err } - daemonSet, err := commonkube.CreateDaemonSet(remote.kubeClient, newDaemonSet) + var daemonSet *appsv1.DaemonSet + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + daemonSet, err = remote.kubeClient.AppsV1().DaemonSets(remote.Namespace).Create(ctx, newDaemonSet, metav1.CreateOptions{}) + } else { + daemonSet, err = commonkube.CreateDaemonSet(remote.kubeClient, newDaemonSet) + } if err != nil { return "", err } + remote.createdDaemonSet = daemonSet.DeepCopy() + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + if err := validateV2CreatedDaemonSet(newDaemonSet, daemonSet, consts.SubCmdCheck); err != nil { + return "", err + } + } err = kubeutils.MonitorDaemonSetContainer(remote.kubeClient, daemonSet, consts.ContainerNameInit, kubeutils.WaitForDaemonSetContainersExit, ptr.To(consts.ContainerConditionMaxTolerationMedium)) if err != nil { @@ -98,6 +136,9 @@ func (remote *Checker) Run() (string, error) { if err != nil { return "", err } + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + return collectV2Reports(context.Background(), remote.kubeClient, daemonSet, readV2PodReport(remote.kubeClient)) + } podCollections, err := kubeutils.GetDaemonSetPodCollections(remote.kubeClient, daemonSet, consts.ContainerNameOutput, false, false, nil) if err != nil { @@ -133,6 +174,20 @@ func (remote *Checker) Run() (string, error) { // Cleanup deletes the DaemonSet created for the preflight check. func (remote *Checker) Cleanup() error { var resultErr error + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if remote.createdDaemonSet != nil { + uid := remote.createdDaemonSet.UID + if uid == "" { + return errors.New("refusing V2 checker cleanup without its created UID") + } + if err := remote.kubeClient.AppsV1().DaemonSets(remote.createdDaemonSet.Namespace).Delete(ctx, remote.createdDaemonSet.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return cleanupV2RBAC(ctx, remote.kubeClient, remote.createdRBAC) + } if err := commonkube.DeleteDaemonSet(remote.kubeClient, remote.Namespace, remote.appName); err != nil { resultErr = errors.Wrap(err, "failed to delete DaemonSet") @@ -152,6 +207,12 @@ func (remote *Checker) Cleanup() error { // newDaemonSet prepares a DaemonSet for the preflight check. func (remote *Checker) newDaemonSet() *appsv1.DaemonSet { outputFilePath := filepath.Join(consts.VolumeMountSharedDirectory, consts.FileNameOutputJSON) + command := []string{consts.CmdLonghornctlLocal, consts.SubCmdCheck, consts.SubCmdPreflight} + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + // Older local images must reject the explicit flag, rather than + // silently ignore an environment variable and report V1 checks. + command = append(command, "--"+consts.CmdOptDataEngineMode, remote.DataEngineMode) + } return &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: remote.appName, @@ -179,7 +240,7 @@ func (remote *Checker) newDaemonSet() *appsv1.DaemonSet { { Name: consts.ContainerNameInit, Image: utils.BuildImageName(remote.Image, remote.ImageRegistry), - Command: []string{consts.CmdLonghornctlLocal, consts.SubCmdCheck, consts.SubCmdPreflight}, + Command: command, Env: []corev1.EnvVar{ { Name: consts.EnvLogLevel, @@ -189,6 +250,14 @@ func (remote *Checker) newDaemonSet() *appsv1.DaemonSet { Name: consts.EnvOutputFilePath, Value: outputFilePath, }, + { + Name: consts.EnvDataEngineMode, + Value: remote.DataEngineMode, + }, + { + Name: consts.EnvPreflightCollectionOnly, + Value: commonutils.ConvertTypeToString(remote.DataEngineMode == string(types.DataEngineModeV2Only)), + }, { Name: consts.EnvEnableSpdk, Value: commonutils.ConvertTypeToString(remote.EnableSpdk), diff --git a/pkg/remote/preflight/checker_collection_v2.go b/pkg/remote/preflight/checker_collection_v2.go new file mode 100644 index 000000000..1b798a816 --- /dev/null +++ b/pkg/remote/preflight/checker_collection_v2.go @@ -0,0 +1,475 @@ +package preflight + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "reflect" + "sort" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + kubeclient "k8s.io/client-go/kubernetes" + "sigs.k8s.io/kustomize/kyaml/yaml" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" +) + +const maxV2ReportBytes = 256 * 1024 + +func decodeV2Report(data []byte) (*types.NodeCollection, error) { + if len(data) == 0 || len(data) > maxV2ReportBytes { + return nil, fmt.Errorf("missing or oversized V2 report") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + if err := uniqueJSONValue(decoder, 0); err != nil { + return nil, fmt.Errorf("invalid or ambiguous V2 report") + } + if _, err := decoder.Token(); err != io.EOF { + return nil, fmt.Errorf("V2 report contains trailing content") + } + decoder = json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return nil, fmt.Errorf("V2 report must be an object") + } + for name := range fields { + if name != "dataEngineMode" && name != "log" { + return nil, fmt.Errorf("V2 report has a noncanonical field") + } + } + var logs map[string]json.RawMessage + if err := json.Unmarshal(fields["log"], &logs); err != nil { + return nil, fmt.Errorf("V2 report log must be an object") + } + for name := range logs { + if name != "info" && name != "warn" && name != "error" { + return nil, fmt.Errorf("V2 report log has a noncanonical field") + } + } + var report types.NodeCollection + if err := decoder.Decode(&report); err != nil { + return nil, fmt.Errorf("V2 report has an invalid schema") + } + if report.DataEngineMode != string(types.DataEngineModeV2Only) || report.Log == nil || len(report.Log.Info) == 0 { + return nil, fmt.Errorf("V2 report is missing capability or check evidence") + } + return &report, nil +} + +func uniqueJSONValue(decoder *json.Decoder, depth int) error { + if depth > 32 { + return fmt.Errorf("JSON nesting exceeds limit") + } + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, ok := token.(json.Delim) + if !ok { + return nil + } + switch delimiter { + case '{': + keys := map[string]bool{} + for decoder.More() { + key, err := decoder.Token() + if err != nil { + return err + } + name, ok := key.(string) + if !ok || keys[name] { + return fmt.Errorf("duplicate JSON key") + } + keys[name] = true + if err := uniqueJSONValue(decoder, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := uniqueJSONValue(decoder, depth+1); err != nil { + return err + } + } + default: + return fmt.Errorf("unexpected JSON delimiter") + } + end, err := decoder.Token() + if err != nil || (delimiter == '{' && end != json.Delim('}')) || (delimiter == '[' && end != json.Delim(']')) { + return fmt.Errorf("incomplete JSON value") + } + return nil +} + +type v2ReportLogReader func(context.Context, string, string) ([]byte, error) + +type v2PodEvidence struct { + UID string + Name, Node string + Spec corev1.PodSpec + Owners []metav1.OwnerReference + Instances []corev1.ContainerStatus +} + +// The legacy collector intentionally tolerates missing logs. V2 qualification +// uses a separate complete inventory and brackets logs against exact instances. +func collectV2Reports(ctx context.Context, client kubeclient.Interface, expected *appsv1.DaemonSet, readLog v2ReportLogReader) (string, error) { + return collectV2OperationReports(ctx, client, expected, readLog, consts.SubCmdCheck) +} + +func collectV2OperationReports(ctx context.Context, client kubeclient.Interface, expected *appsv1.DaemonSet, readLog v2ReportLogReader, operation string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + if expected == nil || expected.UID == "" || expected.Generation <= 0 || expected.Spec.Selector == nil { + return "", fmt.Errorf("V2 collection requires a newly created identified DaemonSet") + } + beforeDS, beforePods, err := snapshotV2Checker(ctx, client, expected, operation) + if err != nil { + return "", err + } + nodes := map[string]*types.LogCollection{} + qualified := true + total := 0 + for _, pod := range beforePods { + data, err := readLog(ctx, expected.Namespace, pod.Name) + if err != nil { + return "", fmt.Errorf("failed to read a complete V2 report for node %s: %w", pod.Node, err) + } + total += len(data) + if total > 32*1024*1024 { + return "", fmt.Errorf("V2 report collection exceeds total limit") + } + report, err := decodeV2Report(data) + if err != nil { + return "", fmt.Errorf("invalid V2 report for node %s: %w", pod.Node, err) + } + nodes[pod.Node] = report.Log + qualified = qualified && len(report.Log.Error) == 0 + } + afterDS, afterPods, err := snapshotV2Checker(ctx, client, expected, operation) + if err != nil { + return "", err + } + if beforeDS.Status.DesiredNumberScheduled != afterDS.Status.DesiredNumberScheduled || !reflect.DeepEqual(beforePods, afterPods) { + return "", fmt.Errorf("V2 checker membership or execution changed while collecting reports") + } + output, err := yaml.Marshal(nodes) + if err != nil { + return "", err + } + if !qualified { + return string(output), fmt.Errorf("V2-only host preflight is not qualified; inspect the complete report") + } + return string(output), nil +} + +func snapshotV2Checker(ctx context.Context, client kubeclient.Interface, expected *appsv1.DaemonSet, operation string) (*appsv1.DaemonSet, []v2PodEvidence, error) { + ds, err := client.AppsV1().DaemonSets(expected.Namespace).Get(ctx, expected.Name, metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + if ds.UID != expected.UID || ds.Generation != expected.Generation || ds.DeletionTimestamp != nil || !reflect.DeepEqual(ds.Spec, expected.Spec) || ds.Status.ObservedGeneration < ds.Generation || ds.Status.DesiredNumberScheduled <= 0 || ds.Status.DesiredNumberScheduled > 4096 || ds.Status.CurrentNumberScheduled != ds.Status.DesiredNumberScheduled || ds.Status.UpdatedNumberScheduled != ds.Status.DesiredNumberScheduled || ds.Status.NumberMisscheduled != 0 { + return nil, nil, fmt.Errorf("V2 checker DaemonSet identity, specification or membership is unqualified") + } + selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector) + if err != nil || selector.Empty() { + return nil, nil, fmt.Errorf("V2 checker requires a specific selector") + } + evidence := []v2PodEvidence{} + seenNodes, seenUIDs, seenNames := map[string]bool{}, map[string]bool{}, map[string]bool{} + continuation, resourceVersion := "", "" + seenContinuations := map[string]bool{} + total := 0 + for page := 0; ; page++ { + if page >= 32 { + return nil, nil, fmt.Errorf("V2 Pod inventory exceeds page limit") + } + pods, err := client.CoreV1().Pods(ds.Namespace).List(ctx, metav1.ListOptions{Limit: 256, Continue: continuation}) + if err != nil { + return nil, nil, err + } + if pods.ResourceVersion == "" || (resourceVersion != "" && pods.ResourceVersion != resourceVersion) { + return nil, nil, fmt.Errorf("V2 Pod inventory is not a complete consistent snapshot") + } + resourceVersion = pods.ResourceVersion + total += len(pods.Items) + if total > 8192 { + return nil, nil, fmt.Errorf("V2 Pod inventory exceeds object limit") + } + for _, pod := range pods.Items { + owned, controllerCount := false, 0 + for _, owner := range pod.OwnerReferences { + if owner.Controller != nil && *owner.Controller { + controllerCount++ + owned = owner.UID == ds.UID && owner.Name == ds.Name && owner.Kind == "DaemonSet" && owner.APIVersion == "apps/v1" + } + } + matches := selector.Matches(labels.Set(pod.Labels)) + if !owned && !matches { + // A non-controlling reference must not hide an owned-looking Pod. + for _, owner := range pod.OwnerReferences { + if owner.UID == ds.UID { + return nil, nil, fmt.Errorf("V2 checker Pod has an ambiguous owner") + } + } + continue + } + if !owned || !matches || controllerCount != 1 || pod.UID == "" || pod.Name == "" || pod.Namespace != ds.Namespace || pod.Spec.NodeName == "" || pod.DeletionTimestamp != nil || seenNodes[pod.Spec.NodeName] || seenUIDs[string(pod.UID)] || seenNames[pod.Name] { + return nil, nil, fmt.Errorf("V2 checker Pod inventory has missing, foreign or duplicate execution") + } + instances, err := v2PodInstances(ds, &pod, operation) + if err != nil { + return nil, nil, err + } + seenNodes[pod.Spec.NodeName], seenUIDs[string(pod.UID)], seenNames[pod.Name] = true, true, true + evidence = append(evidence, v2PodEvidence{UID: string(pod.UID), Name: pod.Name, Node: pod.Spec.NodeName, Spec: pod.Spec, Owners: pod.OwnerReferences, Instances: instances}) + } + continuation = pods.Continue + if continuation == "" { + break + } + if seenContinuations[continuation] { + return nil, nil, fmt.Errorf("V2 Pod inventory repeated a continuation") + } + seenContinuations[continuation] = true + } + if len(evidence) != int(ds.Status.DesiredNumberScheduled) { + return nil, nil, fmt.Errorf("V2 checker is missing a report-producing Pod") + } + sort.Slice(evidence, func(i, j int) bool { return evidence[i].Node < evidence[j].Node }) + return ds, evidence, nil +} + +func v2PodInstances(ds *appsv1.DaemonSet, pod *corev1.Pod, operation string) ([]corev1.ContainerStatus, error) { + template := ds.Spec.Template.Spec + if !validV2CheckerTemplate(template, operation) || len(pod.Spec.InitContainers) != 2 || len(pod.Spec.EphemeralContainers) != 0 || len(pod.Spec.Containers) != len(template.Containers) || pod.Spec.ServiceAccountName != template.ServiceAccountName || pod.Spec.HostPID != template.HostPID { + return nil, fmt.Errorf("V2 checker Pod execution does not match its DaemonSet") + } + for _, volume := range template.Volumes { + found := false + for _, actual := range pod.Spec.Volumes { + if actual.Name == volume.Name { + found = reflect.DeepEqual(actual, volume) + } + } + if !found { + return nil, fmt.Errorf("V2 checker output or host volume changed") + } + } + serviceAccountVolume, valid := v2AdditionalVolumes(template.Volumes, pod.Spec.Volumes) + if !valid { + return nil, fmt.Errorf("V2 checker has an unrecognized additional volume") + } + instances := make([]corev1.ContainerStatus, 0, 2) + for index, expected := range template.InitContainers { + actual := pod.Spec.InitContainers[index] + if !sameV2ContainerExecution(expected, actual, serviceAccountVolume) { + return nil, fmt.Errorf("V2 checker or reporter execution changed") + } + var status *corev1.ContainerStatus + for _, candidate := range pod.Status.InitContainerStatuses { + if candidate.Name == expected.Name { + if status != nil { + return nil, fmt.Errorf("duplicate V2 checker container status") + } + copy := candidate + status = © + } + } + if status == nil || status.State.Terminated == nil || status.ImageID == "" || status.State.Terminated.ExitCode != 0 || status.State.Terminated.ContainerID == "" || status.State.Terminated.StartedAt.IsZero() || status.State.Terminated.FinishedAt.IsZero() || status.State.Terminated.FinishedAt.Before(&status.State.Terminated.StartedAt) || (status.ContainerID != "" && status.ContainerID != status.State.Terminated.ContainerID) { + return nil, fmt.Errorf("V2 checker or reporter did not complete successfully") + } + if index == 1 && status.State.Terminated.StartedAt.Before(&instances[0].State.Terminated.FinishedAt) { + return nil, fmt.Errorf("V2 reporter ran before its checker finished") + } + instances = append(instances, *status) + } + for index, expected := range template.Containers { + if !sameV2ContainerExecution(expected, pod.Spec.Containers[index], serviceAccountVolume) { + return nil, fmt.Errorf("V2 checker sidecar execution changed") + } + } + return instances, nil +} + +func validV2CheckerTemplate(template corev1.PodSpec, operation string) bool { + if operation != consts.SubCmdCheck && operation != consts.SubCmdInstall { + return false + } + if len(template.InitContainers) != 2 || template.InitContainers[0].Name != consts.ContainerNameInit || template.InitContainers[1].Name != consts.ContainerNameOutput { + return false + } + checker, reporter := template.InitContainers[0], template.InitContainers[1] + if !reflect.DeepEqual(checker.Command, []string{consts.CmdLonghornctlLocal, operation, consts.SubCmdPreflight, "--" + consts.CmdOptDataEngineMode, string(types.DataEngineModeV2Only)}) || len(checker.Args) != 0 || !reflect.DeepEqual(reporter.Command, []string{"cat", "/shared/output.json"}) || len(reporter.Args) != 0 { + return false + } + environment := map[string]corev1.EnvVar{} + for _, variable := range checker.Env { + if _, exists := environment[variable.Name]; exists { + return false + } + environment[variable.Name] = variable + } + mode, collection, node := environment[consts.EnvDataEngineMode], environment[consts.EnvPreflightCollectionOnly], environment[consts.EnvCurrentNodeID] + collectionValue := "true" + if operation == consts.SubCmdInstall { + collectionValue = "" + } + return mode.Value == string(types.DataEngineModeV2Only) && mode.ValueFrom == nil && collection.Value == collectionValue && collection.ValueFrom == nil && node.Value == "" && node.ValueFrom != nil && node.ValueFrom.FieldRef != nil && node.ValueFrom.FieldRef.FieldPath == "spec.nodeName" +} + +func sameV2ContainerExecution(expected, actual corev1.Container, serviceAccountVolume string) bool { + if expected.Name != actual.Name || expected.Image != actual.Image || expected.WorkingDir != actual.WorkingDir || expected.Stdin != actual.Stdin || expected.TTY != actual.TTY || len(actual.VolumeDevices) != 0 || !reflect.DeepEqual(expected.Command, actual.Command) || !reflect.DeepEqual(expected.Args, actual.Args) || !reflect.DeepEqual(expected.EnvFrom, actual.EnvFrom) || !reflect.DeepEqual(expected.SecurityContext, actual.SecurityContext) || !reflect.DeepEqual(expected.Lifecycle, actual.Lifecycle) || len(expected.Env) != len(actual.Env) { + return false + } + seen := map[string]bool{} + for index, variable := range actual.Env { + if seen[variable.Name] { + return false + } + seen[variable.Name] = true + left, right := expected.Env[index].DeepCopy(), variable.DeepCopy() + for _, value := range []*corev1.EnvVar{left, right} { + if value.ValueFrom != nil && value.ValueFrom.FieldRef != nil && value.ValueFrom.FieldRef.APIVersion == "" { + value.ValueFrom.FieldRef.APIVersion = "v1" + } + } + if !reflect.DeepEqual(left, right) { + return false + } + } + for _, mount := range expected.VolumeMounts { + found := false + for _, actualMount := range actual.VolumeMounts { + if actualMount.Name == mount.Name { + found = reflect.DeepEqual(mount, actualMount) + } + } + if !found { + return false + } + } + for _, mount := range actual.VolumeMounts { + found := false + for _, expectedMount := range expected.VolumeMounts { + if reflect.DeepEqual(mount, expectedMount) { + found = true + } + } + if !found && (serviceAccountVolume == "" || !reflect.DeepEqual(mount, corev1.VolumeMount{Name: serviceAccountVolume, MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", ReadOnly: true})) { + return false + } + } + return true +} + +// Pod admission adds one API-token volume which is absent from the DaemonSet +// template. No other injected volume or mount may shadow the checker/report. +func v2AdditionalVolumes(expected, actual []corev1.Volume) (string, bool) { + names := map[string]bool{} + for _, volume := range expected { + names[volume.Name] = true + } + serviceAccountVolume := "" + seen := map[string]bool{} + for _, volume := range actual { + if seen[volume.Name] { + return "", false + } + seen[volume.Name] = true + if names[volume.Name] { + continue + } + projection := volume.Projected + if serviceAccountVolume != "" || !strings.HasPrefix(volume.Name, "kube-api-access-") || projection == nil || len(projection.Sources) != 3 || !reflect.DeepEqual(volume.VolumeSource, corev1.VolumeSource{Projected: projection}) { + return "", false + } + serviceAccountVolume = volume.Name + foundToken, foundCA, foundNamespace := false, false, false + for _, source := range projection.Sources { + switch { + case source.ServiceAccountToken != nil: + token := source.ServiceAccountToken + if foundToken || token.Path != "token" || !reflect.DeepEqual(source, corev1.VolumeProjection{ServiceAccountToken: token}) { + return "", false + } + foundToken = true + case source.ConfigMap != nil: + config := source.ConfigMap + if foundCA || config.Name != "kube-root-ca.crt" || len(config.Items) != 1 || config.Items[0].Key != "ca.crt" || config.Items[0].Path != "ca.crt" || !reflect.DeepEqual(source, corev1.VolumeProjection{ConfigMap: config}) { + return "", false + } + foundCA = true + case source.DownwardAPI != nil: + info := source.DownwardAPI + if foundNamespace || len(info.Items) != 1 || info.Items[0].Path != "namespace" || info.Items[0].FieldRef == nil || info.Items[0].FieldRef.FieldPath != "metadata.namespace" || info.Items[0].ResourceFieldRef != nil || !reflect.DeepEqual(source, corev1.VolumeProjection{DownwardAPI: info}) { + return "", false + } + foundNamespace = true + default: + return "", false + } + } + if !foundToken || !foundCA || !foundNamespace { + return "", false + } + } + return serviceAccountVolume, true +} + +func readV2PodReport(client kubeclient.Interface) v2ReportLogReader { + return func(ctx context.Context, namespace, pod string) ([]byte, error) { + limit := int64(maxV2ReportBytes + 1) + stream, err := client.CoreV1().Pods(namespace).GetLogs(pod, &corev1.PodLogOptions{Container: consts.ContainerNameOutput, LimitBytes: &limit}).Stream(ctx) + if err != nil { + return nil, err + } + defer stream.Close() + return io.ReadAll(io.LimitReader(stream, limit)) + } +} + +// Bind the admitted Create response to this invocation's requested inputs. +// Otherwise an admission rewrite could lower the requested hugepage check or +// replace the fresh output volume before either inventory sample is taken. +func validateV2CreatedDaemonSet(request, created *appsv1.DaemonSet, operation string) error { + if created == nil || created.UID == "" || created.Generation <= 0 || created.Name != request.Name || created.Namespace != request.Namespace { + return fmt.Errorf("V2 DaemonSet creation returned no identified receipt") + } + left, right := request.Spec.Template.Spec, created.Spec.Template.Spec + if !validV2CheckerTemplate(right, operation) || !reflect.DeepEqual(request.Spec.Selector, created.Spec.Selector) || !reflect.DeepEqual(left.NodeSelector, right.NodeSelector) || !reflect.DeepEqual(left.Affinity, right.Affinity) || !reflect.DeepEqual(left.Tolerations, right.Tolerations) || left.NodeName != right.NodeName || left.ServiceAccountName != right.ServiceAccountName || left.HostPID != right.HostPID || left.HostNetwork != right.HostNetwork || left.HostIPC != right.HostIPC || len(left.InitContainers) != len(right.InitContainers) || len(left.Containers) != len(right.Containers) || len(right.EphemeralContainers) != 0 || len(left.Volumes) != len(right.Volumes) { + return fmt.Errorf("V2 DaemonSet admission changed required execution or node coverage") + } + for index, container := range left.InitContainers { + if !sameV2ContainerExecution(container, right.InitContainers[index], "") { + return fmt.Errorf("V2 DaemonSet admission changed checker inputs") + } + } + for index, container := range left.Containers { + if !sameV2ContainerExecution(container, right.Containers[index], "") { + return fmt.Errorf("V2 DaemonSet admission changed companion execution") + } + } + for index, volume := range left.Volumes { + actual := right.Volumes[index].DeepCopy() + expected := volume.DeepCopy() + for _, item := range []*corev1.Volume{expected, actual} { + if item.HostPath != nil && item.HostPath.Type != nil && *item.HostPath.Type == corev1.HostPathUnset { + item.HostPath.Type = nil + } + } + if !reflect.DeepEqual(expected, actual) { + return fmt.Errorf("V2 DaemonSet admission changed the host or output volume") + } + } + return nil +} diff --git a/pkg/remote/preflight/checker_collection_v2_test.go b/pkg/remote/preflight/checker_collection_v2_test.go new file mode 100644 index 000000000..f95e698a4 --- /dev/null +++ b/pkg/remote/preflight/checker_collection_v2_test.go @@ -0,0 +1,270 @@ +package preflight + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" +) + +func TestV2ReportRejectsMissingAmbiguousAndPartialEvidence(t *testing.T) { + for _, input := range []string{ + ``, `{}`, `{"dataEngineMode":"v2-only","log":null}`, + `{"dataEngineMode":"v2-only","log":{}}`, + `{"dataEngineMode":"v1-capable","log":{"info":["ok"]}}`, + `{"dataEngineMode":"v2-only","dataEngineMode":"v2-only","log":{"info":["ok"]}}`, + `{"dataEngineMode":"v2-only","log":{"info":["ok"],"info":["ok"]}}`, + `{"dataEngineMode":"v2-only","log":{"info":["ok"],"unknown":true}}`, + `{"dataEngineMode":"v2-only","DataEngineMode":"v2-only","log":{"info":["ok"]}}`, + `{"dataEngineMode":"v2-only","log":{"Info":["ok"]}}`, + `{"dataEngineMode":"v2-only","log":{"info":["ok"]}} {}`, + `{"dataEngineMode":"v2-only","log":{"info":["ok"]}`, + strings.Repeat(" ", maxV2ReportBytes+1), + } { + if _, err := decodeV2Report([]byte(input)); err == nil { + t.Fatal("accepted missing, ambiguous, partial or unbounded V2 report") + } + } + for _, input := range []string{ + `{"dataEngineMode":"v2-only","log":{"info":["qualified"]}}`, + `{"dataEngineMode":"v2-only","log":{"info":["checks ran"],"error":["NVMe missing"]}}`, + } { + if _, err := decodeV2Report([]byte(input)); err != nil { + t.Fatalf("lost complete report: %v", err) + } + } +} + +type v2CollectionFixture struct { + mu sync.Mutex + ds *appsv1.DaemonSet + pods []corev1.Pod + paginate, failList bool +} + +func newV2CollectionFixture(t *testing.T) (*v2CollectionFixture, *kubeclient.Clientset, *appsv1.DaemonSet) { + t.Helper() + checker := Checker{CheckerCmdOptions: CheckerCmdOptions{DataEngineMode: string(types.DataEngineModeV2Only), GlobalCmdOptions: types.GlobalCmdOptions{Namespace: "test", Image: "fixture/cli:current"}, HugePageSize: 2048}, appName: "checker"} + ds := checker.newDaemonSet() + ds.UID, ds.Generation, ds.ResourceVersion = "ds-uid", 1, "1" + ds.Status = appsv1.DaemonSetStatus{ObservedGeneration: 1, DesiredNumberScheduled: 1, CurrentNumberScheduled: 1, UpdatedNumberScheduled: 1} + // The expected object is the API Create response, not the pre-serialization + // request: omitted empty slices have already become nil on that response. + encoded, err := json.Marshal(ds) + if err != nil { + t.Fatal(err) + } + var admitted appsv1.DaemonSet + if err := json.Unmarshal(encoded, &admitted); err != nil { + t.Fatal(err) + } + ds = &admitted + pod := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "checker-node", Namespace: "test", UID: "pod-uid", Labels: map[string]string{"app": "checker"}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "DaemonSet", Name: ds.Name, UID: ds.UID, Controller: ptr.To(true)}}}, Spec: *ds.Spec.Template.Spec.DeepCopy()} + pod.Spec.NodeName = "node-1" + for index, name := range []string{consts.ContainerNameInit, consts.ContainerNameOutput} { + start := metav1.NewTime(time.Unix(100+int64(index)*2, 0)) + end := metav1.NewTime(start.Add(time.Second)) + id := fmt.Sprintf("containerd://instance-%d", index) + pod.Status.InitContainerStatuses = append(pod.Status.InitContainerStatuses, corev1.ContainerStatus{Name: name, Image: ds.Spec.Template.Spec.InitContainers[index].Image, ImageID: "sha256:fixture", ContainerID: id, State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0, StartedAt: start, FinishedAt: end, ContainerID: id}}}) + } + fixture := &v2CollectionFixture{ds: ds.DeepCopy(), pods: []corev1.Pod{pod}} + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + fixture.mu.Lock() + defer fixture.mu.Unlock() + response.Header().Set("Content-Type", "application/json") + var object interface{} + switch request.URL.Path { + case "/apis/apps/v1/namespaces/test/daemonsets/checker": + object = fixture.ds + case "/api/v1/namespaces/test/pods": + if request.URL.Query().Get("labelSelector") != "" { + t.Error("V2 collector used an incomplete label-filtered inventory") + } + if fixture.failList { + http.Error(response, "inventory unavailable", http.StatusForbidden) + return + } + list := &corev1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "10"}, Items: fixture.pods} + if fixture.paginate && request.URL.Query().Get("continue") == "" { + list.Items = nil + list.Continue = "next" + } + object = list + default: + t.Errorf("unexpected API request %s", request.URL.Path) + http.NotFound(response, request) + return + } + if err := json.NewEncoder(response).Encode(object); err != nil { + t.Error(err) + } + })) + t.Cleanup(server.Close) + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + return fixture, client, ds +} + +func TestV2CollectorBindsCompleteCurrentExecution(t *testing.T) { + for _, test := range []struct { + name string + before, after func(*v2CollectionFixture) + logError bool + valid bool + }{ + {name: "complete", valid: true}, + {name: "native-service-account-projection", before: addV2FixtureServiceAccount, valid: true}, + {name: "complete-paginated", before: func(f *v2CollectionFixture) { f.paginate = true }, valid: true}, + {name: "missing-pod", before: func(f *v2CollectionFixture) { f.pods = nil }}, + {name: "lost-labels", before: func(f *v2CollectionFixture) { f.pods[0].Labels = nil }}, + {name: "foreign-controller", before: func(f *v2CollectionFixture) { f.pods[0].OwnerReferences[0].UID = "foreign" }}, + {name: "noncontroller-owner", before: func(f *v2CollectionFixture) { f.pods[0].OwnerReferences[0].Controller = ptr.To(false) }}, + {name: "terminating", before: func(f *v2CollectionFixture) { now := metav1.Now(); f.pods[0].DeletionTimestamp = &now }}, + {name: "duplicate-node", before: func(f *v2CollectionFixture) { + p := f.pods[0].DeepCopy() + p.Name = "duplicate" + p.UID = "other" + f.pods = append(f.pods, *p) + f.ds.Status.DesiredNumberScheduled = 2 + f.ds.Status.CurrentNumberScheduled = 2 + f.ds.Status.UpdatedNumberScheduled = 2 + }}, + {name: "unbound-executable", before: func(f *v2CollectionFixture) { f.pods[0].Spec.InitContainers[0].Command = []string{"old-checker"} }}, + {name: "reporter-shadowed-output", before: func(f *v2CollectionFixture) { + f.pods[0].Spec.Volumes = append(f.pods[0].Spec.Volumes, corev1.Volume{Name: "stale", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "stale"}}}}) + f.pods[0].Spec.InitContainers[1].VolumeMounts = append(f.pods[0].Spec.InitContainers[1].VolumeMounts, corev1.VolumeMount{Name: "stale", MountPath: "/shared/output.json", SubPath: "report.json"}) + }}, + {name: "service-account-shadowed-output", before: func(f *v2CollectionFixture) { + addV2FixtureServiceAccount(f) + f.pods[0].Spec.InitContainers[1].VolumeMounts[1].MountPath = "/shared/output.json" + }}, + {name: "failed-checker", before: func(f *v2CollectionFixture) { f.pods[0].Status.InitContainerStatuses[0].State.Terminated.ExitCode = 1 }}, + {name: "missing-container-id", before: func(f *v2CollectionFixture) { + f.pods[0].Status.InitContainerStatuses[0].State.Terminated.ContainerID = "" + }}, + {name: "early-reporter", before: func(f *v2CollectionFixture) { + f.pods[0].Status.InitContainerStatuses[1].State.Terminated.StartedAt = metav1.NewTime(time.Unix(99, 0)) + }}, + {name: "list-error", before: func(f *v2CollectionFixture) { f.failList = true }}, + {name: "log-error", logError: true}, + {name: "replacement-pod", after: func(f *v2CollectionFixture) { f.pods[0].UID = k8stypes.UID("replacement") }}, + {name: "new-container-attempt", after: func(f *v2CollectionFixture) { f.pods[0].Status.InitContainerStatuses[0].RestartCount++ }}, + {name: "changed-spec", after: func(f *v2CollectionFixture) { f.ds.Spec.Template.Spec.InitContainers[0].Image = "other" }}, + {name: "inventory-disappears", after: func(f *v2CollectionFixture) { f.pods = nil }}, + } { + t.Run(test.name, func(t *testing.T) { + fixture, client, ds := newV2CollectionFixture(t) + if test.before != nil { + test.before(fixture) + } + output, err := collectV2Reports(t.Context(), client, ds, func(context.Context, string, string) ([]byte, error) { + if test.logError { + return nil, fmt.Errorf("logs unavailable") + } + if test.after != nil { + fixture.mu.Lock() + test.after(fixture) + fixture.mu.Unlock() + } + return []byte(`{"dataEngineMode":"v2-only","log":{"info":["checks completed"]}}`), nil + }) + if (err == nil) != test.valid || (test.valid && !strings.Contains(output, "node-1:")) || (!test.valid && output != "") { + t.Fatalf("incorrect execution qualification: output=%q err=%v", output, err) + } + }) + } +} + +func addV2FixtureServiceAccount(f *v2CollectionFixture) { + name := "kube-api-access-abcde" + f.pods[0].Spec.Volumes = append(f.pods[0].Spec.Volumes, corev1.Volume{Name: name, VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{DefaultMode: ptr.To(int32(420)), Sources: []corev1.VolumeProjection{ + {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Path: "token", ExpirationSeconds: ptr.To(int64(3607))}}, + {ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "kube-root-ca.crt"}, Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}}}, + {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}}, + }}}}) + for index := range f.pods[0].Spec.InitContainers { + f.pods[0].Spec.InitContainers[index].VolumeMounts = append(f.pods[0].Spec.InitContainers[index].VolumeMounts, corev1.VolumeMount{Name: name, ReadOnly: true, MountPath: "/var/run/secrets/kubernetes.io/serviceaccount"}) + } + for index := range f.pods[0].Spec.Containers { + f.pods[0].Spec.Containers[index].VolumeMounts = append(f.pods[0].Spec.Containers[index].VolumeMounts, corev1.VolumeMount{Name: name, ReadOnly: true, MountPath: "/var/run/secrets/kubernetes.io/serviceaccount"}) + } +} + +func TestV2LogReadUsesTheFixedBoundedReporter(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + if r.URL.Path != "/api/v1/namespaces/test/pods/checker/log" || query.Get("container") != consts.ContainerNameOutput || query.Get("limitBytes") != fmt.Sprint(maxV2ReportBytes+1) || query.Get("follow") == "true" || query.Get("previous") == "true" || query.Get("timestamps") == "true" || query.Get("tailLines") != "" { + t.Errorf("unqualified log request %s", r.URL) + } + _, _ = w.Write([]byte(strings.Repeat("x", maxV2ReportBytes+100))) + })) + t.Cleanup(server.Close) + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + data, err := readV2PodReport(client)(t.Context(), "test", "checker") + if err != nil || len(data) != maxV2ReportBytes+1 { + t.Fatalf("log stream not bounded: length=%d err=%v", len(data), err) + } + if _, err := decodeV2Report(data); err == nil { + t.Fatal("oversized log qualified") + } +} + +func TestV2CollectorReturnsFullReportWithFailedQualification(t *testing.T) { + _, client, ds := newV2CollectionFixture(t) + output, err := collectV2Reports(t.Context(), client, ds, func(context.Context, string, string) ([]byte, error) { + return []byte(`{"dataEngineMode":"v2-only","log":{"info":["checks completed"],"error":["NVMe missing"]}}`), nil + }) + if err == nil || !strings.Contains(output, "NVMe missing") || !strings.Contains(output, "node-1:") { + t.Fatalf("failed qualification lost its complete report: %q %v", output, err) + } +} + +func TestV2AdmissionCannotLowerRequestedChecksOrReplaceOutput(t *testing.T) { + _, _, created := newV2CollectionFixture(t) + request := created.DeepCopy() + request.Spec.Template.Spec.InitContainers[1].Env = []corev1.EnvVar{} + created.Spec.Template.Spec.Volumes[0].HostPath.Type = ptr.To(corev1.HostPathUnset) + if err := validateV2CreatedDaemonSet(request, created, consts.SubCmdCheck); err != nil { + t.Fatalf("native defaults were mistaken for an execution rewrite: %v", err) + } + for _, mutate := range []func(*appsv1.DaemonSet){ + func(ds *appsv1.DaemonSet) { + for index, variable := range ds.Spec.Template.Spec.InitContainers[0].Env { + if variable.Name == consts.EnvHugePageSize { + ds.Spec.Template.Spec.InitContainers[0].Env[index].Value = "2" + } + } + }, + func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Volumes[1].VolumeSource = corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "stale-report"}}} + }, + func(ds *appsv1.DaemonSet) { ds.Spec.Template.Spec.NodeSelector = map[string]string{"node": "other"} }, + } { + changed := created.DeepCopy() + mutate(changed) + if err := validateV2CreatedDaemonSet(request, changed, consts.SubCmdCheck); err == nil { + t.Fatal("admission rewrote the requested qualification") + } + } +} diff --git a/pkg/remote/preflight/data_engine_mode.go b/pkg/remote/preflight/data_engine_mode.go new file mode 100644 index 000000000..8f698845c --- /dev/null +++ b/pkg/remote/preflight/data_engine_mode.go @@ -0,0 +1,149 @@ +package preflight + +import ( + "fmt" + "math" + "regexp" + "strconv" + "strings" + "time" + + "github.com/longhorn/cli/pkg/types" +) + +func validateV2Hugepages(size int) error { + if size <= 0 || size%2 != 0 || int64(size) > math.MaxInt64/(1024*1024) { + return fmt.Errorf("v2-only requires a positive, bounded amount of 2MiB hugepages") + } + return nil +} + +func validateV2Driver(driver string) error { + if driver != "" && driver != "vfio-pci" && driver != "uio_pci_generic" { + return fmt.Errorf("unsupported v2-only userspace driver") + } + return nil +} + +func (options *CheckerCmdOptions) ValidateDataEngineMode() error { + mode, err := types.ParseDataEngineMode(options.DataEngineMode) + if err != nil { + return err + } + options.DataEngineMode = string(mode) + if mode != types.DataEngineModeV2Only { + return nil + } + if err := validateV2Hugepages(options.HugePageSize); err != nil { + return err + } + if options.UserspaceDriver != "" && options.UserspaceDriver != "vfio_pci" && options.UserspaceDriver != "uio_pci_generic" { + return fmt.Errorf("unsupported v2-only userspace kernel module") + } + return nil +} + +func (options *InstallerCmdOptions) ValidateDataEngineMode() error { + mode, err := types.ParseDataEngineMode(options.DataEngineMode) + if err != nil { + return err + } + options.DataEngineMode = string(mode) + if mode != types.DataEngineModeV2Only { + return nil + } + _, _, err = options.V2SPDKSetupEnvironment() + return err +} + +var v2PCIAddress = regexp.MustCompile(`^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[01][0-9a-fA-F]\.[0-7]$`) + +func normalizeV2PCIAllowed(value string) (string, error) { + if value == "" || value == "none" { + return "none", nil + } + addresses := strings.FieldsFunc(value, func(r rune) bool { return r == ',' || r == ' ' || r == '\t' }) + if len(addresses) == 0 { + return "", fmt.Errorf("v2-only PCI selection is unresolved") + } + seen := map[string]bool{} + for index, address := range addresses { + address = strings.ToLower(address) + if !v2PCIAddress.MatchString(address) || seen[address] { + return "", fmt.Errorf("v2-only requires unique full PCI addresses or none") + } + seen[address], addresses[index] = true, address + } + return strings.Join(addresses, " "), nil +} + +// V2SPDKSetupEnvironment owns the pinned setup script's relevant inputs. +// It does not establish host qualification: callers must bind the script and +// host prerequisites before writes, then verify actual allocation afterward. +func (options InstallerCmdOptions) V2SPDKSetupEnvironment() ([]string, string, error) { + mode, err := types.ParseDataEngineMode(options.DataEngineMode) + if err != nil || mode != types.DataEngineModeV2Only { + return nil, "", fmt.Errorf("v2-only setup requires an explicit capability") + } + if options.OperatingSystem != "" { + return nil, "", fmt.Errorf("v2-only installation supports qualified package-manager hosts; COS and explicit OS overrides are unsupported") + } + if err := validateV2Hugepages(options.HugePageSize); err != nil { + return nil, "", err + } + if err := validateV2Driver(options.DriverOverride); err != nil { + return nil, "", err + } + if options.RestartKubelet { + window, err := time.ParseDuration(options.RestartKubeletWindow) + if err != nil || window < 0 { + return nil, "", fmt.Errorf("v2-only requires a valid nonnegative kubelet restart window") + } + } + pciAllowed, err := normalizeV2PCIAllowed(options.AllowPci) + if err != nil { + return nil, "", err + } + node, seen := "0", map[string]bool{} + if options.SpdkOptions != "" { + for _, option := range strings.Split(options.SpdkOptions, ",") { + name, value, assigned := strings.Cut(option, "=") + if !assigned || seen[name] { + return nil, "", fmt.Errorf("invalid or duplicate v2-only SPDK option") + } + seen[name] = true + switch name { + case "HUGEMEM": + if value != strconv.Itoa(options.HugePageSize) { + return nil, "", fmt.Errorf("SPDK memory override contradicts v2-only hugepage intent") + } + case "HUGEPGSZ": + if value != "2048" { + return nil, "", fmt.Errorf("v2-only setup requires 2MiB hugepages") + } + case "PCI_ALLOWED": + selected, err := normalizeV2PCIAllowed(value) + if err != nil || selected != pciAllowed { + return nil, "", fmt.Errorf("SPDK PCI override contradicts v2-only selection") + } + case "DRIVER_OVERRIDE": + if value != options.DriverOverride { + return nil, "", fmt.Errorf("SPDK driver override contradicts v2-only selection") + } + case "HUGENODE": + number, err := strconv.ParseUint(value, 10, 31) + if err != nil || strconv.FormatUint(number, 10) != value { + return nil, "", fmt.Errorf("v2-only setup requires one canonical NUMA node number") + } + node = value + default: + return nil, "", fmt.Errorf("unsupported SPDK option in v2-only setup") + } + } + } + return []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "LC_ALL=C", + "HUGEMEM=" + strconv.Itoa(options.HugePageSize), "HUGEPGSZ=2048", "HUGENODE=" + node, + "PCI_ALLOWED=" + pciAllowed, "DRIVER_OVERRIDE=" + options.DriverOverride, "TARGET_USER=root", + }, node, nil +} diff --git a/pkg/remote/preflight/data_engine_mode_test.go b/pkg/remote/preflight/data_engine_mode_test.go new file mode 100644 index 000000000..fbe341ad1 --- /dev/null +++ b/pkg/remote/preflight/data_engine_mode_test.go @@ -0,0 +1,145 @@ +package preflight + +import ( + "reflect" + "strconv" + "strings" + "testing" +) + +func TestCheckerModeRequiresResolvedV2Prerequisites(t *testing.T) { + for _, test := range []struct { + name, mode, driver string + hugepages int + valid bool + }{ + {"default", "", "", 0, true}, + {"v1", "v1-capable", "", 0, true}, + {"v2", "v2-only", "", 2048, true}, + {"v2-vfio-module", "v2-only", "vfio_pci", 2048, true}, + {"v2-uio-module", "v2-only", "uio_pci_generic", 2048, true}, + {"vfio-driver-is-not-module", "v2-only", "vfio-pci", 2048, false}, + {"invalid-mode", "v2", "", 2048, false}, + {"missing-pages", "v2-only", "", 0, false}, + {"negative-pages", "v2-only", "", -2, false}, + {"unaligned-pages", "v2-only", "", 1, false}, + {"invalid-driver", "v2-only", "unknown", 2048, false}, + } { + t.Run(test.name, func(t *testing.T) { + options := CheckerCmdOptions{DataEngineMode: test.mode, HugePageSize: test.hugepages, UserspaceDriver: test.driver} + if err := options.ValidateDataEngineMode(); (err == nil) != test.valid { + t.Fatalf("unexpected checker mode validation: %v", err) + } + if test.valid && options.DataEngineMode == "" { + t.Fatal("default capability was not normalized") + } + }) + } +} + +func TestV2SPDKEnvironmentBindsNativeMemoryPCIAndNUMAInputs(t *testing.T) { + options := InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048, AllowPci: "0000:00:01.0,0000:00:02.0", DriverOverride: "vfio-pci", SpdkOptions: "HUGENODE=1"} + environment, node, err := options.V2SPDKSetupEnvironment() + if err != nil || node != "1" { + t.Fatalf("unresolved setup profile: %v", err) + } + want := []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "LC_ALL=C", "HUGEMEM=2048", "HUGEPGSZ=2048", "HUGENODE=1", "PCI_ALLOWED=0000:00:01.0 0000:00:02.0", "DRIVER_OVERRIDE=vfio-pci", "TARGET_USER=root"} + if !reflect.DeepEqual(environment, want) { + t.Fatalf("setup environment does not bind the resolved options: %v", environment) + } + options.AllowPci, options.SpdkOptions = "", "" + environment, node, err = options.V2SPDKSetupEnvironment() + if err != nil || node != "0" || !strings.Contains(strings.Join(environment, "\n"), "PCI_ALLOWED=none\n") { + t.Fatal("empty PCI selection did not remain fenced, or native node-0 default changed") + } +} + +func TestRemoteCheckerBindsCapabilityToTheLocalExecutable(t *testing.T) { + for _, mode := range []string{"", "v1-capable", "v2-only"} { + for _, explicitSPDK := range []bool{false, true} { + checker := Checker{CheckerCmdOptions: CheckerCmdOptions{DataEngineMode: mode, HugePageSize: 2048, EnableSpdk: explicitSPDK}, appName: "checker"} + if err := checker.ValidateDataEngineMode(); err != nil { + t.Fatal(err) + } + pod := checker.newDaemonSet().Spec.Template.Spec + container := pod.InitContainers[0] + environment := map[string]string{} + for _, variable := range container.Env { + environment[variable.Name] = variable.Value + } + if environment["DATA_ENGINE_MODE"] != checker.DataEngineMode || environment["ENABLE_SPDK"] != strconv.FormatBool(explicitSPDK) { + t.Fatal("remote job lost capability or overwrote the explicit SPDK flag") + } + command := strings.Join(append(append([]string{}, container.Command...), container.Args...), " ") + if strings.Contains(command, "--data-engine-mode v2-only") != (mode == "v2-only") { + t.Fatal("V2-only job can silently run an older local executable, or default argv changed") + } + } + } +} + +func TestRemoteCheckerRejectsInvalidModeBeforeConnectionInitialization(t *testing.T) { + checker := Checker{CheckerCmdOptions: CheckerCmdOptions{DataEngineMode: "unknown"}} + if err := checker.Init(); err == nil || !strings.Contains(err.Error(), "data-engine-mode") { + t.Fatal("invalid capability reached connection initialization") + } +} + +func TestRemoteInstallerBindsV2ExecutableBeforeAnyMutation(t *testing.T) { + installer := Installer{InstallerCmdOptions: InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}, appName: "installer"} + if err := installer.ValidateDataEngineMode(); err != nil { + t.Fatal(err) + } + container := installer.newDaemonSetForPackageManager().Spec.Template.Spec.InitContainers[0] + if !strings.Contains(strings.Join(container.Command, " "), "--data-engine-mode v2-only") { + t.Fatal("V2 installer may silently execute an older V1-capable image") + } + if err := installer.InstallByContainerOptimizedOS(); err == nil { + t.Fatal("V2 installer reached unsupported COS mutations") + } +} + +func TestRemoteInstallerRejectsInvalidModeBeforeConnectionInitialization(t *testing.T) { + installer := Installer{InstallerCmdOptions: InstallerCmdOptions{DataEngineMode: "unknown"}} + if err := installer.Init(); err == nil || !strings.Contains(err.Error(), "data-engine-mode") { + t.Fatal("invalid mode reached remote installer initialization") + } +} + +func TestInstallerModeRejectsUnqualifiedOrConflictingSetupBeforeWrites(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*InstallerCmdOptions) + valid bool + }{ + {"compatible", func(*InstallerCmdOptions) {}, true}, + {"invalid-mode", func(o *InstallerCmdOptions) { o.DataEngineMode = "v2" }, false}, + {"cos", func(o *InstallerCmdOptions) { o.OperatingSystem = "cos" }, false}, + {"talos", func(o *InstallerCmdOptions) { o.OperatingSystem = "talos" }, false}, + {"zero-pages", func(o *InstallerCmdOptions) { o.HugePageSize = 0 }, false}, + {"unaligned-pages", func(o *InstallerCmdOptions) { o.HugePageSize = 3 }, false}, + {"bad-window", func(o *InstallerCmdOptions) { o.RestartKubelet, o.RestartKubeletWindow = true, "invalid" }, false}, + {"negative-window", func(o *InstallerCmdOptions) { o.RestartKubelet, o.RestartKubeletWindow = true, "-1m" }, false}, + {"invalid-driver", func(o *InstallerCmdOptions) { o.DriverOverride = "none" }, false}, + {"invalid-pci", func(o *InstallerCmdOptions) { o.AllowPci = "$(untrusted)" }, false}, + {"conflicting-pages", func(o *InstallerCmdOptions) { o.SpdkOptions = "HUGEMEM=0" }, false}, + {"conflicting-size", func(o *InstallerCmdOptions) { o.SpdkOptions = "HUGEPGSZ=1048576" }, false}, + {"override-page-count", func(o *InstallerCmdOptions) { o.SpdkOptions = "NRHUGE=0" }, false}, + {"unknown-option", func(o *InstallerCmdOptions) { o.SpdkOptions = "PERSIST_HUGE=yes" }, false}, + {"duplicate-option", func(o *InstallerCmdOptions) { o.SpdkOptions = "HUGEMEM=2048,HUGEMEM=2048" }, false}, + {"eval-node", func(o *InstallerCmdOptions) { o.SpdkOptions = "HUGENODE=$(untrusted)" }, false}, + {"resolved-options", func(o *InstallerCmdOptions) { o.SpdkOptions = "HUGEMEM=2048,HUGEPGSZ=2048,HUGENODE=0" }, true}, + } { + t.Run(test.name, func(t *testing.T) { + options := InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048, AllowPci: "none"} + test.mutate(&options) + if err := options.ValidateDataEngineMode(); (err == nil) != test.valid { + t.Fatalf("unexpected installer profile validation: %v", err) + } + }) + } + options := InstallerCmdOptions{SpdkOptions: "legacy-unparsed", OperatingSystem: "cos"} + if err := options.ValidateDataEngineMode(); err != nil || options.DataEngineMode != "v1-capable" { + t.Fatal("default installation behavior changed") + } +} diff --git a/pkg/remote/preflight/installer.go b/pkg/remote/preflight/installer.go index 1ba175422..02515a0ec 100644 --- a/pkg/remote/preflight/installer.go +++ b/pkg/remote/preflight/installer.go @@ -1,9 +1,12 @@ package preflight import ( + "context" + "crypto/rand" "encoding/json" "path/filepath" "reflect" + "strings" "time" "github.com/pkg/errors" @@ -15,6 +18,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeclient "k8s.io/client-go/kubernetes" @@ -34,13 +38,16 @@ type Installer struct { kubeClient *kubeclient.Clientset - appName string // App name of the DaemonSet. + appName string // App name of the DaemonSet. + createdDaemonSet *appsv1.DaemonSet + createdRBAC *v2RBACReceipt } // InstallerCmdOptions holds the options for the command. type InstallerCmdOptions struct { types.GlobalCmdOptions + DataEngineMode string OperatingSystem string UpdatePackages bool @@ -55,6 +62,13 @@ type InstallerCmdOptions struct { // Init initializes the Installer. func (remote *Installer) Init() error { + // Cleanup-only commands do not require host setup options. Validate the + // capability here and validate the complete installation profile in Run. + mode, err := types.ParseDataEngineMode(remote.DataEngineMode) + if err != nil { + return err + } + remote.DataEngineMode = string(mode) kubeClient, err := kubeutils.NewKubeClient("", remote.KubeConfigPath) if err != nil { return err @@ -68,6 +82,9 @@ func (remote *Installer) Init() error { default: remote.appName = consts.AppNamePreflightInstaller } + if mode == types.DataEngineModeV2Only { + remote.appName += "-" + strings.ToLower(rand.Text()) + } return nil } @@ -76,6 +93,9 @@ func (remote *Installer) Init() error { // It checks if the operating system is specified, and installs the dependencies accordingly. // If the operating system is not specified, it installs the dependencies with package manager. func (remote *Installer) Run() (string, error) { + if err := remote.ValidateDataEngineMode(); err != nil { + return "", err + } operatingSystem := consts.OperatingSystem(remote.OperatingSystem) switch operatingSystem { case consts.OperatingSystemContainerOptimizedOS: @@ -99,7 +119,14 @@ func (remote *Installer) Run() (string, error) { Verbs: []string{"get"}, }, } - err := kubeutils.CreateRbac(remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + var err error + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + remote.createdRBAC, err = createV2RBAC(ctx, remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + } else { + err = kubeutils.CreateRbac(remote.kubeClient, remote.Namespace, remote.appName, rbacRules) + } if err != nil { return "", err } @@ -111,7 +138,7 @@ func (remote *Installer) Run() (string, error) { } output, err := remote.InstallByPackageManager() if err != nil { - return "", errors.Wrapf(err, "failed to install dependencies with package manager") + return output, errors.Wrapf(err, "failed to install dependencies with package manager") } logrus.Info("Installed dependencies with package manager") @@ -122,6 +149,20 @@ func (remote *Installer) Run() (string, error) { // Cleanup deletes the DaemonSet created for the preflight install when it's installed with package manager. func (remote *Installer) Cleanup() error { var resultErr error + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if remote.createdDaemonSet != nil { + uid := remote.createdDaemonSet.UID + if uid == "" { + return errors.New("refusing V2 installer cleanup without its created UID") + } + if err := remote.kubeClient.AppsV1().DaemonSets(remote.createdDaemonSet.Namespace).Delete(ctx, remote.createdDaemonSet.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return cleanupV2RBAC(ctx, remote.kubeClient, remote.createdRBAC) + } if err := commonkube.DeleteDaemonSet(remote.kubeClient, remote.Namespace, remote.appName); err != nil { resultErr = errors.Wrap(err, "failed to delete DaemonSet") @@ -141,6 +182,12 @@ func (remote *Installer) Cleanup() error { // InstallByContainerOptimizedOS installs the dependencies on Container Optimized OS. // It creates a ConfigMap and a DaemonSet. Then it waits for the DaemonSet to be ready. func (remote *Installer) InstallByContainerOptimizedOS() error { + if err := remote.ValidateDataEngineMode(); err != nil { + return err + } + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + return errors.New("V2-only installation is not qualified for Container-Optimized OS") + } newConfigMap := remote.newConfigMapForContainerOptimizedOS() _, err := commonkube.CreateConfigMap(remote.kubeClient, newConfigMap) if err != nil { @@ -172,14 +219,30 @@ func (remote *Installer) InstallByContainerOptimizedOS() error { // - Successfully probed module dm_crypt // - Successfully started service iscsid func (remote *Installer) InstallByPackageManager() (string, error) { + if err := remote.ValidateDataEngineMode(); err != nil { + return "", err + } newDaemonSet, err := kubeutils.PrepareDaemonSet(remote.newDaemonSetForPackageManager(), remote.kubeClient, remote.NodeSelector, remote.ImagePullSecret, remote.Tolerations) if err != nil { return "", err } - daemonSet, err := commonkube.CreateDaemonSet(remote.kubeClient, newDaemonSet) + var daemonSet *appsv1.DaemonSet + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + daemonSet, err = remote.kubeClient.AppsV1().DaemonSets(remote.Namespace).Create(ctx, newDaemonSet, metav1.CreateOptions{}) + } else { + daemonSet, err = commonkube.CreateDaemonSet(remote.kubeClient, newDaemonSet) + } if err != nil { return "", err } + remote.createdDaemonSet = daemonSet.DeepCopy() + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + if err := validateV2CreatedDaemonSet(newDaemonSet, daemonSet, consts.SubCmdInstall); err != nil { + return "", err + } + } err = kubeutils.MonitorDaemonSetContainer(remote.kubeClient, daemonSet, consts.ContainerNameInit, kubeutils.WaitForDaemonSetContainersExit, ptr.To(consts.ContainerConditionMaxTolerationLong)) if err != nil { @@ -191,6 +254,9 @@ func (remote *Installer) InstallByPackageManager() (string, error) { return "", err } + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + return collectV2OperationReports(context.Background(), remote.kubeClient, daemonSet, readV2PodReport(remote.kubeClient), consts.SubCmdInstall) + } podCollections, err := kubeutils.GetDaemonSetPodCollections(remote.kubeClient, daemonSet, consts.ContainerNameOutput, false, false, nil) if err != nil { return "", err @@ -448,6 +514,10 @@ func (remote *Installer) newDaemonSetForContainerOptimizedOS() *appsv1.DaemonSet // newDaemonSetForPackageManager prepares a DaemonSet for installing dependencies with the package manager. func (remote *Installer) newDaemonSetForPackageManager() *appsv1.DaemonSet { + command := []string{consts.CmdLonghornctlLocal, consts.SubCmdInstall, consts.SubCmdPreflight} + if remote.DataEngineMode == string(types.DataEngineModeV2Only) { + command = append(command, "--"+consts.CmdOptDataEngineMode, remote.DataEngineMode) + } outputFilePath := filepath.Join(consts.VolumeMountSharedDirectory, consts.FileNameOutputJSON) return &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ @@ -479,8 +549,9 @@ func (remote *Installer) newDaemonSetForPackageManager() *appsv1.DaemonSet { { Name: consts.ContainerNameInit, Image: utils.BuildImageName(remote.Image, remote.ImageRegistry), - Command: []string{consts.CmdLonghornctlLocal, consts.SubCmdInstall, consts.SubCmdPreflight}, + Command: command, Env: []corev1.EnvVar{ + {Name: consts.EnvDataEngineMode, Value: remote.DataEngineMode}, { Name: consts.EnvLogLevel, Value: remote.LogLevel, diff --git a/pkg/remote/preflight/rbac_v2.go b/pkg/remote/preflight/rbac_v2.go new file mode 100644 index 000000000..7e97b04ed --- /dev/null +++ b/pkg/remote/preflight/rbac_v2.go @@ -0,0 +1,76 @@ +package preflight + +import ( + "context" + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeclient "k8s.io/client-go/kubernetes" +) + +type v2RBACReceipt struct { + Namespace, Name string + ServiceAccount, ClusterRole, ClusterRoleBinding k8stypes.UID +} + +// Never adopt an AlreadyExists resource, or delete an object whose creation +// did not return its UID. Partial receipts retain ownership of partial creates. +func createV2RBAC(ctx context.Context, client kubeclient.Interface, namespace, name string, rules []rbacv1.PolicyRule) (*v2RBACReceipt, error) { + receipt := &v2RBACReceipt{Namespace: namespace, Name: name} + account, err := client.CoreV1().ServiceAccounts(namespace).Create(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}}, metav1.CreateOptions{}) + if err != nil { + return receipt, err + } + if account.UID == "" || account.Name != name || account.Namespace != namespace { + return receipt, fmt.Errorf("V2 checker service account creation returned no identified receipt") + } + receipt.ServiceAccount = account.UID + role, err := client.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: name}, Rules: rules}, metav1.CreateOptions{}) + if err != nil { + return receipt, err + } + if role.UID == "" || role.Name != name { + return receipt, fmt.Errorf("V2 checker role creation returned no identified receipt") + } + receipt.ClusterRole = role.UID + binding, err := client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: name}, RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: name}, Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: name, Namespace: namespace}}}, metav1.CreateOptions{}) + if err != nil { + return receipt, err + } + if binding.UID == "" || binding.Name != name { + return receipt, fmt.Errorf("V2 checker binding creation returned no identified receipt") + } + receipt.ClusterRoleBinding = binding.UID + return receipt, nil +} + +func cleanupV2RBAC(ctx context.Context, client kubeclient.Interface, receipt *v2RBACReceipt) error { + if receipt == nil { + return nil + } + var result error + remove := func(uid k8stypes.UID, deleteObject func(metav1.DeleteOptions) error) { + if uid == "" { + return + } + err := deleteObject(metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + if err != nil && !apierrors.IsNotFound(err) { + result = errors.Join(result, err) + } + } + remove(receipt.ClusterRoleBinding, func(options metav1.DeleteOptions) error { + return client.RbacV1().ClusterRoleBindings().Delete(ctx, receipt.Name, options) + }) + remove(receipt.ClusterRole, func(options metav1.DeleteOptions) error { + return client.RbacV1().ClusterRoles().Delete(ctx, receipt.Name, options) + }) + remove(receipt.ServiceAccount, func(options metav1.DeleteOptions) error { + return client.CoreV1().ServiceAccounts(receipt.Namespace).Delete(ctx, receipt.Name, options) + }) + return result +} diff --git a/pkg/remote/preflight/rbac_v2_test.go b/pkg/remote/preflight/rbac_v2_test.go new file mode 100644 index 000000000..d5d739399 --- /dev/null +++ b/pkg/remote/preflight/rbac_v2_test.go @@ -0,0 +1,92 @@ +package preflight + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +func TestV2RBACNeverAdoptsOrDeletesUnownedObjects(t *testing.T) { + for _, test := range []struct { + name, failResource string + replacement bool + created int + }{ + {"complete", "", false, 3}, + {"account-exists", "serviceaccounts", false, 0}, + {"role-exists", "clusterroles", false, 1}, + {"binding-exists", "clusterrolebindings", false, 2}, + {"replacement-conflict", "", true, 3}, + } { + t.Run(test.name, func(t *testing.T) { + var mu sync.Mutex + deletes := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("Content-Type", "application/json") + parts := strings.Split(r.URL.Path, "/") + resource := parts[len(parts)-1] + if r.Method == http.MethodDelete { + resource = parts[len(parts)-2] + } + if r.Method == http.MethodPost && resource == test.failResource { + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(&metav1.Status{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, Status: "Failure", Reason: metav1.StatusReasonAlreadyExists, Code: 409}) + return + } + if r.Method == http.MethodDelete { + deletes++ + var options metav1.DeleteOptions + if err := json.NewDecoder(r.Body).Decode(&options); err != nil || options.Preconditions == nil || options.Preconditions.UID == nil || string(*options.Preconditions.UID) != "owned-"+resource { + t.Errorf("cleanup missing created UID: %+v %v", options, err) + } + if test.replacement && resource == "clusterroles" { + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(&metav1.Status{Status: "Failure", Reason: metav1.StatusReasonConflict, Code: 409}) + return + } + _ = json.NewEncoder(w).Encode(&metav1.Status{Status: "Success"}) + return + } + if r.Method != http.MethodPost { + t.Errorf("unexpected ownership API method: %s", r.Method) + http.Error(w, "unexpected", 500) + return + } + var object map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&object); err != nil { + t.Error(err) + return + } + metadata := object["metadata"].(map[string]interface{}) + metadata["uid"] = "owned-" + resource + _ = json.NewEncoder(w).Encode(object) + })) + t.Cleanup(server.Close) + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL, ContentConfig: rest.ContentConfig{ContentType: "application/json", AcceptContentTypes: "application/json"}}) + if err != nil { + t.Fatal(err) + } + receipt, err := createV2RBAC(t.Context(), client, "test", "owned", nil) + if (err != nil) != (test.failResource != "") { + t.Fatalf("strict creation adopted existing resources: %v", err) + } + err = cleanupV2RBAC(t.Context(), client, receipt) + if (err != nil) != test.replacement { + t.Fatalf("cleanup hid replacement conflict: %v", err) + } + if deletes != test.created { + t.Fatal(fmt.Sprintf("deleted objects without successful creation receipts: got=%d want=%d", deletes, test.created)) + } + }) + } +} diff --git a/pkg/types/nodes.go b/pkg/types/nodes.go index 854caab14..719b21ad3 100644 --- a/pkg/types/nodes.go +++ b/pkg/types/nodes.go @@ -2,5 +2,6 @@ package types // NodeCollection represents a collection of nodes. type NodeCollection struct { - Log *LogCollection `json:"log,omitempty" yaml:"log,omitempty"` + Log *LogCollection `json:"log,omitempty" yaml:"log,omitempty"` + DataEngineMode string `json:"dataEngineMode,omitempty" yaml:"dataEngineMode,omitempty"` } diff --git a/pkg/utils/kubernetes/node.go b/pkg/utils/kubernetes/node.go index 1611cdc8d..a8087c609 100644 --- a/pkg/utils/kubernetes/node.go +++ b/pkg/utils/kubernetes/node.go @@ -13,7 +13,7 @@ import ( ) // GetHugePagesCapacity returns hugepages-2Mi capacity of current node -func GetHugePagesCapacity(kubeClient *kubeclient.Clientset) (*resource.Quantity, error) { +func GetHugePagesCapacity(kubeClient kubeclient.Interface) (*resource.Quantity, error) { currentNodeID := os.Getenv(consts.EnvCurrentNodeID) node, err := kubeClient.CoreV1().Nodes().Get(context.TODO(), currentNodeID, metav1.GetOptions{}) if err != nil { From 960960b6dd1bd6d1001e91da7e4d0db3f142976f Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 01:24:56 +0100 Subject: [PATCH 3/7] fix(preflight): preserve failed installs without automatic replay Issue longhorn/longhorn#14438 Signed-off-by: Nikolai Emil Damm Co-authored-by: Codex (GPT-6.1 Sol) --- README.md | 2 + cmd/local/subcmd/install.go | 6 ++ pkg/consts/environment.go | 1 + pkg/local/preflight/installer.go | 1 + .../preflight/installer_collection_v2.go | 82 +++++++++++++++++++ .../preflight/installer_collection_v2_test.go | 53 ++++++++++++ pkg/remote/preflight/checker.go | 13 +-- pkg/remote/preflight/checker_collection_v2.go | 19 ++++- .../preflight/checker_collection_v2_test.go | 12 +++ pkg/remote/preflight/data_engine_mode_test.go | 12 +++ pkg/remote/preflight/installer.go | 15 ++-- pkg/remote/preflight/rbac_v2_test.go | 57 ++++++++++++- 12 files changed, 259 insertions(+), 14 deletions(-) create mode 100644 pkg/local/preflight/installer_collection_v2.go create mode 100644 pkg/local/preflight/installer_collection_v2_test.go diff --git a/README.md b/README.md index 31c1659b9..594eb7745 100644 --- a/README.md +++ b/README.md @@ -71,3 +71,5 @@ An empty PCI selection is fenced as `none`. Explicit device selection requires u V2 remote runs use separate resource names and collect complete reports from the identified DaemonSet and completed container instances. Cleanup deletes only objects created by that invocation, with UID preconditions. This mode neither converts existing volumes nor enables the manager's separate V2-only installation capability. Retire V1 data before removing its host prerequisites. +Remote V2 installation records its invocation under `/var/lib/longhorn-cli/preflight-attempts` before changing host prerequisites. A Kubernetes retry or replacement Pod reports an error instead of repeating that invocation's host writes. The small attempt marker remains for audit and replay protection; assess any partial installation or required reboot before explicitly starting a new command. Failed and reboot-pending attempts are transported as error reports and the remote command exits nonzero. + diff --git a/cmd/local/subcmd/install.go b/cmd/local/subcmd/install.go index 8ad26846f..3205dffcb 100644 --- a/cmd/local/subcmd/install.go +++ b/cmd/local/subcmd/install.go @@ -42,6 +42,7 @@ These dependencies ensure your Kubernetes cluster meets the requirements for suc utils.CheckErr(err) } localInstaller.LogLevel = globalOpts.LogLevel + localInstaller.AttemptID = os.Getenv(consts.EnvPreflightAttemptID) if err := localInstaller.Init(); err != nil { utils.CheckErr(errors.Wrap(err, "Failed to initialize preflight installer")) @@ -50,6 +51,11 @@ These dependencies ensure your Kubernetes cluster meets the requirements for suc Run: func(cmd *cobra.Command, args []string) { if localInstaller.DataEngineMode == string(types.DataEngineModeV2Only) { + if os.Getenv(consts.EnvPreflightCollectionOnly) == "true" { + utils.CheckErr(localInstaller.RunForCollection()) + utils.CheckErr(localInstaller.Output()) + return + } utils.CheckErr(runAndOutputV2Preflight(&localInstaller)) return } diff --git a/pkg/consts/environment.go b/pkg/consts/environment.go index c0479364e..304732a1f 100644 --- a/pkg/consts/environment.go +++ b/pkg/consts/environment.go @@ -13,6 +13,7 @@ const ( EnvOutputFilePath = "OUTPUT_FILE_PATH" EnvDataEngineMode = "DATA_ENGINE_MODE" EnvPreflightCollectionOnly = "PREFLIGHT_COLLECTION_ONLY" + EnvPreflightAttemptID = "PREFLIGHT_ATTEMPT_ID" EnvLonghornDataDirectory = "LONGHORN_DATA_DIRECTORY" EnvLonghornNamespace = "LONGHORN_NAMESPACE" diff --git a/pkg/local/preflight/installer.go b/pkg/local/preflight/installer.go index 22e371cd9..a64c4ba9f 100644 --- a/pkg/local/preflight/installer.go +++ b/pkg/local/preflight/installer.go @@ -39,6 +39,7 @@ type Installer struct { logger *logrus.Entry OutputFilePath string + AttemptID string kubeClient *kubeclient.Clientset diff --git a/pkg/local/preflight/installer_collection_v2.go b/pkg/local/preflight/installer_collection_v2.go new file mode 100644 index 000000000..f65b98d21 --- /dev/null +++ b/pkg/local/preflight/installer_collection_v2.go @@ -0,0 +1,82 @@ +package preflight + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" +) + +var v2InstallAttemptID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// RunForCollection transports a failed or reboot-pending installation as a +// complete error report. The remote collector, not the init-container exit +// code, owns the verdict. A persistent per-invocation host marker prevents +// Kubernetes init retries or replacement Pods from repeating host writes. +func (local *Installer) RunForCollection() error { + if err := local.ValidateDataEngineMode(); err != nil { + return err + } + if local.DataEngineMode != string(types.DataEngineModeV2Only) || local.collection.Log == nil || local.packageManager == nil || local.kubeClient == nil { + return fmt.Errorf("V2 collection requires an initialized installer") + } + if local.OutputFilePath != filepath.Join(consts.VolumeMountSharedDirectory, consts.FileNameOutputJSON) || len(local.AttemptID) > 63 || !strings.HasPrefix(local.AttemptID, consts.AppNamePreflightInstaller+"-") || !v2InstallAttemptID.MatchString(local.AttemptID) { + return fmt.Errorf("V2 installer collection requires its identified invocation and shared output") + } + local.collection.DataEngineMode = string(types.DataEngineModeV2Only) + if err := local.qualifyV2SPDKHost(); err != nil { + local.recordV2InstallFailure(err) + return nil + } + directory := filepath.Join(consts.VolumeMountHostDirectory, "var/lib/longhorn-cli/preflight-attempts") + return runV2InstallAttempt(directory, local.AttemptID, local.Run, local.recordV2InstallFailure) +} + +func (local *Installer) recordV2InstallFailure(err error) { + local.collection.Log.Info = append(local.collection.Log.Info, "V2 prerequisite setup returned a failure report; readiness is not established") + local.collection.Log.Error = append(local.collection.Log.Error, err.Error()) +} + +func runV2InstallAttempt(directory, attemptID string, run func() error, recordFailure func(error)) error { + if err := claimV2InstallAttempt(directory, attemptID); err != nil { + recordFailure(err) + return nil + } + if err := run(); err != nil { + recordFailure(err) + } + return nil +} + +func claimV2InstallAttempt(directory, attemptID string) error { + if err := os.MkdirAll(directory, 0700); err != nil { + return err + } + info, err := os.Lstat(directory) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("V2 attempt directory is not an identified directory") + } + file, err := os.OpenFile(filepath.Join(directory, attemptID), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("V2 installation was already attempted on this host; automatic replay is refused; inspect the prior failure or reboot state before a new invocation") + } + if err != nil { + return err + } + if err := errors.Join(file.Sync(), file.Close()); err != nil { + return err + } + dir, err := os.Open(directory) + if err != nil { + return err + } + return errors.Join(dir.Sync(), dir.Close()) +} diff --git a/pkg/local/preflight/installer_collection_v2_test.go b/pkg/local/preflight/installer_collection_v2_test.go new file mode 100644 index 000000000..1ffa4dbeb --- /dev/null +++ b/pkg/local/preflight/installer_collection_v2_test.go @@ -0,0 +1,53 @@ +package preflight + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/longhorn/cli/pkg/types" +) + +func TestV2InstallerTransportReportsFailureWithoutAutomaticReplay(t *testing.T) { + directory := filepath.Join(t.TempDir(), "persistent-attempts") + calls := 0 + installer := Installer{collection: types.NodeCollection{Log: &types.LogCollection{}}} + run := func() error { calls++; return errors.New("installation pending a host reboot") } + if err := runV2InstallAttempt(directory, "invocation-1", run, installer.recordV2InstallFailure); err != nil { + t.Fatal(err) + } + if calls != 1 || len(installer.collection.Log.Error) != 1 || !strings.Contains(installer.collection.Log.Error[0], "host reboot") { + t.Fatal("failed installation did not produce its failure report") + } + // A new container/Pod uses the same persistent host marker. It must be + // able to report failure without calling the mutating installer again. + replacement := Installer{collection: types.NodeCollection{Log: &types.LogCollection{}}} + if err := runV2InstallAttempt(directory, "invocation-1", run, replacement.recordV2InstallFailure); err != nil { + t.Fatal(err) + } + if calls != 1 || len(replacement.collection.Log.Error) != 1 || !strings.Contains(replacement.collection.Log.Error[0], "automatic replay is refused") { + t.Fatal("container replacement replayed host writes or lost the refusal report") + } + if err := runV2InstallAttempt(directory, "explicit-new-invocation", run, replacement.recordV2InstallFailure); err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatal("an explicit new invocation could not run") + } +} + +func TestV2InstallerCannotMutateWithoutItsAttemptRecord(t *testing.T) { + directory := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(directory, []byte("existing"), 0600); err != nil { + t.Fatal(err) + } + runs, failures := 0, 0 + if err := runV2InstallAttempt(directory, "invocation", func() error { runs++; return nil }, func(error) { failures++ }); err != nil { + t.Fatal(err) + } + if runs != 0 || failures != 1 { + t.Fatal("host mutation ran without a persistent attempt record") + } +} diff --git a/pkg/remote/preflight/checker.go b/pkg/remote/preflight/checker.go index 712a90007..90ac4ac84 100644 --- a/pkg/remote/preflight/checker.go +++ b/pkg/remote/preflight/checker.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/json" + stderrors "errors" "path/filepath" "reflect" "strings" @@ -177,16 +178,18 @@ func (remote *Checker) Cleanup() error { if remote.DataEngineMode == string(types.DataEngineModeV2Only) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() + var daemonSetErr error if remote.createdDaemonSet != nil { uid := remote.createdDaemonSet.UID if uid == "" { - return errors.New("refusing V2 checker cleanup without its created UID") - } - if err := remote.kubeClient.AppsV1().DaemonSets(remote.createdDaemonSet.Namespace).Delete(ctx, remote.createdDaemonSet.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { - return err + daemonSetErr = errors.New("refusing V2 checker cleanup without its created UID") + } else if err := remote.kubeClient.AppsV1().DaemonSets(remote.createdDaemonSet.Namespace).Delete(ctx, remote.createdDaemonSet.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + daemonSetErr = err } } - return cleanupV2RBAC(ctx, remote.kubeClient, remote.createdRBAC) + rbacContext, rbacCancel := context.WithTimeout(context.Background(), time.Minute) + defer rbacCancel() + return stderrors.Join(daemonSetErr, cleanupV2RBAC(rbacContext, remote.kubeClient, remote.createdRBAC)) } if err := commonkube.DeleteDaemonSet(remote.kubeClient, remote.Namespace, remote.appName); err != nil { diff --git a/pkg/remote/preflight/checker_collection_v2.go b/pkg/remote/preflight/checker_collection_v2.go index 1b798a816..cbdfb2f2b 100644 --- a/pkg/remote/preflight/checker_collection_v2.go +++ b/pkg/remote/preflight/checker_collection_v2.go @@ -322,12 +322,18 @@ func validV2CheckerTemplate(template corev1.PodSpec, operation string) bool { mode, collection, node := environment[consts.EnvDataEngineMode], environment[consts.EnvPreflightCollectionOnly], environment[consts.EnvCurrentNodeID] collectionValue := "true" if operation == consts.SubCmdInstall { - collectionValue = "" + attempt := environment[consts.EnvPreflightAttemptID] + if attempt.Value == "" || attempt.ValueFrom != nil { + return false + } } return mode.Value == string(types.DataEngineModeV2Only) && mode.ValueFrom == nil && collection.Value == collectionValue && collection.ValueFrom == nil && node.Value == "" && node.ValueFrom != nil && node.ValueFrom.FieldRef != nil && node.ValueFrom.FieldRef.FieldPath == "spec.nodeName" } func sameV2ContainerExecution(expected, actual corev1.Container, serviceAccountVolume string) bool { + if !reflect.DeepEqual(expected.RestartPolicy, actual.RestartPolicy) || !reflect.DeepEqual(expected.StartupProbe, actual.StartupProbe) || !reflect.DeepEqual(expected.LivenessProbe, actual.LivenessProbe) || !reflect.DeepEqual(expected.ReadinessProbe, actual.ReadinessProbe) { + return false + } if expected.Name != actual.Name || expected.Image != actual.Image || expected.WorkingDir != actual.WorkingDir || expected.Stdin != actual.Stdin || expected.TTY != actual.TTY || len(actual.VolumeDevices) != 0 || !reflect.DeepEqual(expected.Command, actual.Command) || !reflect.DeepEqual(expected.Args, actual.Args) || !reflect.DeepEqual(expected.EnvFrom, actual.EnvFrom) || !reflect.DeepEqual(expected.SecurityContext, actual.SecurityContext) || !reflect.DeepEqual(expected.Lifecycle, actual.Lifecycle) || len(expected.Env) != len(actual.Env) { return false } @@ -433,8 +439,15 @@ func readV2PodReport(client kubeclient.Interface) v2ReportLogReader { if err != nil { return nil, err } - defer stream.Close() - return io.ReadAll(io.LimitReader(stream, limit)) + data, readErr := io.ReadAll(io.LimitReader(stream, limit)) + closeErr := stream.Close() + if readErr != nil { + return nil, readErr + } + if closeErr != nil { + return nil, closeErr + } + return data, nil } } diff --git a/pkg/remote/preflight/checker_collection_v2_test.go b/pkg/remote/preflight/checker_collection_v2_test.go index f95e698a4..f6647ce10 100644 --- a/pkg/remote/preflight/checker_collection_v2_test.go +++ b/pkg/remote/preflight/checker_collection_v2_test.go @@ -148,6 +148,12 @@ func TestV2CollectorBindsCompleteCurrentExecution(t *testing.T) { f.ds.Status.UpdatedNumberScheduled = 2 }}, {name: "unbound-executable", before: func(f *v2CollectionFixture) { f.pods[0].Spec.InitContainers[0].Command = []string{"old-checker"} }}, + {name: "restartable-installer", before: func(f *v2CollectionFixture) { + f.pods[0].Spec.InitContainers[0].RestartPolicy = ptr.To(corev1.ContainerRestartPolicyAlways) + }}, + {name: "injected-liveness-probe", before: func(f *v2CollectionFixture) { + f.pods[0].Spec.InitContainers[0].LivenessProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"false"}}}} + }}, {name: "reporter-shadowed-output", before: func(f *v2CollectionFixture) { f.pods[0].Spec.Volumes = append(f.pods[0].Spec.Volumes, corev1.Volume{Name: "stale", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "stale"}}}}) f.pods[0].Spec.InitContainers[1].VolumeMounts = append(f.pods[0].Spec.InitContainers[1].VolumeMounts, corev1.VolumeMount{Name: "stale", MountPath: "/shared/output.json", SubPath: "report.json"}) @@ -249,6 +255,12 @@ func TestV2AdmissionCannotLowerRequestedChecksOrReplaceOutput(t *testing.T) { t.Fatalf("native defaults were mistaken for an execution rewrite: %v", err) } for _, mutate := range []func(*appsv1.DaemonSet){ + func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.InitContainers[0].RestartPolicy = ptr.To(corev1.ContainerRestartPolicyAlways) + }, + func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.InitContainers[0].StartupProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"false"}}}} + }, func(ds *appsv1.DaemonSet) { for index, variable := range ds.Spec.Template.Spec.InitContainers[0].Env { if variable.Name == consts.EnvHugePageSize { diff --git a/pkg/remote/preflight/data_engine_mode_test.go b/pkg/remote/preflight/data_engine_mode_test.go index fbe341ad1..c2300cf37 100644 --- a/pkg/remote/preflight/data_engine_mode_test.go +++ b/pkg/remote/preflight/data_engine_mode_test.go @@ -1,6 +1,7 @@ package preflight import ( + "github.com/longhorn/cli/pkg/consts" "reflect" "strconv" "strings" @@ -99,6 +100,17 @@ func TestRemoteInstallerBindsV2ExecutableBeforeAnyMutation(t *testing.T) { } } +func TestRemoteV2InstallerBindsItsReportAndAttemptProfile(t *testing.T) { + installer := Installer{InstallerCmdOptions: InstallerCmdOptions{DataEngineMode: "v2-only", HugePageSize: 2048}, appName: consts.AppNamePreflightInstaller + "-fixture"} + if err := installer.ValidateDataEngineMode(); err != nil { + t.Fatal(err) + } + template := installer.newDaemonSetForPackageManager().Spec.Template.Spec + if !validV2CheckerTemplate(template, consts.SubCmdInstall) || validV2CheckerTemplate(template, consts.SubCmdCheck) { + t.Fatal("V2 installer transport lost its fixed executable, report, or attempt identity") + } +} + func TestRemoteInstallerRejectsInvalidModeBeforeConnectionInitialization(t *testing.T) { installer := Installer{InstallerCmdOptions: InstallerCmdOptions{DataEngineMode: "unknown"}} if err := installer.Init(); err == nil || !strings.Contains(err.Error(), "data-engine-mode") { diff --git a/pkg/remote/preflight/installer.go b/pkg/remote/preflight/installer.go index 02515a0ec..88cbc7944 100644 --- a/pkg/remote/preflight/installer.go +++ b/pkg/remote/preflight/installer.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/json" + stderrors "errors" "path/filepath" "reflect" "strings" @@ -152,16 +153,18 @@ func (remote *Installer) Cleanup() error { if remote.DataEngineMode == string(types.DataEngineModeV2Only) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() + var daemonSetErr error if remote.createdDaemonSet != nil { uid := remote.createdDaemonSet.UID if uid == "" { - return errors.New("refusing V2 installer cleanup without its created UID") - } - if err := remote.kubeClient.AppsV1().DaemonSets(remote.createdDaemonSet.Namespace).Delete(ctx, remote.createdDaemonSet.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { - return err + daemonSetErr = errors.New("refusing V2 installer cleanup without its created UID") + } else if err := remote.kubeClient.AppsV1().DaemonSets(remote.createdDaemonSet.Namespace).Delete(ctx, remote.createdDaemonSet.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + daemonSetErr = err } } - return cleanupV2RBAC(ctx, remote.kubeClient, remote.createdRBAC) + rbacContext, rbacCancel := context.WithTimeout(context.Background(), time.Minute) + defer rbacCancel() + return stderrors.Join(daemonSetErr, cleanupV2RBAC(rbacContext, remote.kubeClient, remote.createdRBAC)) } if err := commonkube.DeleteDaemonSet(remote.kubeClient, remote.Namespace, remote.appName); err != nil { @@ -552,6 +555,8 @@ func (remote *Installer) newDaemonSetForPackageManager() *appsv1.DaemonSet { Command: command, Env: []corev1.EnvVar{ {Name: consts.EnvDataEngineMode, Value: remote.DataEngineMode}, + {Name: consts.EnvPreflightCollectionOnly, Value: commonutils.ConvertTypeToString(remote.DataEngineMode == string(types.DataEngineModeV2Only))}, + {Name: consts.EnvPreflightAttemptID, Value: remote.appName}, { Name: consts.EnvLogLevel, Value: remote.LogLevel, diff --git a/pkg/remote/preflight/rbac_v2_test.go b/pkg/remote/preflight/rbac_v2_test.go index d5d739399..dfe9a6ace 100644 --- a/pkg/remote/preflight/rbac_v2_test.go +++ b/pkg/remote/preflight/rbac_v2_test.go @@ -9,7 +9,9 @@ import ( "sync" "testing" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" kubeclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -85,8 +87,61 @@ func TestV2RBACNeverAdoptsOrDeletesUnownedObjects(t *testing.T) { t.Fatalf("cleanup hid replacement conflict: %v", err) } if deletes != test.created { - t.Fatal(fmt.Sprintf("deleted objects without successful creation receipts: got=%d want=%d", deletes, test.created)) + t.Fatalf("deleted objects without successful creation receipts: got=%d want=%d", deletes, test.created) } }) } } + +func TestV2CleanupStillRemovesOwnedRBACAfterDaemonSetFailure(t *testing.T) { + for _, installer := range []bool{false, true} { + for _, missingUID := range []bool{false, true} { + t.Run(fmt.Sprintf("installer=%v/missingUID=%v", installer, missingUID), func(t *testing.T) { + var mu sync.Mutex + deletes := map[string]int{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("Content-Type", "application/json") + parts := strings.Split(r.URL.Path, "/") + resource := parts[len(parts)-2] + deletes[resource]++ + var options metav1.DeleteOptions + if r.Method != http.MethodDelete || json.NewDecoder(r.Body).Decode(&options) != nil || options.Preconditions == nil || options.Preconditions.UID == nil || *options.Preconditions.UID == "" { + t.Error("cleanup attempted an unidentified deletion") + } + if resource == "daemonsets" { + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(&metav1.Status{Status: "Failure", Reason: metav1.StatusReasonConflict, Code: 409}) + return + } + _ = json.NewEncoder(w).Encode(&metav1.Status{Status: "Success"}) + })) + t.Cleanup(server.Close) + client, err := kubeclient.NewForConfig(&rest.Config{Host: server.URL, ContentConfig: rest.ContentConfig{ContentType: "application/json", AcceptContentTypes: "application/json"}}) + if err != nil { + t.Fatal(err) + } + uid := k8stypes.UID("owned-ds") + if missingUID { + uid = "" + } + ds := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "owned", Namespace: "test", UID: uid}} + receipt := &v2RBACReceipt{Name: "owned", Namespace: "test", ServiceAccount: "sa-uid", ClusterRole: "role-uid", ClusterRoleBinding: "binding-uid"} + if installer { + checker := Installer{InstallerCmdOptions: InstallerCmdOptions{DataEngineMode: "v2-only"}, kubeClient: client, createdDaemonSet: ds, createdRBAC: receipt} + err = checker.Cleanup() + } else { + checker := Checker{CheckerCmdOptions: CheckerCmdOptions{DataEngineMode: "v2-only"}, kubeClient: client, createdDaemonSet: ds, createdRBAC: receipt} + err = checker.Cleanup() + } + if err == nil || deletes["serviceaccounts"] != 1 || deletes["clusterroles"] != 1 || deletes["clusterrolebindings"] != 1 { + t.Fatalf("DaemonSet failure leaked owned RBAC: deleted=%v err=%v", deletes, err) + } + if missingUID && deletes["daemonsets"] != 0 { + t.Fatal("deleted DaemonSet without UID") + } + }) + } + } +} From f6057f8ad8b7b45aa7ae18eb7c3c58684af2987f Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 01:31:46 +0100 Subject: [PATCH 4/7] fix(preflight): persist attempt directory entries before host writes Issue longhorn/longhorn#14438 Signed-off-by: Nikolai Emil Damm Co-authored-by: Codex (GPT-6.1 Sol) --- README.md | 2 + .../preflight/installer_collection_v2.go | 49 ++++++++++++++++--- .../preflight/installer_collection_v2_test.go | 21 +++++++- .../installer_journal_filesystem_linux.go | 39 +++++++++++++++ ...installer_journal_filesystem_linux_test.go | 11 +++++ .../installer_journal_filesystem_other.go | 9 ++++ 6 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 pkg/local/preflight/installer_journal_filesystem_linux.go create mode 100644 pkg/local/preflight/installer_journal_filesystem_linux_test.go create mode 100644 pkg/local/preflight/installer_journal_filesystem_other.go diff --git a/README.md b/README.md index 594eb7745..88048d59e 100644 --- a/README.md +++ b/README.md @@ -73,3 +73,5 @@ V2 remote runs use separate resource names and collect complete reports from the Remote V2 installation records its invocation under `/var/lib/longhorn-cli/preflight-attempts` before changing host prerequisites. A Kubernetes retry or replacement Pod reports an error instead of repeating that invocation's host writes. The small attempt marker remains for audit and replay protection; assess any partial installation or required reboot before explicitly starting a new command. Failed and reboot-pending attempts are transported as error reports and the remote command exits nonzero. +Replay protection requires a private installer-owned directory on a verified ext, XFS, Btrfs or F2FS filesystem. Volatile and unverified stores are refused. Directory entries and the marker are synced before host changes start. + diff --git a/pkg/local/preflight/installer_collection_v2.go b/pkg/local/preflight/installer_collection_v2.go index f65b98d21..e96ed34b5 100644 --- a/pkg/local/preflight/installer_collection_v2.go +++ b/pkg/local/preflight/installer_collection_v2.go @@ -7,6 +7,7 @@ import ( "path/filepath" "regexp" "strings" + "syscall" "github.com/longhorn/cli/pkg/consts" "github.com/longhorn/cli/pkg/types" @@ -34,6 +35,10 @@ func (local *Installer) RunForCollection() error { return nil } directory := filepath.Join(consts.VolumeMountHostDirectory, "var/lib/longhorn-cli/preflight-attempts") + if err := qualifyV2AttemptFilesystem(directory); err != nil { + local.recordV2InstallFailure(err) + return nil + } return runV2InstallAttempt(directory, local.AttemptID, local.Run, local.recordV2InstallFailure) } @@ -54,16 +59,9 @@ func runV2InstallAttempt(directory, attemptID string, run func() error, recordFa } func claimV2InstallAttempt(directory, attemptID string) error { - if err := os.MkdirAll(directory, 0700); err != nil { - return err - } - info, err := os.Lstat(directory) - if err != nil { + if err := prepareV2AttemptDirectory(directory); err != nil { return err } - if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { - return fmt.Errorf("V2 attempt directory is not an identified directory") - } file, err := os.OpenFile(filepath.Join(directory, attemptID), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if errors.Is(err, os.ErrExist) { return fmt.Errorf("V2 installation was already attempted on this host; automatic replay is refused; inspect the prior failure or reboot state before a new invocation") @@ -74,9 +72,44 @@ func claimV2InstallAttempt(directory, attemptID string) error { if err := errors.Join(file.Sync(), file.Close()); err != nil { return err } + return syncV2AttemptDirectory(directory) +} + +func syncV2AttemptDirectory(directory string) error { dir, err := os.Open(directory) if err != nil { return err } return errors.Join(dir.Sync(), dir.Close()) } + +func prepareV2AttemptDirectory(directory string) error { + // The runtime path has two private levels below the existing /var/lib. + // Persist each child's entry in its parent, including when a concurrent + // invocation created it, before the host-changing callback can run. + root := filepath.Dir(filepath.Dir(directory)) + info, err := os.Lstat(root) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("V2 attempt root is not an identified directory") + } + for _, path := range []string{filepath.Dir(directory), directory} { + if err := os.Mkdir(path, 0700); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + info, err := os.Lstat(path) + if err != nil { + return err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0077 != 0 || !ok || int(stat.Uid) != os.Geteuid() { + return fmt.Errorf("V2 attempt directory is not private and owned by this installer") + } + if err := errors.Join(syncV2AttemptDirectory(path), syncV2AttemptDirectory(filepath.Dir(path))); err != nil { + return err + } + } + return nil +} diff --git a/pkg/local/preflight/installer_collection_v2_test.go b/pkg/local/preflight/installer_collection_v2_test.go index 1ffa4dbeb..63d5377aa 100644 --- a/pkg/local/preflight/installer_collection_v2_test.go +++ b/pkg/local/preflight/installer_collection_v2_test.go @@ -11,7 +11,7 @@ import ( ) func TestV2InstallerTransportReportsFailureWithoutAutomaticReplay(t *testing.T) { - directory := filepath.Join(t.TempDir(), "persistent-attempts") + directory := filepath.Join(t.TempDir(), "private-tool", "persistent-attempts") calls := 0 installer := Installer{collection: types.NodeCollection{Log: &types.LogCollection{}}} run := func() error { calls++; return errors.New("installation pending a host reboot") } @@ -51,3 +51,22 @@ func TestV2InstallerCannotMutateWithoutItsAttemptRecord(t *testing.T) { t.Fatal("host mutation ran without a persistent attempt record") } } + +func TestV2AttemptStoreRejectsSymlinksBeforeHostWrites(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "tool")); err != nil { + t.Fatal(err) + } + runs, failures := 0, 0 + if err := runV2InstallAttempt(filepath.Join(root, "tool", "attempts"), "invocation", func() error { runs++; return nil }, func(error) { failures++ }); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(outside) + if err != nil { + t.Fatal(err) + } + if runs != 0 || failures != 1 || len(entries) != 0 { + t.Fatal("unidentified attempt directory allowed host writes or changed its symlink target") + } +} diff --git a/pkg/local/preflight/installer_journal_filesystem_linux.go b/pkg/local/preflight/installer_journal_filesystem_linux.go new file mode 100644 index 000000000..e074a4c06 --- /dev/null +++ b/pkg/local/preflight/installer_journal_filesystem_linux.go @@ -0,0 +1,39 @@ +//go:build linux + +package preflight + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +func qualifyV2AttemptFilesystem(directory string) error { + path := directory + for { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) && filepath.Dir(path) != path { + path = filepath.Dir(path) + continue + } + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("V2 attempt store is not an identified directory") + } + var status unix.Statfs_t + if err := unix.Statfs(path, &status); err != nil { + return err + } + switch status.Type { + case unix.EXT4_SUPER_MAGIC, unix.XFS_SUPER_MAGIC, unix.BTRFS_SUPER_MAGIC, unix.F2FS_SUPER_MAGIC: + return nil + default: + return fmt.Errorf("V2 installer replay protection requires a qualified persistent ext, XFS, Btrfs or F2FS attempt store") + } + } +} diff --git a/pkg/local/preflight/installer_journal_filesystem_linux_test.go b/pkg/local/preflight/installer_journal_filesystem_linux_test.go new file mode 100644 index 000000000..365e0c403 --- /dev/null +++ b/pkg/local/preflight/installer_journal_filesystem_linux_test.go @@ -0,0 +1,11 @@ +//go:build linux + +package preflight + +import "testing" + +func TestV2AttemptStoreRejectsVirtualFilesystems(t *testing.T) { + if err := qualifyV2AttemptFilesystem("/proc/longhorn-cli/nonexistent-attempts"); err == nil { + t.Fatal("virtual attempt store incorrectly promised durable replay protection") + } +} diff --git a/pkg/local/preflight/installer_journal_filesystem_other.go b/pkg/local/preflight/installer_journal_filesystem_other.go new file mode 100644 index 000000000..a718fbe1b --- /dev/null +++ b/pkg/local/preflight/installer_journal_filesystem_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package preflight + +import "fmt" + +func qualifyV2AttemptFilesystem(string) error { + return fmt.Errorf("V2 installer replay protection requires a qualified Linux host") +} From 71412f1b1d47290bb0409a5dfe2ef22d62ded797 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 01:45:56 +0100 Subject: [PATCH 5/7] docs: generate CLI references with the versioned build Issue longhorn/longhorn#14438 Signed-off-by: Nikolai Emil Damm Co-authored-by: Codex (GPT-6.1 Sol) --- docs/longhornctl.md | 2 +- docs/longhornctl_check.md | 2 +- docs/longhornctl_check_preflight.md | 2 +- docs/longhornctl_checksum.md | 2 +- docs/longhornctl_checksum_volume.md | 2 +- docs/longhornctl_doc.md | 2 +- docs/longhornctl_export.md | 2 +- docs/longhornctl_export_replica.md | 4 ++-- docs/longhornctl_export_replica_stop.md | 2 +- docs/longhornctl_get.md | 2 +- docs/longhornctl_get_replica.md | 2 +- docs/longhornctl_global-options.md | 2 +- docs/longhornctl_install.md | 2 +- docs/longhornctl_install_preflight.md | 2 +- docs/longhornctl_install_preflight_stop.md | 2 +- docs/longhornctl_trim.md | 2 +- docs/longhornctl_trim_volume.md | 2 +- docs/longhornctl_version.md | 2 +- 18 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/longhornctl.md b/docs/longhornctl.md index ef5689778..5df62253f 100644 --- a/docs/longhornctl.md +++ b/docs/longhornctl.md @@ -10,7 +10,7 @@ A CLI tool for troubleshooting and managing Longhorn operations. ``` -h, --help help for longhornctl - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_check.md b/docs/longhornctl_check.md index 4e0712379..6d4dd5ecd 100644 --- a/docs/longhornctl_check.md +++ b/docs/longhornctl_check.md @@ -6,7 +6,7 @@ Longhorn checking operations ``` -h, --help help for check - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_check_preflight.md b/docs/longhornctl_check_preflight.md index 1b29dce95..87617c274 100644 --- a/docs/longhornctl_check_preflight.md +++ b/docs/longhornctl_check_preflight.md @@ -47,7 +47,7 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker --enable-spdk Enable checking of SPDK required packages, modules, and setup. -h, --help help for preflight --huge-page-size int Specify the huge page size in MiB for SPDK. (default 2048) - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_checksum.md b/docs/longhornctl_checksum.md index 1d40b5711..4bdf44b04 100644 --- a/docs/longhornctl_checksum.md +++ b/docs/longhornctl_checksum.md @@ -6,7 +6,7 @@ Snapshot checksum operations ``` -h, --help help for checksum - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_checksum_volume.md b/docs/longhornctl_checksum_volume.md index 29e746a91..d0602fcd4 100644 --- a/docs/longhornctl_checksum_volume.md +++ b/docs/longhornctl_checksum_volume.md @@ -27,7 +27,7 @@ $ longhornctl checksum volume --name=v1 --namespace=longhorn-system ``` --all Apply to all volumes -h, --help help for volume - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_doc.md b/docs/longhornctl_doc.md index 6f0eb48ad..17c52d6f2 100644 --- a/docs/longhornctl_doc.md +++ b/docs/longhornctl_doc.md @@ -19,7 +19,7 @@ longhornctl doc [output directory] [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_export.md b/docs/longhornctl_export.md index bedb8cfe9..f9ffaf4e9 100644 --- a/docs/longhornctl_export.md +++ b/docs/longhornctl_export.md @@ -6,7 +6,7 @@ Export Longhorn resources ``` -h, --help help for export - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_export_replica.md b/docs/longhornctl_export_replica.md index 7aa0a1f17..cc692b0c6 100644 --- a/docs/longhornctl_export_replica.md +++ b/docs/longhornctl_export_replica.md @@ -48,9 +48,9 @@ lost+found ``` --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") - --engine-image string Engine image to use to create volume from the replica. (default "longhornio/longhorn-engine:") + --engine-image string Engine image to use to create volume from the replica. (default "longhornio/longhorn-engine:v1.14.0-dev") -h, --help help for replica - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_export_replica_stop.md b/docs/longhornctl_export_replica_stop.md index e6d214963..eaf017629 100644 --- a/docs/longhornctl_export_replica_stop.md +++ b/docs/longhornctl_export_replica_stop.md @@ -22,7 +22,7 @@ INFO[2024-07-16T17:29:14+08:00] Successfully stopped exporting replica ``` -h, --help help for stop - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_get.md b/docs/longhornctl_get.md index df413b4b9..dbf7d93cf 100644 --- a/docs/longhornctl_get.md +++ b/docs/longhornctl_get.md @@ -6,7 +6,7 @@ Longhorn information gathering operations ``` -h, --help help for get - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_get_replica.md b/docs/longhornctl_get_replica.md index 15834a59b..e078bd85a 100644 --- a/docs/longhornctl_get_replica.md +++ b/docs/longhornctl_get_replica.md @@ -49,7 +49,7 @@ INFO[2024-07-16T17:23:51+08:00] Completed replica getter ``` --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") -h, --help help for replica - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_global-options.md b/docs/longhornctl_global-options.md index 04ca1f7aa..0cb9d9ccc 100644 --- a/docs/longhornctl_global-options.md +++ b/docs/longhornctl_global-options.md @@ -19,7 +19,7 @@ longhornctl global-options [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_install.md b/docs/longhornctl_install.md index d840acfc0..14b4938c6 100644 --- a/docs/longhornctl_install.md +++ b/docs/longhornctl_install.md @@ -6,7 +6,7 @@ Longhorn installation operations ``` -h, --help help for install - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_install_preflight.md b/docs/longhornctl_install_preflight.md index f5fa254b6..d818b19e9 100644 --- a/docs/longhornctl_install_preflight.md +++ b/docs/longhornctl_install_preflight.md @@ -52,7 +52,7 @@ If a reboot is required, the following message will be displayed: --enable-spdk Enable installation of SPDK required packages, modules, and setup. -h, --help help for preflight --huge-page-size int Specify the huge page size in MiB for SPDK. (default 2048) - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_install_preflight_stop.md b/docs/longhornctl_install_preflight_stop.md index ed1bad377..2ae95cf7a 100644 --- a/docs/longhornctl_install_preflight_stop.md +++ b/docs/longhornctl_install_preflight_stop.md @@ -22,7 +22,7 @@ INFO[2024-07-16T17:21:32+08:00] Successfully stopped preflight installer ``` -h, --help help for stop - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_trim.md b/docs/longhornctl_trim.md index 9955e6b22..beb126787 100644 --- a/docs/longhornctl_trim.md +++ b/docs/longhornctl_trim.md @@ -6,7 +6,7 @@ Longhorn trimming operations ``` -h, --help help for trim - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_trim_volume.md b/docs/longhornctl_trim_volume.md index d021c78d0..7b0bd3939 100644 --- a/docs/longhornctl_trim_volume.md +++ b/docs/longhornctl_trim_volume.md @@ -31,7 +31,7 @@ INFO[2024-07-16T17:32:01+08:00] Completed volume trimmer vo ``` -h, --help help for volume - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path diff --git a/docs/longhornctl_version.md b/docs/longhornctl_version.md index f27d6d590..db01103ae 100644 --- a/docs/longhornctl_version.md +++ b/docs/longhornctl_version.md @@ -15,7 +15,7 @@ longhornctl version [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path From dfc66353d854bdad6ed64340c272d481feb027a6 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 01:52:25 +0100 Subject: [PATCH 6/7] docs: use the native huge-page-size flag in V2 examples Issue longhorn/longhorn#14438 Signed-off-by: Nikolai Emil Damm Co-authored-by: Codex (GPT-6.1 Sol) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 88048d59e..8fa337762 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,8 @@ The default `--data-engine-mode=v1-capable` retains iSCSI requirements. An expli Use matching CLI binaries and a CLI image containing this capability. The remote command passes an explicit flag to the local executable so an older image fails instead of silently running V1 checks. ```bash -longhornctl install preflight --data-engine-mode=v2-only --hugepage-size=2048 --allow-pci=none -longhornctl check preflight --data-engine-mode=v2-only --hugepage-size=2048 +longhornctl install preflight --data-engine-mode=v2-only --huge-page-size=2048 --allow-pci=none +longhornctl check preflight --data-engine-mode=v2-only --huge-page-size=2048 ``` Installation success is prerequisite setup, followed by a separate host qualification check. A required reboot remains unfinished installation; reboot and rerun installation before checking. Restart kubelet when necessary for its Node capacity to reflect the hugepage allocation. V2 check failures return a nonzero exit status after printing the complete report. From d53747cda320717c1df9feb79645ec3322f39672 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 02:16:41 +0100 Subject: [PATCH 7/7] style(preflight): follow upstream Go import groups Issue longhorn/longhorn#14438 Signed-off-by: Nikolai Emil Damm Co-authored-by: Codex (GPT-6.1 Sol) --- cmd/local/subcmd/check.go | 3 ++- cmd/local/subcmd/install.go | 6 +++--- cmd/remote/subcmd/check.go | 1 + cmd/remote/subcmd/install.go | 3 ++- pkg/local/preflight/checker_data_engine_mode_test.go | 4 ++-- pkg/local/preflight/installer_spdk_v2_test.go | 3 ++- pkg/remote/preflight/checker.go | 3 ++- pkg/remote/preflight/checker_collection_v2.go | 5 +++-- pkg/remote/preflight/checker_collection_v2_test.go | 5 +++-- pkg/remote/preflight/data_engine_mode_test.go | 3 ++- pkg/remote/preflight/installer.go | 3 ++- pkg/remote/preflight/rbac_v2_test.go | 3 ++- 12 files changed, 26 insertions(+), 16 deletions(-) diff --git a/cmd/local/subcmd/check.go b/cmd/local/subcmd/check.go index 757f98914..3a4304ff7 100644 --- a/cmd/local/subcmd/check.go +++ b/cmd/local/subcmd/check.go @@ -8,9 +8,10 @@ import ( "github.com/spf13/cobra" "github.com/longhorn/cli/pkg/consts" - local "github.com/longhorn/cli/pkg/local/preflight" "github.com/longhorn/cli/pkg/types" "github.com/longhorn/cli/pkg/utils" + + local "github.com/longhorn/cli/pkg/local/preflight" ) func NewCmdCheck(globalOpts *types.GlobalCmdOptions) *cobra.Command { diff --git a/cmd/local/subcmd/install.go b/cmd/local/subcmd/install.go index 3205dffcb..d10f01545 100644 --- a/cmd/local/subcmd/install.go +++ b/cmd/local/subcmd/install.go @@ -4,15 +4,15 @@ import ( "fmt" "os" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/pkg/errors" - "github.com/longhorn/cli/pkg/consts" - local "github.com/longhorn/cli/pkg/local/preflight" "github.com/longhorn/cli/pkg/types" "github.com/longhorn/cli/pkg/utils" + + local "github.com/longhorn/cli/pkg/local/preflight" ) func NewCmdInstall(globalOpts *types.GlobalCmdOptions) *cobra.Command { diff --git a/cmd/remote/subcmd/check.go b/cmd/remote/subcmd/check.go index 92e6facfa..c733dcaee 100644 --- a/cmd/remote/subcmd/check.go +++ b/cmd/remote/subcmd/check.go @@ -2,6 +2,7 @@ package subcmd import ( stderrors "errors" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/cmd/remote/subcmd/install.go b/cmd/remote/subcmd/install.go index 29eaa9b1a..69e4812a7 100644 --- a/cmd/remote/subcmd/install.go +++ b/cmd/remote/subcmd/install.go @@ -1,9 +1,10 @@ package subcmd import ( - stderrors "errors" "fmt" + stderrors "errors" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/pkg/local/preflight/checker_data_engine_mode_test.go b/pkg/local/preflight/checker_data_engine_mode_test.go index f1467de09..c80ae3fb2 100644 --- a/pkg/local/preflight/checker_data_engine_mode_test.go +++ b/pkg/local/preflight/checker_data_engine_mode_test.go @@ -14,19 +14,19 @@ import ( "github.com/stretchr/testify/mock" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/rest" "k8s.io/utils/ptr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeclient "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/local/preflight/packagemanager/mocks" "github.com/longhorn/cli/pkg/types" pkgmgr "github.com/longhorn/cli/pkg/local/preflight/packagemanager" - "github.com/longhorn/cli/pkg/local/preflight/packagemanager/mocks" remote "github.com/longhorn/cli/pkg/remote/preflight" ) diff --git a/pkg/local/preflight/installer_spdk_v2_test.go b/pkg/local/preflight/installer_spdk_v2_test.go index d1a835771..e9eebe413 100644 --- a/pkg/local/preflight/installer_spdk_v2_test.go +++ b/pkg/local/preflight/installer_spdk_v2_test.go @@ -8,8 +8,9 @@ import ( "github.com/stretchr/testify/mock" "github.com/longhorn/cli/pkg/local/preflight/packagemanager/mocks" - remote "github.com/longhorn/cli/pkg/remote/preflight" "github.com/longhorn/cli/pkg/types" + + remote "github.com/longhorn/cli/pkg/remote/preflight" ) func TestV2SPDKSetupClearsInheritedOptionsAndVerifiesActualAllocation(t *testing.T) { diff --git a/pkg/remote/preflight/checker.go b/pkg/remote/preflight/checker.go index 90ac4ac84..ca587a1fd 100644 --- a/pkg/remote/preflight/checker.go +++ b/pkg/remote/preflight/checker.go @@ -4,12 +4,13 @@ import ( "context" "crypto/rand" "encoding/json" - stderrors "errors" "path/filepath" "reflect" "strings" "time" + stderrors "errors" + "github.com/pkg/errors" "k8s.io/utils/ptr" diff --git a/pkg/remote/preflight/checker_collection_v2.go b/pkg/remote/preflight/checker_collection_v2.go index cbdfb2f2b..26b0aa0d1 100644 --- a/pkg/remote/preflight/checker_collection_v2.go +++ b/pkg/remote/preflight/checker_collection_v2.go @@ -11,12 +11,13 @@ import ( "strings" "time" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/kustomize/kyaml/yaml" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" kubeclient "k8s.io/client-go/kubernetes" - "sigs.k8s.io/kustomize/kyaml/yaml" "github.com/longhorn/cli/pkg/consts" "github.com/longhorn/cli/pkg/types" diff --git a/pkg/remote/preflight/checker_collection_v2_test.go b/pkg/remote/preflight/checker_collection_v2_test.go index f6647ce10..63c2e2fdf 100644 --- a/pkg/remote/preflight/checker_collection_v2_test.go +++ b/pkg/remote/preflight/checker_collection_v2_test.go @@ -11,13 +11,14 @@ import ( "testing" "time" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" kubeclient "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/utils/ptr" "github.com/longhorn/cli/pkg/consts" "github.com/longhorn/cli/pkg/types" diff --git a/pkg/remote/preflight/data_engine_mode_test.go b/pkg/remote/preflight/data_engine_mode_test.go index c2300cf37..0bc0f66ae 100644 --- a/pkg/remote/preflight/data_engine_mode_test.go +++ b/pkg/remote/preflight/data_engine_mode_test.go @@ -1,11 +1,12 @@ package preflight import ( - "github.com/longhorn/cli/pkg/consts" "reflect" "strconv" "strings" "testing" + + "github.com/longhorn/cli/pkg/consts" ) func TestCheckerModeRequiresResolvedV2Prerequisites(t *testing.T) { diff --git a/pkg/remote/preflight/installer.go b/pkg/remote/preflight/installer.go index 88cbc7944..8e1c897bc 100644 --- a/pkg/remote/preflight/installer.go +++ b/pkg/remote/preflight/installer.go @@ -4,12 +4,13 @@ import ( "context" "crypto/rand" "encoding/json" - stderrors "errors" "path/filepath" "reflect" "strings" "time" + stderrors "errors" + "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/pkg/remote/preflight/rbac_v2_test.go b/pkg/remote/preflight/rbac_v2_test.go index dfe9a6ace..9b9c33d7f 100644 --- a/pkg/remote/preflight/rbac_v2_test.go +++ b/pkg/remote/preflight/rbac_v2_test.go @@ -9,11 +9,12 @@ import ( "sync" "testing" + "k8s.io/client-go/rest" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" kubeclient "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" ) func TestV2RBACNeverAdoptsOrDeletesUnownedObjects(t *testing.T) {