From d0babc521786db32b5481e0c05f08f4f609cb6dd Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Wed, 4 Dec 2024 14:29:57 +0100 Subject: [PATCH 1/8] add postprocessing logic for sync tasks which handles job ttl Signed-off-by: Dejan Zele Pejchev --- pkg/health/health.go | 13 ++++- pkg/sync/sync_context.go | 102 +++++++++++++++++++++++++++++++++++++++ pkg/sync/sync_task.go | 2 + 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/pkg/health/health.go b/pkg/health/health.go index b93d8c967..6cb37e879 100644 --- a/pkg/health/health.go +++ b/pkg/health/health.go @@ -64,7 +64,7 @@ func IsWorse(current, new HealthStatusCode) bool { // GetResourceHealth returns the health of a k8s resource func GetResourceHealth(obj *unstructured.Unstructured, healthOverride HealthOverride) (health *HealthStatus, err error) { - if obj.GetDeletionTimestamp() != nil { + if obj.GetDeletionTimestamp() != nil && !hasHookFinalizer(obj) { return &HealthStatus{ Status: HealthStatusProgressing, Message: "Pending deletion", @@ -97,6 +97,17 @@ func GetResourceHealth(obj *unstructured.Unstructured, healthOverride HealthOver } +func hasHookFinalizer(obj *unstructured.Unstructured) bool { + hookFinalizer := "argoproj.io/hook-finalizer" + finalizers := obj.GetFinalizers() + for _, finalizer := range finalizers { + if finalizer == hookFinalizer { + return true + } + } + return false +} + // GetHealthCheckFunc returns built-in health check function or nil if health check is not supported func GetHealthCheckFunc(gvk schema.GroupVersionKind) func(obj *unstructured.Unstructured) (*HealthStatus, error) { switch gvk.Group { diff --git a/pkg/sync/sync_context.go b/pkg/sync/sync_context.go index 35981ebaa..f01b34795 100644 --- a/pkg/sync/sync_context.go +++ b/pkg/sync/sync_context.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "k8s.io/client-go/kubernetes" "sort" "strings" "sync" @@ -228,6 +229,10 @@ func NewSyncContext( if err != nil { return nil, nil, err } + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, nil, err + } ctx := &syncContext{ revision: revision, resources: groupResources(reconciliationResult), @@ -236,6 +241,7 @@ func NewSyncContext( rawConfig: rawConfig, dynamicIf: dynamicIf, disco: disco, + clientset: clientset, extensionsclientset: extensionsclientset, kubectl: kubectl, resourceOps: resourceOps, @@ -331,6 +337,7 @@ type syncContext struct { dynamicIf dynamic.Interface disco discovery.DiscoveryInterface extensionsclientset *clientset.Clientset + clientset *kubernetes.Clientset kubectl kube.Kubectl resourceOps kube.ResourceOperations namespace string @@ -476,6 +483,17 @@ func (sc *syncContext) Sync() { return } + hooksCompleted := tasks.Filter(func(task *syncTask) bool { + return task.isHook() && task.completed() + }) + for _, task := range hooksCompleted { + if task.cleanup != nil { + if err := task.cleanup(); err != nil { + sc.log.V(1).Error(err, "failed to run hook task cleanup") + } + } + } + // collect all completed hooks which have appropriate delete policy hooksPendingDeletionSuccessful := tasks.Filter(func(task *syncTask) bool { return task.isHook() && task.liveObj != nil && !task.running() && task.deleteOnPhaseSuccessful() @@ -688,6 +706,8 @@ func (sc *syncContext) getSyncTasks() (_ syncTasks, successful bool) { sc.log.WithValues("hookTasks", hookTasks).V(1).Info("tasks from hooks") + sc.processHookTasks(hookTasks) + tasks := resourceTasks tasks = append(tasks, hookTasks...) @@ -830,6 +850,83 @@ func (sc *syncContext) getSyncTasks() (_ syncTasks, successful bool) { return tasks, successful } +// processHookTasks applies additional logic to hook tasks. +func (sc *syncContext) processHookTasks(tasks syncTasks) { + for _, task := range tasks { + // This is a safety check to ensure that we process only currently running hook tasks. + if !task.isHook() || !task.pending() { + continue + } + // Safety check to ensure that the target object is not nil. + if task.targetObj == nil { + continue + } + // Currently, we only process hook tasks where the target object is a Job. + if task.targetObj.GetKind() == "Job" { + sc.processJobHookTask(task) + } + } +} + +// processJobHookTask processes a hook task where the target object is a Job and has defined ttlSecondsAfterFinished. +// This addresses the issue where a Job with a ttlSecondsAfterFinished set to a low value gets deleted fast and the hook phase gets stuck. +// For more info, see issue https://github.com/argoproj/argo-cd/issues/6880 +func (sc *syncContext) processJobHookTask(task *syncTask) { + hookFinalizer := "argoproj.io/hook-finalizer" + + task.postprocess = func() error { + sc.log.V(1).Info("Processing hook task with a Job resource - attaching hook finalizer", "name", task.targetObj.GetName(), "namespace", task.targetObj.GetNamespace()) + + job, err := sc.clientset.BatchV1().Jobs(task.targetObj.GetNamespace()).Get(context.TODO(), task.targetObj.GetName(), metav1.GetOptions{}) + if err != nil { + return err + } + + // Skip postprocessing if the Job does not have a ttlSecondsAfterFinished set. + if job.Spec.TTLSecondsAfterFinished == nil { + return nil + } + // Attach the hook finalizer to the Job resource so it does not get deleted before the sync phase is marked as completed. + job.Finalizers = append(job.Finalizers, hookFinalizer) + + _, err = sc.clientset. + BatchV1(). + Jobs(job.Namespace). + Update(context.TODO(), job, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + } + + task.cleanup = func() error { + sc.log.V(1).Info("Cleaning up hook task with a Job resource - removing hook finalizer", "name", task.targetObj.GetName(), "namespace", task.targetObj.GetNamespace()) + + job, err := sc.clientset.BatchV1().Jobs(task.targetObj.GetNamespace()).Get(context.TODO(), task.targetObj.GetName(), metav1.GetOptions{}) + if err != nil { + return err + } + + // Remove the hook finalizer from the Job resource. + var filtered []string + for _, s := range job.Finalizers { + if s != hookFinalizer { + filtered = append(filtered, s) + } + } + job.Finalizers = filtered + + _, err = sc.clientset. + BatchV1(). + Jobs(job.Namespace). + Update(context.TODO(), job, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + } +} + func (sc *syncContext) autoCreateNamespace(tasks syncTasks) syncTasks { isNamespaceCreationNeeded := true @@ -1007,6 +1104,11 @@ func (sc *syncContext) applyObject(t *syncTask, dryRun, validate bool) (common.R if err != nil { return common.ResultCodeSyncFailed, err.Error() } + if t.postprocess != nil && !dryRun { + if err := t.postprocess(); err != nil { + sc.log.Error(err, "failed to call postprocess function for task") + } + } if kube.IsCRD(t.targetObj) && !dryRun { crdName := t.targetObj.GetName() if err = sc.ensureCRDReady(crdName); err != nil { diff --git a/pkg/sync/sync_task.go b/pkg/sync/sync_task.go index 01c67a98b..289993ba7 100644 --- a/pkg/sync/sync_task.go +++ b/pkg/sync/sync_task.go @@ -24,6 +24,8 @@ type syncTask struct { operationState common.OperationPhase message string waveOverride *int + postprocess func() error + cleanup func() error } func ternary(val bool, a, b string) string { From 4e93b3e026c443214e6647bd5e50278c17e54b00 Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Mon, 9 Dec 2024 16:27:42 +0100 Subject: [PATCH 2/8] refactor logic for attaching and cleaning up hook finalizers to make it generic Signed-off-by: Dejan Zele Pejchev --- pkg/health/health.go | 14 +--- pkg/sync/hook/hook.go | 15 ++++ pkg/sync/sync_context.go | 153 +++++++++++++--------------------- pkg/sync/sync_context_test.go | 7 ++ pkg/sync/sync_task.go | 2 - 5 files changed, 82 insertions(+), 109 deletions(-) diff --git a/pkg/health/health.go b/pkg/health/health.go index 6cb37e879..f9b5022ff 100644 --- a/pkg/health/health.go +++ b/pkg/health/health.go @@ -1,6 +1,7 @@ package health import ( + "github.com/argoproj/gitops-engine/pkg/sync/hook" "github.com/argoproj/gitops-engine/pkg/utils/kube" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -64,7 +65,7 @@ func IsWorse(current, new HealthStatusCode) bool { // GetResourceHealth returns the health of a k8s resource func GetResourceHealth(obj *unstructured.Unstructured, healthOverride HealthOverride) (health *HealthStatus, err error) { - if obj.GetDeletionTimestamp() != nil && !hasHookFinalizer(obj) { + if obj.GetDeletionTimestamp() != nil && !hook.HasHookFinalizer(obj) { return &HealthStatus{ Status: HealthStatusProgressing, Message: "Pending deletion", @@ -97,17 +98,6 @@ func GetResourceHealth(obj *unstructured.Unstructured, healthOverride HealthOver } -func hasHookFinalizer(obj *unstructured.Unstructured) bool { - hookFinalizer := "argoproj.io/hook-finalizer" - finalizers := obj.GetFinalizers() - for _, finalizer := range finalizers { - if finalizer == hookFinalizer { - return true - } - } - return false -} - // GetHealthCheckFunc returns built-in health check function or nil if health check is not supported func GetHealthCheckFunc(gvk schema.GroupVersionKind) func(obj *unstructured.Unstructured) (*HealthStatus, error) { switch gvk.Group { diff --git a/pkg/sync/hook/hook.go b/pkg/sync/hook/hook.go index 7e0c38752..66dfc26e5 100644 --- a/pkg/sync/hook/hook.go +++ b/pkg/sync/hook/hook.go @@ -8,6 +8,21 @@ import ( resourceutil "github.com/argoproj/gitops-engine/pkg/sync/resource" ) +const ( + // HookFinalizer is the finalizer added to hooks to ensure they are deleted only after the sync phase is completed. + HookFinalizer = "argocd.argoproj.io/hook-finalizer" +) + +func HasHookFinalizer(obj *unstructured.Unstructured) bool { + finalizers := obj.GetFinalizers() + for _, finalizer := range finalizers { + if finalizer == HookFinalizer { + return true + } + } + return false +} + func IsHook(obj *unstructured.Unstructured) bool { _, ok := obj.GetAnnotations()[common.AnnotationKeyHook] if ok { diff --git a/pkg/sync/sync_context.go b/pkg/sync/sync_context.go index f01b34795..b9a3a433d 100644 --- a/pkg/sync/sync_context.go +++ b/pkg/sync/sync_context.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "k8s.io/client-go/kubernetes" "sort" "strings" "sync" @@ -229,10 +228,6 @@ func NewSyncContext( if err != nil { return nil, nil, err } - clientset, err := kubernetes.NewForConfig(restConfig) - if err != nil { - return nil, nil, err - } ctx := &syncContext{ revision: revision, resources: groupResources(reconciliationResult), @@ -241,7 +236,6 @@ func NewSyncContext( rawConfig: rawConfig, dynamicIf: dynamicIf, disco: disco, - clientset: clientset, extensionsclientset: extensionsclientset, kubectl: kubectl, resourceOps: resourceOps, @@ -337,7 +331,6 @@ type syncContext struct { dynamicIf dynamic.Interface disco discovery.DiscoveryInterface extensionsclientset *clientset.Clientset - clientset *kubernetes.Clientset kubectl kube.Kubectl resourceOps kube.ResourceOperations namespace string @@ -487,10 +480,8 @@ func (sc *syncContext) Sync() { return task.isHook() && task.completed() }) for _, task := range hooksCompleted { - if task.cleanup != nil { - if err := task.cleanup(); err != nil { - sc.log.V(1).Error(err, "failed to run hook task cleanup") - } + if err := sc.removeHookFinalizer(task); err != nil { + sc.setResourceResult(task, task.syncStatus, common.OperationError, fmt.Sprintf("Failed to remove hook finalizer: %v", err)) } } @@ -595,6 +586,56 @@ func (sc *syncContext) filterOutOfSyncTasks(tasks syncTasks) syncTasks { }) } +func (sc *syncContext) removeHookFinalizer(task *syncTask) error { + if task.liveObj == nil { + return nil + } + finalizers := task.targetObj.GetFinalizers() + var mutated bool + for i, finalizer := range finalizers { + if finalizer == hook.HookFinalizer { + finalizers = append(finalizers[:i], finalizers[i+1:]...) + mutated = true + break + } + } + if mutated { + task.targetObj.SetFinalizers(finalizers) + task.liveObj.SetFinalizers(finalizers) + // The cached live object may be stale in the controller cache, and the actual object may have been updated in the meantime, + // and Kubernetes API will return a conflict error on the Update call. + // In that case, we need to get the latest version of the object and retry the update. + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + updateErr := sc.updateResource(task) + if apierr.IsConflict(updateErr) { + sc.log.WithValues("task", task).V(1).Info("Retrying hook finalizer removal due to conflict on update") + resIf, err := sc.getResourceIf(task, "get") + if err != nil { + return err + } + liveObj, err := resIf.Get(context.TODO(), task.targetObj.GetName(), metav1.GetOptions{}) + if err != nil { + return err + } + task.liveObj = liveObj + } + return updateErr + }) + + } + return nil +} + +func (sc *syncContext) updateResource(task *syncTask) error { + sc.log.WithValues("task", task).V(1).Info("Updating resource") + resIf, err := sc.getResourceIf(task, "update") + if err != nil { + return err + } + _, err = resIf.Update(context.TODO(), task.liveObj, metav1.UpdateOptions{}) + return err +} + func (sc *syncContext) deleteHooks(hooksPendingDeletion syncTasks) { for _, task := range hooksPendingDeletion { err := sc.deleteResource(task) @@ -698,6 +739,7 @@ func (sc *syncContext) getSyncTasks() (_ syncTasks, successful bool) { generateName := obj.GetGenerateName() targetObj.SetName(fmt.Sprintf("%s%s", generateName, postfix)) } + targetObj.SetFinalizers(append(targetObj.GetFinalizers(), hook.HookFinalizer)) hookTasks = append(hookTasks, &syncTask{phase: phase, targetObj: targetObj}) } @@ -706,8 +748,6 @@ func (sc *syncContext) getSyncTasks() (_ syncTasks, successful bool) { sc.log.WithValues("hookTasks", hookTasks).V(1).Info("tasks from hooks") - sc.processHookTasks(hookTasks) - tasks := resourceTasks tasks = append(tasks, hookTasks...) @@ -850,83 +890,6 @@ func (sc *syncContext) getSyncTasks() (_ syncTasks, successful bool) { return tasks, successful } -// processHookTasks applies additional logic to hook tasks. -func (sc *syncContext) processHookTasks(tasks syncTasks) { - for _, task := range tasks { - // This is a safety check to ensure that we process only currently running hook tasks. - if !task.isHook() || !task.pending() { - continue - } - // Safety check to ensure that the target object is not nil. - if task.targetObj == nil { - continue - } - // Currently, we only process hook tasks where the target object is a Job. - if task.targetObj.GetKind() == "Job" { - sc.processJobHookTask(task) - } - } -} - -// processJobHookTask processes a hook task where the target object is a Job and has defined ttlSecondsAfterFinished. -// This addresses the issue where a Job with a ttlSecondsAfterFinished set to a low value gets deleted fast and the hook phase gets stuck. -// For more info, see issue https://github.com/argoproj/argo-cd/issues/6880 -func (sc *syncContext) processJobHookTask(task *syncTask) { - hookFinalizer := "argoproj.io/hook-finalizer" - - task.postprocess = func() error { - sc.log.V(1).Info("Processing hook task with a Job resource - attaching hook finalizer", "name", task.targetObj.GetName(), "namespace", task.targetObj.GetNamespace()) - - job, err := sc.clientset.BatchV1().Jobs(task.targetObj.GetNamespace()).Get(context.TODO(), task.targetObj.GetName(), metav1.GetOptions{}) - if err != nil { - return err - } - - // Skip postprocessing if the Job does not have a ttlSecondsAfterFinished set. - if job.Spec.TTLSecondsAfterFinished == nil { - return nil - } - // Attach the hook finalizer to the Job resource so it does not get deleted before the sync phase is marked as completed. - job.Finalizers = append(job.Finalizers, hookFinalizer) - - _, err = sc.clientset. - BatchV1(). - Jobs(job.Namespace). - Update(context.TODO(), job, metav1.UpdateOptions{}) - if err != nil { - return err - } - return nil - } - - task.cleanup = func() error { - sc.log.V(1).Info("Cleaning up hook task with a Job resource - removing hook finalizer", "name", task.targetObj.GetName(), "namespace", task.targetObj.GetNamespace()) - - job, err := sc.clientset.BatchV1().Jobs(task.targetObj.GetNamespace()).Get(context.TODO(), task.targetObj.GetName(), metav1.GetOptions{}) - if err != nil { - return err - } - - // Remove the hook finalizer from the Job resource. - var filtered []string - for _, s := range job.Finalizers { - if s != hookFinalizer { - filtered = append(filtered, s) - } - } - job.Finalizers = filtered - - _, err = sc.clientset. - BatchV1(). - Jobs(job.Namespace). - Update(context.TODO(), job, metav1.UpdateOptions{}) - if err != nil { - return err - } - return nil - } -} - func (sc *syncContext) autoCreateNamespace(tasks syncTasks) syncTasks { isNamespaceCreationNeeded := true @@ -1104,11 +1067,6 @@ func (sc *syncContext) applyObject(t *syncTask, dryRun, validate bool) (common.R if err != nil { return common.ResultCodeSyncFailed, err.Error() } - if t.postprocess != nil && !dryRun { - if err := t.postprocess(); err != nil { - sc.log.Error(err, "failed to call postprocess function for task") - } - } if kube.IsCRD(t.targetObj) && !dryRun { crdName := t.targetObj.GetName() if err = sc.ensureCRDReady(crdName); err != nil { @@ -1195,6 +1153,11 @@ func (sc *syncContext) Terminate() { if !task.isHook() || task.liveObj == nil { continue } + if err := sc.removeHookFinalizer(task); err != nil { + sc.setResourceResult(task, task.syncStatus, common.OperationError, fmt.Sprintf("Failed to remove hook finalizer: %v", err)) + terminateSuccessful = false + continue + } phase, msg, err := sc.getOperationPhase(task.liveObj) if err != nil { sc.setOperationPhase(common.OperationError, fmt.Sprintf("Failed to get hook health: %v", err)) diff --git a/pkg/sync/sync_context_test.go b/pkg/sync/sync_context_test.go index 7e416d20b..f9dd38e66 100644 --- a/pkg/sync/sync_context_test.go +++ b/pkg/sync/sync_context_test.go @@ -1362,6 +1362,12 @@ func TestRunSync_HooksDeletedAfterPhaseCompleted(t *testing.T) { )) fakeDynamicClient := fake.NewSimpleDynamicClient(runtime.NewScheme()) syncCtx.dynamicIf = fakeDynamicClient + // Each completed hook needs to have its hook finalizer removed in an Update call to the dynamic client. + updatedCount := 0 + fakeDynamicClient.PrependReactor("update", "*", func(action testcore.Action) (handled bool, ret runtime.Object, err error) { + updatedCount += 1 + return true, nil, nil + }) deletedCount := 0 fakeDynamicClient.PrependReactor("delete", "*", func(action testcore.Action) (handled bool, ret runtime.Object, err error) { deletedCount += 1 @@ -1381,6 +1387,7 @@ func TestRunSync_HooksDeletedAfterPhaseCompleted(t *testing.T) { assert.Equal(t, synccommon.OperationSucceeded, syncCtx.phase) assert.Equal(t, 2, deletedCount) + assert.Equal(t, 2, updatedCount) } func TestRunSync_HooksDeletedAfterPhaseCompletedFailed(t *testing.T) { diff --git a/pkg/sync/sync_task.go b/pkg/sync/sync_task.go index 289993ba7..01c67a98b 100644 --- a/pkg/sync/sync_task.go +++ b/pkg/sync/sync_task.go @@ -24,8 +24,6 @@ type syncTask struct { operationState common.OperationPhase message string waveOverride *int - postprocess func() error - cleanup func() error } func ternary(val bool, a, b string) string { From 3ab9d57c70c475ba3d61cf6e833de81982e63a35 Mon Sep 17 00:00:00 2001 From: Alexandre Gaudreault Date: Fri, 7 Feb 2025 15:23:58 -0500 Subject: [PATCH 3/8] fix conflict loop Signed-off-by: Alexandre Gaudreault --- pkg/sync/sync_context.go | 70 ++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/pkg/sync/sync_context.go b/pkg/sync/sync_context.go index 7caccb867..4029f62cf 100644 --- a/pkg/sync/sync_context.go +++ b/pkg/sync/sync_context.go @@ -590,40 +590,48 @@ func (sc *syncContext) removeHookFinalizer(task *syncTask) error { if task.liveObj == nil { return nil } - finalizers := task.targetObj.GetFinalizers() - var mutated bool - for i, finalizer := range finalizers { - if finalizer == hook.HookFinalizer { - finalizers = append(finalizers[:i], finalizers[i+1:]...) - mutated = true - break + removeFinalizerMMutation := func(obj *unstructured.Unstructured) bool { + finalizers := obj.GetFinalizers() + for i, finalizer := range finalizers { + if finalizer == hook.HookFinalizer { + obj.SetFinalizers(append(finalizers[:i], finalizers[i+1:]...)) + return true + } } + return false } - if mutated { - task.targetObj.SetFinalizers(finalizers) - task.liveObj.SetFinalizers(finalizers) - // The cached live object may be stale in the controller cache, and the actual object may have been updated in the meantime, - // and Kubernetes API will return a conflict error on the Update call. - // In that case, we need to get the latest version of the object and retry the update. - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - updateErr := sc.updateResource(task) - if apierr.IsConflict(updateErr) { - sc.log.WithValues("task", task).V(1).Info("Retrying hook finalizer removal due to conflict on update") - resIf, err := sc.getResourceIf(task, "get") - if err != nil { - return err - } - liveObj, err := resIf.Get(context.TODO(), task.targetObj.GetName(), metav1.GetOptions{}) - if err != nil { - return err - } - task.liveObj = liveObj - } - return updateErr - }) - } - return nil + // The cached live object may be stale in the controller cache, and the actual object may have been updated in the meantime, + // and Kubernetes API will return a conflict error on the Update call. + // In that case, we need to get the latest version of the object and retry the update. + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + mutated := removeFinalizerMMutation(task.liveObj) + if !mutated { + return nil + } + + updateErr := sc.updateResource(task) + if apierr.IsConflict(updateErr) { + sc.log.WithValues("task", task).V(1).Info("Retrying hook finalizer removal due to conflict on update") + resIf, err := sc.getResourceIf(task, "get") + if err != nil { + return err + } + liveObj, err := resIf.Get(context.TODO(), task.liveObj.GetName(), metav1.GetOptions{}) + if apierr.IsNotFound(err) { + sc.log.WithValues("task", task).V(1).Info("Resource is already deleted") + return nil + } else if err != nil { + return err + } + task.liveObj = liveObj + } else if apierr.IsNotFound(updateErr) { + // If the resource is already deleted, it is a no-op + sc.log.WithValues("task", task).V(1).Info("Resource is already deleted") + return nil + } + return updateErr + }) } func (sc *syncContext) updateResource(task *syncTask) error { From 6885341b6acdca98569ea260c4a8b79126d5b359 Mon Sep 17 00:00:00 2001 From: Alexandre Gaudreault Date: Fri, 7 Feb 2025 15:27:27 -0500 Subject: [PATCH 4/8] fix failed message Signed-off-by: Alexandre Gaudreault --- pkg/sync/sync_context.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/sync/sync_context.go b/pkg/sync/sync_context.go index 4029f62cf..19da2c412 100644 --- a/pkg/sync/sync_context.go +++ b/pkg/sync/sync_context.go @@ -296,12 +296,12 @@ const ( crdReadinessTimeout = time.Duration(3) * time.Second ) -// getOperationPhase returns a hook status from an _live_ unstructured object -func (sc *syncContext) getOperationPhase(hook *unstructured.Unstructured) (common.OperationPhase, string, error) { +// getOperationPhase returns a health status from a _live_ unstructured object +func (sc *syncContext) getOperationPhase(obj *unstructured.Unstructured) (common.OperationPhase, string, error) { phase := common.OperationSucceeded - message := fmt.Sprintf("%s created", hook.GetName()) + message := fmt.Sprintf("%s created", obj.GetName()) - resHealth, err := health.GetResourceHealth(hook, sc.healthOverride) + resHealth, err := health.GetResourceHealth(obj, sc.healthOverride) if err != nil { return "", "", err } @@ -665,8 +665,8 @@ func (sc *syncContext) GetState() (common.OperationPhase, string, []common.Resou } func (sc *syncContext) setOperationFailed(syncFailTasks, syncFailedTasks syncTasks, message string) { - errorMessageFactory := func(_ []*syncTask, message string) string { - messages := syncFailedTasks.Map(func(task *syncTask) string { + errorMessageFactory := func(tasks syncTasks, message string) string { + messages := tasks.Map(func(task *syncTask) string { return task.message }) if len(messages) > 0 { @@ -687,7 +687,9 @@ func (sc *syncContext) setOperationFailed(syncFailTasks, syncFailedTasks syncTas // the phase, so we make sure we have at least one more sync sc.log.WithValues("syncFailTasks", syncFailTasks).V(1).Info("Running sync fail tasks") if sc.runTasks(syncFailTasks, false) == failed { - sc.setOperationPhase(common.OperationFailed, errorMessage) + failedSyncFailTasks := syncFailTasks.Filter(func(t *syncTask) bool { return t.syncStatus == common.ResultCodeSyncFailed }) + syncFailTasksMessage := errorMessageFactory(failedSyncFailTasks, "one or more SyncFail hooks failed") + sc.setOperationPhase(common.OperationFailed, fmt.Sprintf("%s\n%s", errorMessage, syncFailTasksMessage)) } } else { sc.setOperationPhase(common.OperationFailed, errorMessage) @@ -747,8 +749,9 @@ func (sc *syncContext) getSyncTasks() (_ syncTasks, successful bool) { generateName := obj.GetGenerateName() targetObj.SetName(fmt.Sprintf("%s%s", generateName, postfix)) } - targetObj.SetFinalizers(append(targetObj.GetFinalizers(), hook.HookFinalizer)) - + if !hook.HasHookFinalizer(targetObj) { + targetObj.SetFinalizers(append(targetObj.GetFinalizers(), hook.HookFinalizer)) + } hookTasks = append(hookTasks, &syncTask{phase: phase, targetObj: targetObj}) } } From d161c394e668a4dfb5fa4d3a60f094cf21063c88 Mon Sep 17 00:00:00 2001 From: Alexandre Gaudreault Date: Fri, 7 Feb 2025 16:03:39 -0500 Subject: [PATCH 5/8] terminate running hooks Signed-off-by: Alexandre Gaudreault --- pkg/sync/sync_context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sync/sync_context.go b/pkg/sync/sync_context.go index 19da2c412..1525abda2 100644 --- a/pkg/sync/sync_context.go +++ b/pkg/sync/sync_context.go @@ -1175,7 +1175,7 @@ func (sc *syncContext) Terminate() { } if phase == common.OperationRunning { err := sc.deleteResource(task) - if err != nil { + if err != nil && !apierr.IsNotFound(err) { sc.setResourceResult(task, "", common.OperationFailed, fmt.Sprintf("Failed to delete: %v", err)) terminateSuccessful = false } else { From e5102c0ff89033e33f2cf4bffb47ff9073c7fd3b Mon Sep 17 00:00:00 2001 From: Alexandre Gaudreault Date: Fri, 7 Feb 2025 16:36:15 -0500 Subject: [PATCH 6/8] unit tests Signed-off-by: Alexandre Gaudreault --- pkg/sync/sync_context.go | 4 +- pkg/sync/sync_context_test.go | 91 +++++++++++++++++++++++------------ 2 files changed, 61 insertions(+), 34 deletions(-) diff --git a/pkg/sync/sync_context.go b/pkg/sync/sync_context.go index 1525abda2..0e679f846 100644 --- a/pkg/sync/sync_context.go +++ b/pkg/sync/sync_context.go @@ -590,7 +590,7 @@ func (sc *syncContext) removeHookFinalizer(task *syncTask) error { if task.liveObj == nil { return nil } - removeFinalizerMMutation := func(obj *unstructured.Unstructured) bool { + removeFinalizerMutation := func(obj *unstructured.Unstructured) bool { finalizers := obj.GetFinalizers() for i, finalizer := range finalizers { if finalizer == hook.HookFinalizer { @@ -605,7 +605,7 @@ func (sc *syncContext) removeHookFinalizer(task *syncTask) error { // and Kubernetes API will return a conflict error on the Update call. // In that case, we need to get the latest version of the object and retry the update. return retry.RetryOnConflict(retry.DefaultRetry, func() error { - mutated := removeFinalizerMMutation(task.liveObj) + mutated := removeFinalizerMutation(task.liveObj) if !mutated { return nil } diff --git a/pkg/sync/sync_context_test.go b/pkg/sync/sync_context_test.go index e59bf4f80..8525553a9 100644 --- a/pkg/sync/sync_context_test.go +++ b/pkg/sync/sync_context_test.go @@ -32,6 +32,7 @@ import ( "github.com/argoproj/gitops-engine/pkg/health" "github.com/argoproj/gitops-engine/pkg/sync/common" synccommon "github.com/argoproj/gitops-engine/pkg/sync/common" + "github.com/argoproj/gitops-engine/pkg/sync/hook" "github.com/argoproj/gitops-engine/pkg/utils/kube" "github.com/argoproj/gitops-engine/pkg/utils/kube/kubetest" . "github.com/argoproj/gitops-engine/pkg/utils/testing" @@ -1288,15 +1289,19 @@ func (r resourceNameHealthOverride) GetResourceHealth(obj *unstructured.Unstruct } func TestRunSync_HooksNotDeletedIfPhaseNotCompleted(t *testing.T) { - completedHook := newHook(synccommon.HookTypePreSync) - completedHook.SetName("completed-hook") - completedHook.SetNamespace(FakeArgoCDNamespace) - _ = Annotate(completedHook, synccommon.AnnotationKeyHookDeletePolicy, "HookSucceeded") - - inProgressHook := newHook(synccommon.HookTypePreSync) - inProgressHook.SetNamespace(FakeArgoCDNamespace) - inProgressHook.SetName("in-progress-hook") - _ = Annotate(inProgressHook, synccommon.AnnotationKeyHookDeletePolicy, "HookSucceeded") + hook1 := newHook(synccommon.HookTypePreSync) + hook1.SetName("completed-hook") + hook1.SetNamespace(FakeArgoCDNamespace) + _ = Annotate(hook1, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookSucceeded)) + completedHook := hook1.DeepCopy() + completedHook.SetFinalizers(append(completedHook.GetFinalizers(), hook.HookFinalizer)) + + hook2 := newHook(synccommon.HookTypePreSync) + hook2.SetNamespace(FakeArgoCDNamespace) + hook2.SetName("in-progress-hook") + _ = Annotate(hook2, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookSucceeded)) + inProgressHook := hook2.DeepCopy() + inProgressHook.SetFinalizers(append(inProgressHook.GetFinalizers(), hook.HookFinalizer)) syncCtx := newTestSyncCtx(nil, WithHealthOverride(resourceNameHealthOverride(map[string]health.HealthStatusCode{ @@ -1315,6 +1320,12 @@ func TestRunSync_HooksNotDeletedIfPhaseNotCompleted(t *testing.T) { )) fakeDynamicClient := fake.NewSimpleDynamicClient(runtime.NewScheme()) syncCtx.dynamicIf = fakeDynamicClient + updatedCount := 0 + fakeDynamicClient.PrependReactor("update", "*", func(action testcore.Action) (handled bool, ret runtime.Object, err error) { + // Removing the finalizers + updatedCount += 1 + return true, nil, nil + }) deletedCount := 0 fakeDynamicClient.PrependReactor("delete", "*", func(_ testcore.Action) (handled bool, ret runtime.Object, err error) { deletedCount += 1 @@ -1324,7 +1335,7 @@ func TestRunSync_HooksNotDeletedIfPhaseNotCompleted(t *testing.T) { Live: []*unstructured.Unstructured{completedHook, inProgressHook}, Target: []*unstructured.Unstructured{nil, nil}, }) - syncCtx.hooks = []*unstructured.Unstructured{completedHook, inProgressHook} + syncCtx.hooks = []*unstructured.Unstructured{hook1, hook2} syncCtx.kubectl = &kubetest.MockKubectlCmd{ Commands: map[string]kubetest.KubectlOutput{}, @@ -1333,19 +1344,24 @@ func TestRunSync_HooksNotDeletedIfPhaseNotCompleted(t *testing.T) { syncCtx.Sync() assert.Equal(t, synccommon.OperationRunning, syncCtx.phase) + assert.Equal(t, 0, updatedCount) assert.Equal(t, 0, deletedCount) } func TestRunSync_HooksDeletedAfterPhaseCompleted(t *testing.T) { - completedHook1 := newHook(synccommon.HookTypePreSync) - completedHook1.SetName("completed-hook1") - completedHook1.SetNamespace(FakeArgoCDNamespace) - _ = Annotate(completedHook1, synccommon.AnnotationKeyHookDeletePolicy, "HookSucceeded") - - completedHook2 := newHook(synccommon.HookTypePreSync) - completedHook2.SetNamespace(FakeArgoCDNamespace) - completedHook2.SetName("completed-hook2") - _ = Annotate(completedHook2, synccommon.AnnotationKeyHookDeletePolicy, "HookSucceeded") + hook1 := newHook(synccommon.HookTypePreSync) + hook1.SetName("completed-hook1") + hook1.SetNamespace(FakeArgoCDNamespace) + _ = Annotate(hook1, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookSucceeded)) + completedHook1 := hook1.DeepCopy() + completedHook1.SetFinalizers(append(completedHook1.GetFinalizers(), hook.HookFinalizer)) + + hook2 := newHook(synccommon.HookTypePreSync) + hook2.SetNamespace(FakeArgoCDNamespace) + hook2.SetName("completed-hook2") + _ = Annotate(hook2, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookSucceeded)) + completedHook2 := hook2.DeepCopy() + completedHook2.SetFinalizers(append(completedHook1.GetFinalizers(), hook.HookFinalizer)) syncCtx := newTestSyncCtx(nil, WithInitialState(synccommon.OperationRunning, "", []synccommon.ResourceSyncResult{{ @@ -1361,9 +1377,9 @@ func TestRunSync_HooksDeletedAfterPhaseCompleted(t *testing.T) { )) fakeDynamicClient := fake.NewSimpleDynamicClient(runtime.NewScheme()) syncCtx.dynamicIf = fakeDynamicClient - // Each completed hook needs to have its hook finalizer removed in an Update call to the dynamic client. updatedCount := 0 fakeDynamicClient.PrependReactor("update", "*", func(action testcore.Action) (handled bool, ret runtime.Object, err error) { + // Removing the finalizers updatedCount += 1 return true, nil, nil }) @@ -1376,7 +1392,7 @@ func TestRunSync_HooksDeletedAfterPhaseCompleted(t *testing.T) { Live: []*unstructured.Unstructured{completedHook1, completedHook2}, Target: []*unstructured.Unstructured{nil, nil}, }) - syncCtx.hooks = []*unstructured.Unstructured{completedHook1, completedHook2} + syncCtx.hooks = []*unstructured.Unstructured{hook1, hook2} syncCtx.kubectl = &kubetest.MockKubectlCmd{ Commands: map[string]kubetest.KubectlOutput{}, @@ -1385,20 +1401,24 @@ func TestRunSync_HooksDeletedAfterPhaseCompleted(t *testing.T) { syncCtx.Sync() assert.Equal(t, synccommon.OperationSucceeded, syncCtx.phase) - assert.Equal(t, 2, deletedCount) assert.Equal(t, 2, updatedCount) + assert.Equal(t, 2, deletedCount) } func TestRunSync_HooksDeletedAfterPhaseCompletedFailed(t *testing.T) { - completedHook1 := newHook(synccommon.HookTypeSync) - completedHook1.SetName("completed-hook1") - completedHook1.SetNamespace(FakeArgoCDNamespace) - _ = Annotate(completedHook1, synccommon.AnnotationKeyHookDeletePolicy, "HookFailed") - - completedHook2 := newHook(synccommon.HookTypeSync) - completedHook2.SetNamespace(FakeArgoCDNamespace) - completedHook2.SetName("completed-hook2") - _ = Annotate(completedHook2, synccommon.AnnotationKeyHookDeletePolicy, "HookFailed") + hook1 := newHook(synccommon.HookTypeSync) + hook1.SetName("completed-hook1") + hook1.SetNamespace(FakeArgoCDNamespace) + _ = Annotate(hook1, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookFailed)) + completedHook1 := hook1.DeepCopy() + completedHook1.SetFinalizers(append(completedHook1.GetFinalizers(), hook.HookFinalizer)) + + hook2 := newHook(synccommon.HookTypeSync) + hook2.SetNamespace(FakeArgoCDNamespace) + hook2.SetName("completed-hook2") + _ = Annotate(hook2, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookFailed)) + completedHook2 := hook2.DeepCopy() + completedHook2.SetFinalizers(append(completedHook1.GetFinalizers(), hook.HookFinalizer)) syncCtx := newTestSyncCtx(nil, WithInitialState(synccommon.OperationRunning, "", []synccommon.ResourceSyncResult{{ @@ -1414,6 +1434,12 @@ func TestRunSync_HooksDeletedAfterPhaseCompletedFailed(t *testing.T) { )) fakeDynamicClient := fake.NewSimpleDynamicClient(runtime.NewScheme()) syncCtx.dynamicIf = fakeDynamicClient + updatedCount := 0 + fakeDynamicClient.PrependReactor("update", "*", func(action testcore.Action) (handled bool, ret runtime.Object, err error) { + // Removing the finalizers + updatedCount += 1 + return true, nil, nil + }) deletedCount := 0 fakeDynamicClient.PrependReactor("delete", "*", func(_ testcore.Action) (handled bool, ret runtime.Object, err error) { deletedCount += 1 @@ -1423,7 +1449,7 @@ func TestRunSync_HooksDeletedAfterPhaseCompletedFailed(t *testing.T) { Live: []*unstructured.Unstructured{completedHook1, completedHook2}, Target: []*unstructured.Unstructured{nil, nil}, }) - syncCtx.hooks = []*unstructured.Unstructured{completedHook1, completedHook2} + syncCtx.hooks = []*unstructured.Unstructured{hook1, hook2} syncCtx.kubectl = &kubetest.MockKubectlCmd{ Commands: map[string]kubetest.KubectlOutput{}, @@ -1432,6 +1458,7 @@ func TestRunSync_HooksDeletedAfterPhaseCompletedFailed(t *testing.T) { syncCtx.Sync() assert.Equal(t, synccommon.OperationFailed, syncCtx.phase) + assert.Equal(t, 2, updatedCount) assert.Equal(t, 2, deletedCount) } From d14f10bc8fbecddb190e0286f0b4e9b4eea7bb1b Mon Sep 17 00:00:00 2001 From: Alexandre Gaudreault Date: Fri, 7 Feb 2025 16:59:02 -0500 Subject: [PATCH 7/8] linting Signed-off-by: Alexandre Gaudreault --- pkg/sync/sync_context.go | 2 +- pkg/sync/sync_context_test.go | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/pkg/sync/sync_context.go b/pkg/sync/sync_context.go index 4690ed704..7d43899c9 100644 --- a/pkg/sync/sync_context.go +++ b/pkg/sync/sync_context.go @@ -298,7 +298,7 @@ const ( // getOperationPhase returns a health status from a _live_ unstructured object func (sc *syncContext) getOperationPhase(obj *unstructured.Unstructured) (common.OperationPhase, string, error) { phase := common.OperationSucceeded - message := fmt.Sprintf("%s created", obj.GetName()) + message := obj.GetName() + " created" resHealth, err := health.GetResourceHealth(obj, sc.healthOverride) if err != nil { diff --git a/pkg/sync/sync_context_test.go b/pkg/sync/sync_context_test.go index c3adf6310..21f9730bb 100644 --- a/pkg/sync/sync_context_test.go +++ b/pkg/sync/sync_context_test.go @@ -28,7 +28,6 @@ import ( "github.com/argoproj/gitops-engine/pkg/diff" "github.com/argoproj/gitops-engine/pkg/health" - "github.com/argoproj/gitops-engine/pkg/sync/common" synccommon "github.com/argoproj/gitops-engine/pkg/sync/common" "github.com/argoproj/gitops-engine/pkg/sync/hook" "github.com/argoproj/gitops-engine/pkg/utils/kube" @@ -1290,14 +1289,14 @@ func TestRunSync_HooksNotDeletedIfPhaseNotCompleted(t *testing.T) { hook1 := newHook(synccommon.HookTypePreSync) hook1.SetName("completed-hook") hook1.SetNamespace(testingutils.FakeArgoCDNamespace) - _ = testingutils.Annotate(hook1, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookSucceeded)) + _ = testingutils.Annotate(hook1, synccommon.AnnotationKeyHookDeletePolicy, string(synccommon.HookDeletePolicyHookSucceeded)) completedHook := hook1.DeepCopy() completedHook.SetFinalizers(append(completedHook.GetFinalizers(), hook.HookFinalizer)) hook2 := newHook(synccommon.HookTypePreSync) hook2.SetNamespace(testingutils.FakeArgoCDNamespace) hook2.SetName("in-progress-hook") - _ = testingutils.Annotate(hook2, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookSucceeded)) + _ = testingutils.Annotate(hook2, synccommon.AnnotationKeyHookDeletePolicy, string(synccommon.HookDeletePolicyHookSucceeded)) inProgressHook := hook2.DeepCopy() inProgressHook.SetFinalizers(append(inProgressHook.GetFinalizers(), hook.HookFinalizer)) @@ -1319,9 +1318,9 @@ func TestRunSync_HooksNotDeletedIfPhaseNotCompleted(t *testing.T) { fakeDynamicClient := fake.NewSimpleDynamicClient(runtime.NewScheme()) syncCtx.dynamicIf = fakeDynamicClient updatedCount := 0 - fakeDynamicClient.PrependReactor("update", "*", func(action testcore.Action) (handled bool, ret runtime.Object, err error) { + fakeDynamicClient.PrependReactor("update", "*", func(_ testcore.Action) (handled bool, ret runtime.Object, err error) { // Removing the finalizers - updatedCount += 1 + updatedCount++ return true, nil, nil }) deletedCount := 0 @@ -1350,14 +1349,14 @@ func TestRunSync_HooksDeletedAfterPhaseCompleted(t *testing.T) { hook1 := newHook(synccommon.HookTypePreSync) hook1.SetName("completed-hook1") hook1.SetNamespace(testingutils.FakeArgoCDNamespace) - _ = testingutils.Annotate(hook1, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookSucceeded)) + _ = testingutils.Annotate(hook1, synccommon.AnnotationKeyHookDeletePolicy, string(synccommon.HookDeletePolicyHookSucceeded)) completedHook1 := hook1.DeepCopy() completedHook1.SetFinalizers(append(completedHook1.GetFinalizers(), hook.HookFinalizer)) hook2 := newHook(synccommon.HookTypePreSync) hook2.SetNamespace(testingutils.FakeArgoCDNamespace) hook2.SetName("completed-hook2") - _ = testingutils.Annotate(hook2, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookSucceeded)) + _ = testingutils.Annotate(hook2, synccommon.AnnotationKeyHookDeletePolicy, string(synccommon.HookDeletePolicyHookSucceeded)) completedHook2 := hook2.DeepCopy() completedHook2.SetFinalizers(append(completedHook1.GetFinalizers(), hook.HookFinalizer)) @@ -1376,9 +1375,9 @@ func TestRunSync_HooksDeletedAfterPhaseCompleted(t *testing.T) { fakeDynamicClient := fake.NewSimpleDynamicClient(runtime.NewScheme()) syncCtx.dynamicIf = fakeDynamicClient updatedCount := 0 - fakeDynamicClient.PrependReactor("update", "*", func(action testcore.Action) (handled bool, ret runtime.Object, err error) { + fakeDynamicClient.PrependReactor("update", "*", func(_ testcore.Action) (handled bool, ret runtime.Object, err error) { // Removing the finalizers - updatedCount += 1 + updatedCount++ return true, nil, nil }) deletedCount := 0 @@ -1407,14 +1406,14 @@ func TestRunSync_HooksDeletedAfterPhaseCompletedFailed(t *testing.T) { hook1 := newHook(synccommon.HookTypeSync) hook1.SetName("completed-hook1") hook1.SetNamespace(testingutils.FakeArgoCDNamespace) - _ = testingutils.Annotate(hook1, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookFailed)) + _ = testingutils.Annotate(hook1, synccommon.AnnotationKeyHookDeletePolicy, string(synccommon.HookDeletePolicyHookFailed)) completedHook1 := hook1.DeepCopy() completedHook1.SetFinalizers(append(completedHook1.GetFinalizers(), hook.HookFinalizer)) hook2 := newHook(synccommon.HookTypeSync) hook2.SetNamespace(testingutils.FakeArgoCDNamespace) hook2.SetName("completed-hook2") - _ = testingutils.Annotate(hook2, synccommon.AnnotationKeyHookDeletePolicy, string(common.HookDeletePolicyHookFailed)) + _ = testingutils.Annotate(hook2, synccommon.AnnotationKeyHookDeletePolicy, string(synccommon.HookDeletePolicyHookFailed)) completedHook2 := hook2.DeepCopy() completedHook2.SetFinalizers(append(completedHook1.GetFinalizers(), hook.HookFinalizer)) @@ -1433,9 +1432,9 @@ func TestRunSync_HooksDeletedAfterPhaseCompletedFailed(t *testing.T) { fakeDynamicClient := fake.NewSimpleDynamicClient(runtime.NewScheme()) syncCtx.dynamicIf = fakeDynamicClient updatedCount := 0 - fakeDynamicClient.PrependReactor("update", "*", func(action testcore.Action) (handled bool, ret runtime.Object, err error) { + fakeDynamicClient.PrependReactor("update", "*", func(_ testcore.Action) (handled bool, ret runtime.Object, err error) { // Removing the finalizers - updatedCount += 1 + updatedCount++ return true, nil, nil }) deletedCount := 0 From 4afcb8475ac686def6dd2cb6a4f76addb83510ec Mon Sep 17 00:00:00 2001 From: Alexandre Gaudreault Date: Fri, 7 Feb 2025 16:59:34 -0500 Subject: [PATCH 8/8] lint Signed-off-by: Alexandre Gaudreault --- pkg/health/health.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/health/health.go b/pkg/health/health.go index d2107bd4e..c615deea9 100644 --- a/pkg/health/health.go +++ b/pkg/health/health.go @@ -1,10 +1,11 @@ package health import ( - "github.com/argoproj/gitops-engine/pkg/sync/hook" - "github.com/argoproj/gitops-engine/pkg/utils/kube" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/argoproj/gitops-engine/pkg/sync/hook" + "github.com/argoproj/gitops-engine/pkg/utils/kube" ) // Represents resource health status