From 278e084275cf045d274f17c3337efd8bd4eecec3 Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Thu, 20 Jul 2023 23:17:10 +0200 Subject: [PATCH 1/6] disable chatty spans Signed-off-by: Matthias Bertschy --- pkg/filehandler/v1/file_handler.go | 4 +--- pkg/relevancymanager/v1/relevancy_manager.go | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/filehandler/v1/file_handler.go b/pkg/filehandler/v1/file_handler.go index 1f0f18cf36..7e9d5ef5e3 100644 --- a/pkg/filehandler/v1/file_handler.go +++ b/pkg/filehandler/v1/file_handler.go @@ -25,9 +25,7 @@ func CreateBoltFileHandler() (*BoltFileHandler, error) { return &BoltFileHandler{fileDB: db}, nil } -func (b BoltFileHandler) AddFile(ctx context.Context, bucket, file string) error { - _, span := otel.Tracer("").Start(ctx, "BoltFileHandler.AddFile") - defer span.End() +func (b BoltFileHandler) AddFile(_ context.Context, bucket, file string) error { return b.fileDB.Batch(func(tx *bolt.Tx) error { b, err := tx.CreateBucketIfNotExists([]byte(bucket)) if err != nil { diff --git a/pkg/relevancymanager/v1/relevancy_manager.go b/pkg/relevancymanager/v1/relevancy_manager.go index fa3a834e3c..f05c62a57f 100644 --- a/pkg/relevancymanager/v1/relevancy_manager.go +++ b/pkg/relevancymanager/v1/relevancy_manager.go @@ -344,8 +344,6 @@ func (rm *RelevancyManager) ReportContainerTerminated(ctx context.Context, conta } func (rm *RelevancyManager) ReportFileAccess(ctx context.Context, namespace, pod, container, file string) { - ctx, span := otel.Tracer("").Start(ctx, "RelevancyManager.ReportFileAccess") - defer span.End() // log accessed files for all containers to avoid race condition // this won't record unnecessary containers as the containerCollection takes care of filtering them if file == "" { From 51a28cd49fb33e1b02f3173fb57a8ab706735ab2 Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Fri, 21 Jul 2023 10:58:28 +0200 Subject: [PATCH 2/6] add pprof Signed-off-by: Matthias Bertschy --- main.go | 7 +++++++ pkg/sbom/sbom_test.go | 20 +++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index c758b45456..20e8662281 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "log" + "net/http" "net/url" "node-agent/internal/validator" "node-agent/pkg/config" @@ -14,6 +15,8 @@ import ( "os/signal" "syscall" + _ "net/http/pprof" + "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" "github.com/kubescape/k8s-interface/k8sinterface" @@ -23,6 +26,10 @@ import ( func main() { ctx := context.Background() + go func() { + log.Println(http.ListenAndServe("localhost:6060", nil)) + }() + cfg, err := config.LoadConfig("/etc/config") if err != nil { logger.L().Ctx(ctx).Fatal("load config error", helpers.Error(err)) diff --git a/pkg/sbom/sbom_test.go b/pkg/sbom/sbom_test.go index 43f949b981..9914d17772 100644 --- a/pkg/sbom/sbom_test.go +++ b/pkg/sbom/sbom_test.go @@ -2,6 +2,7 @@ package sbom import ( "context" + "math/rand" "node-agent/pkg/storageclient" "testing" @@ -18,15 +19,28 @@ func TestGetSBOM(t *testing.T) { } +const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +// generate a random string of given length +func randStringBytes(n int) string { + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] //pick a random letter from the letterBytes + } + return string(b) +} + func TestFilterSBOM(t *testing.T) { SBOMClient := CreateSBOMStorageClient(storageclient.CreateSBOMStorageHttpClientMock(), "", &instanceidhandler.InstanceID{}, afero.NewMemMapFs()) err := SBOMClient.GetSBOM(context.TODO(), storageclient.NGINX_IMAGE_TAG, storageclient.NGINX) if err != nil { t.Fatalf("fail to get sbom, %v", err) } - err = SBOMClient.FilterSBOM(context.TODO(), map[string]bool{ - "/usr/share/adduser/adduser.conf": true, - }) + m := make(map[string]bool) + for i := 0; i < 10000; i++ { + m["/tmp/"+randStringBytes(10)] = true + } + err = SBOMClient.FilterSBOM(context.TODO(), m) if err != nil { t.Fatalf("fail to filter sbom, %v", err) } From 1f0a9f83ea68c695053784e729609ffe51467fab Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Sat, 22 Jul 2023 11:30:06 +0200 Subject: [PATCH 3/6] move workerpool to relevancymanager Signed-off-by: Matthias Bertschy --- pkg/containerwatcher/v1/container_watcher.go | 12 ++---------- pkg/relevancymanager/v1/relevancy_manager.go | 16 +++++++++++----- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/pkg/containerwatcher/v1/container_watcher.go b/pkg/containerwatcher/v1/container_watcher.go index c9d3b93857..6e386aa428 100644 --- a/pkg/containerwatcher/v1/container_watcher.go +++ b/pkg/containerwatcher/v1/container_watcher.go @@ -9,7 +9,6 @@ import ( "os" "time" - "github.com/gammazero/workerpool" containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection" tracerexec "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/exec/tracer" tracerexectype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/exec/types" @@ -24,7 +23,6 @@ import ( ) const ( - concurrency = 4 execTraceName = "trace_exec" openTraceName = "trace_open" ) @@ -36,7 +34,6 @@ type IGContainerWatcher struct { tracerCollection *tracercollection.TracerCollection tracerExec *tracerexec.Tracer tracerOpen *traceropen.Tracer - workerPool *workerpool.WorkerPool } var _ containerwatcher.ContainerWatcher = (*IGContainerWatcher)(nil) @@ -55,7 +52,6 @@ func CreateIGContainerWatcher(k8sClient *k8sinterface.KubernetesApi, relevancyMa k8sClient: k8sClient, tracerCollection: tracerCollection, relevancyManager: relevancyManager, - workerPool: workerpool.New(concurrency), }, nil } @@ -121,9 +117,7 @@ func (ch *IGContainerWatcher) Start(ctx context.Context) error { if len(event.Args) > 0 { procImageName = event.Args[0] } - ch.workerPool.Submit(func() { - ch.relevancyManager.ReportFileAccess(ctx, event.Namespace, event.Pod, event.Container, procImageName) - }) + ch.relevancyManager.ReportFileAccess(ctx, event.Namespace, event.Pod, event.Container, procImageName) } } if err := ch.tracerCollection.AddTracer(execTraceName, containerSelector); err != nil { @@ -138,9 +132,7 @@ func (ch *IGContainerWatcher) Start(ctx context.Context) error { return } if event.Ret > -1 { - ch.workerPool.Submit(func() { - ch.relevancyManager.ReportFileAccess(ctx, event.Namespace, event.Pod, event.Container, event.Path) - }) + ch.relevancyManager.ReportFileAccess(ctx, event.Namespace, event.Pod, event.Container, event.Path) } } if err := ch.tracerCollection.AddTracer(openTraceName, containerSelector); err != nil { diff --git a/pkg/relevancymanager/v1/relevancy_manager.go b/pkg/relevancymanager/v1/relevancy_manager.go index f05c62a57f..77d6bb4d85 100644 --- a/pkg/relevancymanager/v1/relevancy_manager.go +++ b/pkg/relevancymanager/v1/relevancy_manager.go @@ -16,6 +16,7 @@ import ( "time" "github.com/armosec/utils-k8s-go/wlid" + "github.com/gammazero/workerpool" containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection" "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" @@ -30,6 +31,7 @@ import ( ) const ( + concurrency = 4 RelevantCVEsService = "RelevantCVEsService" StepGetSBOM = "StepGetSBOM" StepValidateSBOM = "StepValidateSBOM" @@ -51,6 +53,7 @@ type RelevancyManager struct { sbomFs afero.Fs storageClient storageclient.StorageClient watchedContainers sync.Map + workerPool *workerpool.WorkerPool } var _ relevancymanager.RelevancyManagerClient = (*RelevancyManager)(nil) @@ -65,6 +68,7 @@ func CreateRelevancyManager(cfg config.Config, clusterName string, fileHandler f sbomFs: sbomFs, storageClient: storageClient, watchedContainers: sync.Map{}, + workerPool: workerpool.New(concurrency), }, nil } @@ -349,11 +353,13 @@ func (rm *RelevancyManager) ReportFileAccess(ctx context.Context, namespace, pod if file == "" { return } - k8sContainerID := utils.CreateK8sContainerID(namespace, pod, container) - err := rm.fileHandler.AddFile(ctx, k8sContainerID, file) - if err != nil { - logger.L().Ctx(ctx).Error("failed to add file to container file list", helpers.Error(err), helpers.Interface("k8sContainerID", k8sContainerID), helpers.String("file", file)) - } + rm.workerPool.Submit(func() { + k8sContainerID := utils.CreateK8sContainerID(namespace, pod, container) + err := rm.fileHandler.AddFile(ctx, k8sContainerID, file) + if err != nil { + logger.L().Ctx(ctx).Error("failed to add file to container file list", helpers.Error(err), helpers.Interface("k8sContainerID", k8sContainerID), helpers.String("file", file)) + } + }) } func (rm *RelevancyManager) SetContainerHandler(containerHandler containerwatcher.ContainerWatcher) { From 87565f7423d4201b7c84cc12a7b5e02b6d89af8d Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Sat, 22 Jul 2023 16:12:43 +0200 Subject: [PATCH 4/6] remove context from callback func Signed-off-by: Matthias Bertschy --- pkg/containerwatcher/v1/container_watcher.go | 8 ++++---- pkg/filehandler/file_handler_interface.go | 2 +- pkg/filehandler/v1/file_handler.go | 2 +- pkg/relevancymanager/relevancy_manager_interface.go | 2 +- pkg/relevancymanager/v1/relevancy_manager.go | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/containerwatcher/v1/container_watcher.go b/pkg/containerwatcher/v1/container_watcher.go index 6e386aa428..0d92c3c65b 100644 --- a/pkg/containerwatcher/v1/container_watcher.go +++ b/pkg/containerwatcher/v1/container_watcher.go @@ -109,7 +109,7 @@ func (ch *IGContainerWatcher) Start(ctx context.Context) error { execEventCallback := func(event *tracerexectype.Event) { if event.Type != types.NORMAL { // dropped event - logger.L().Ctx(ctx).Warning("container monitoring got drop events - we may miss some realtime data", helpers.Interface("event", event), helpers.String("error", event.Message)) + logger.L().Warning("container monitoring got drop events - we may miss some realtime data", helpers.Interface("event", event), helpers.String("error", event.Message)) return } if event.Retval > -1 { @@ -117,7 +117,7 @@ func (ch *IGContainerWatcher) Start(ctx context.Context) error { if len(event.Args) > 0 { procImageName = event.Args[0] } - ch.relevancyManager.ReportFileAccess(ctx, event.Namespace, event.Pod, event.Container, procImageName) + ch.relevancyManager.ReportFileAccess(event.Namespace, event.Pod, event.Container, procImageName) } } if err := ch.tracerCollection.AddTracer(execTraceName, containerSelector); err != nil { @@ -128,11 +128,11 @@ func (ch *IGContainerWatcher) Start(ctx context.Context) error { openEventCallback := func(event *traceropentype.Event) { if event.Type != types.NORMAL { // dropped event - logger.L().Ctx(ctx).Warning("container monitoring got drop events - we may miss some realtime data", helpers.Interface("event", event), helpers.String("error", event.Message)) + logger.L().Warning("container monitoring got drop events - we may miss some realtime data", helpers.Interface("event", event), helpers.String("error", event.Message)) return } if event.Ret > -1 { - ch.relevancyManager.ReportFileAccess(ctx, event.Namespace, event.Pod, event.Container, event.Path) + ch.relevancyManager.ReportFileAccess(event.Namespace, event.Pod, event.Container, event.Path) } } if err := ch.tracerCollection.AddTracer(openTraceName, containerSelector); err != nil { diff --git a/pkg/filehandler/file_handler_interface.go b/pkg/filehandler/file_handler_interface.go index ae1a95bcc5..717a32b63a 100644 --- a/pkg/filehandler/file_handler_interface.go +++ b/pkg/filehandler/file_handler_interface.go @@ -3,7 +3,7 @@ package filehandler import "context" type FileHandler interface { - AddFile(ctx context.Context, bucket, file string) error + AddFile(bucket, file string) error Close() GetFiles(ctx context.Context, container string) (map[string]bool, error) RemoveBucket(ctx context.Context, bucket string) error diff --git a/pkg/filehandler/v1/file_handler.go b/pkg/filehandler/v1/file_handler.go index 7e9d5ef5e3..2ca34934a5 100644 --- a/pkg/filehandler/v1/file_handler.go +++ b/pkg/filehandler/v1/file_handler.go @@ -25,7 +25,7 @@ func CreateBoltFileHandler() (*BoltFileHandler, error) { return &BoltFileHandler{fileDB: db}, nil } -func (b BoltFileHandler) AddFile(_ context.Context, bucket, file string) error { +func (b BoltFileHandler) AddFile(bucket, file string) error { return b.fileDB.Batch(func(tx *bolt.Tx) error { b, err := tx.CreateBucketIfNotExists([]byte(bucket)) if err != nil { diff --git a/pkg/relevancymanager/relevancy_manager_interface.go b/pkg/relevancymanager/relevancy_manager_interface.go index 52928e0c67..cd67ec9fd4 100644 --- a/pkg/relevancymanager/relevancy_manager_interface.go +++ b/pkg/relevancymanager/relevancy_manager_interface.go @@ -10,7 +10,7 @@ import ( type RelevancyManagerClient interface { ReportContainerStarted(ctx context.Context, container *containercollection.Container) ReportContainerTerminated(ctx context.Context, container *containercollection.Container) - ReportFileAccess(ctx context.Context, namespace, pod, container, file string) + ReportFileAccess(namespace, pod, container, file string) SetContainerHandler(containerHandler containerwatcher.ContainerWatcher) StartRelevancyManager(ctx context.Context) } diff --git a/pkg/relevancymanager/v1/relevancy_manager.go b/pkg/relevancymanager/v1/relevancy_manager.go index 77d6bb4d85..34a5e02041 100644 --- a/pkg/relevancymanager/v1/relevancy_manager.go +++ b/pkg/relevancymanager/v1/relevancy_manager.go @@ -347,7 +347,7 @@ func (rm *RelevancyManager) ReportContainerTerminated(ctx context.Context, conta } } -func (rm *RelevancyManager) ReportFileAccess(ctx context.Context, namespace, pod, container, file string) { +func (rm *RelevancyManager) ReportFileAccess(namespace, pod, container, file string) { // log accessed files for all containers to avoid race condition // this won't record unnecessary containers as the containerCollection takes care of filtering them if file == "" { @@ -355,9 +355,9 @@ func (rm *RelevancyManager) ReportFileAccess(ctx context.Context, namespace, pod } rm.workerPool.Submit(func() { k8sContainerID := utils.CreateK8sContainerID(namespace, pod, container) - err := rm.fileHandler.AddFile(ctx, k8sContainerID, file) + err := rm.fileHandler.AddFile(k8sContainerID, file) if err != nil { - logger.L().Ctx(ctx).Error("failed to add file to container file list", helpers.Error(err), helpers.Interface("k8sContainerID", k8sContainerID), helpers.String("file", file)) + logger.L().Error("failed to add file to container file list", helpers.Error(err), helpers.Interface("k8sContainerID", k8sContainerID), helpers.String("file", file)) } }) } From 60bc3570a721d65b9f00fd731e80332aeaff4a6d Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Sat, 22 Jul 2023 16:48:28 +0200 Subject: [PATCH 5/6] increase concurrency Signed-off-by: Matthias Bertschy --- pkg/relevancymanager/v1/relevancy_manager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/relevancymanager/v1/relevancy_manager.go b/pkg/relevancymanager/v1/relevancy_manager.go index 34a5e02041..c31409d296 100644 --- a/pkg/relevancymanager/v1/relevancy_manager.go +++ b/pkg/relevancymanager/v1/relevancy_manager.go @@ -31,7 +31,7 @@ import ( ) const ( - concurrency = 4 + concurrency = 8 RelevantCVEsService = "RelevantCVEsService" StepGetSBOM = "StepGetSBOM" StepValidateSBOM = "StepValidateSBOM" @@ -353,8 +353,8 @@ func (rm *RelevancyManager) ReportFileAccess(namespace, pod, container, file str if file == "" { return } + k8sContainerID := utils.CreateK8sContainerID(namespace, pod, container) rm.workerPool.Submit(func() { - k8sContainerID := utils.CreateK8sContainerID(namespace, pod, container) err := rm.fileHandler.AddFile(k8sContainerID, file) if err != nil { logger.L().Error("failed to add file to container file list", helpers.Error(err), helpers.Interface("k8sContainerID", k8sContainerID), helpers.String("file", file)) From 2fa5c9c23e3a11ceda5aaebd91b1c3d70693d3aa Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Sat, 22 Jul 2023 17:41:18 +0200 Subject: [PATCH 6/6] use string builder Signed-off-by: Matthias Bertschy --- pkg/utils/utils.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 751b7860e4..b819169061 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -1,7 +1,6 @@ package utils import ( - "fmt" "math/rand" "path/filepath" "runtime" @@ -47,7 +46,7 @@ func CurrentDir() string { } func CreateK8sContainerID(namespaceName string, podName string, containerName string) string { - return fmt.Sprintf("%s/%s/%s", namespaceName, podName, containerName) + return strings.Join([]string{namespaceName, podName, containerName}, "/") } // RandomSleep sleeps between min and max seconds