lower memory and cpu usage

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"log"
+	"net/http"
 	"net/url"
 	"node-agent/internal/validator"
 	"node-agent/pkg/config"
@@ -14,6 +15,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	_ "net/http/pprof"
+
 	"github.com/kubescape/go-logger"
 	"github.com/kubescape/go-logger/helpers"
 	"github.com/kubescape/k8s-interface/k8sinterface"
@@ -23,6 +26,10 @@ import (
 func main() {
 	ctx := context.Background()
 
+	go func() {
+		log.Println(http.ListenAndServe("localhost:6060", nil))
+	}()
+
 	cfg, err := config.LoadConfig("/etc/config")
 	if err != nil {
 		logger.L().Ctx(ctx).Fatal("load config error", helpers.Error(err))

pkg/containerwatcher/v1/container_watcher.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"time"
 
-	"github.com/gammazero/workerpool"
 	containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection"
 	tracerexec "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/exec/tracer"
 	tracerexectype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/exec/types"
@@ -24,7 +23,6 @@ import (
 )
 
 const (
-	concurrency   = 4
 	execTraceName = "trace_exec"
 	openTraceName = "trace_open"
 )
@@ -36,7 +34,6 @@ type IGContainerWatcher struct {
 	tracerCollection    *tracercollection.TracerCollection
 	tracerExec          *tracerexec.Tracer
 	tracerOpen          *traceropen.Tracer
-	workerPool          *workerpool.WorkerPool
 }
 
 var _ containerwatcher.ContainerWatcher = (*IGContainerWatcher)(nil)
@@ -55,7 +52,6 @@ func CreateIGContainerWatcher(k8sClient *k8sinterface.KubernetesApi, relevancyMa
 		k8sClient:           k8sClient,
 		tracerCollection:    tracerCollection,
 		relevancyManager:    relevancyManager,
-		workerPool:          workerpool.New(concurrency),
 	}, nil
 }
 
@@ -113,17 +109,15 @@ func (ch *IGContainerWatcher) Start(ctx context.Context) error {
 	execEventCallback := func(event *tracerexectype.Event) {
 		if event.Type != types.NORMAL {
 			// dropped event
-			logger.L().Ctx(ctx).Warning("container monitoring got drop events - we may miss some realtime data", helpers.Interface("event", event), helpers.String("error", event.Message))
+			logger.L().Warning("container monitoring got drop events - we may miss some realtime data", helpers.Interface("event", event), helpers.String("error", event.Message))
 			return
 		}
 		if event.Retval > -1 {
 			procImageName := event.Comm
 			if len(event.Args) > 0 {
 				procImageName = event.Args[0]
 			}
-			ch.workerPool.Submit(func() {
-				ch.relevancyManager.ReportFileAccess(ctx, event.Namespace, event.Pod, event.Container, procImageName)
-			})
+			ch.relevancyManager.ReportFileAccess(event.Namespace, event.Pod, event.Container, procImageName)
 		}
 	}
 	if err := ch.tracerCollection.AddTracer(execTraceName, containerSelector); err != nil {
@@ -134,13 +128,11 @@ func (ch *IGContainerWatcher) Start(ctx context.Context) error {
 	openEventCallback := func(event *traceropentype.Event) {
 		if event.Type != types.NORMAL {
 			// dropped event
-			logger.L().Ctx(ctx).Warning("container monitoring got drop events - we may miss some realtime data", helpers.Interface("event", event), helpers.String("error", event.Message))
+			logger.L().Warning("container monitoring got drop events - we may miss some realtime data", helpers.Interface("event", event), helpers.String("error", event.Message))
 			return
 		}
 		if event.Ret > -1 {
-			ch.workerPool.Submit(func() {
-				ch.relevancyManager.ReportFileAccess(ctx, event.Namespace, event.Pod, event.Container, event.Path)
-			})
+			ch.relevancyManager.ReportFileAccess(event.Namespace, event.Pod, event.Container, event.Path)
 		}
 	}
 	if err := ch.tracerCollection.AddTracer(openTraceName, containerSelector); err != nil {

pkg/filehandler/file_handler_interface.go

@@ -3,7 +3,7 @@ package filehandler
 import "context"
 
 type FileHandler interface {
-	AddFile(ctx context.Context, bucket, file string) error
+	AddFile(bucket, file string) error
 	Close()
 	GetFiles(ctx context.Context, container string) (map[string]bool, error)
 	RemoveBucket(ctx context.Context, bucket string) error

pkg/filehandler/v1/file_handler.go

@@ -25,9 +25,7 @@ func CreateBoltFileHandler() (*BoltFileHandler, error) {
 	return &BoltFileHandler{fileDB: db}, nil
 }
 
-func (b BoltFileHandler) AddFile(ctx context.Context, bucket, file string) error {
-	_, span := otel.Tracer("").Start(ctx, "BoltFileHandler.AddFile")
-	defer span.End()
+func (b BoltFileHandler) AddFile(bucket, file string) error {
 	return b.fileDB.Batch(func(tx *bolt.Tx) error {
 		b, err := tx.CreateBucketIfNotExists([]byte(bucket))
 		if err != nil {

pkg/relevancymanager/relevancy_manager_interface.go

@@ -10,7 +10,7 @@ import (
 type RelevancyManagerClient interface {
 	ReportContainerStarted(ctx context.Context, container *containercollection.Container)
 	ReportContainerTerminated(ctx context.Context, container *containercollection.Container)
-	ReportFileAccess(ctx context.Context, namespace, pod, container, file string)
+	ReportFileAccess(namespace, pod, container, file string)
 	SetContainerHandler(containerHandler containerwatcher.ContainerWatcher)
 	StartRelevancyManager(ctx context.Context)
 }

pkg/relevancymanager/v1/relevancy_manager.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/armosec/utils-k8s-go/wlid"
+	"github.com/gammazero/workerpool"
 	containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection"
 	"github.com/kubescape/go-logger"
 	"github.com/kubescape/go-logger/helpers"
@@ -30,6 +31,7 @@ import (
 )
 
 const (
+	concurrency         = 8
 	RelevantCVEsService = "RelevantCVEsService"
 	StepGetSBOM         = "StepGetSBOM"
 	StepValidateSBOM    = "StepValidateSBOM"
@@ -51,6 +53,7 @@ type RelevancyManager struct {
 	sbomFs            afero.Fs
 	storageClient     storageclient.StorageClient
 	watchedContainers sync.Map
+	workerPool        *workerpool.WorkerPool
 }
 
 var _ relevancymanager.RelevancyManagerClient = (*RelevancyManager)(nil)
@@ -65,6 +68,7 @@ func CreateRelevancyManager(cfg config.Config, clusterName string, fileHandler f
 		sbomFs:                   sbomFs,
 		storageClient:            storageClient,
 		watchedContainers:        sync.Map{},
+		workerPool:               workerpool.New(concurrency),
 	}, nil
 }
 
@@ -343,19 +347,19 @@ func (rm *RelevancyManager) ReportContainerTerminated(ctx context.Context, conta
 	}
 }
 
-func (rm *RelevancyManager) ReportFileAccess(ctx context.Context, namespace, pod, container, file string) {
-	ctx, span := otel.Tracer("").Start(ctx, "RelevancyManager.ReportFileAccess")
-	defer span.End()
+func (rm *RelevancyManager) ReportFileAccess(namespace, pod, container, file string) {
 	// log accessed files for all containers to avoid race condition
 	// this won't record unnecessary containers as the containerCollection takes care of filtering them
 	if file == "" {
 		return
 	}
 	k8sContainerID := utils.CreateK8sContainerID(namespace, pod, container)
-	err := rm.fileHandler.AddFile(ctx, k8sContainerID, file)
-	if err != nil {
-		logger.L().Ctx(ctx).Error("failed to add file to container file list", helpers.Error(err), helpers.Interface("k8sContainerID", k8sContainerID), helpers.String("file", file))
-	}
+	rm.workerPool.Submit(func() {
+		err := rm.fileHandler.AddFile(k8sContainerID, file)
+		if err != nil {
+			logger.L().Error("failed to add file to container file list", helpers.Error(err), helpers.Interface("k8sContainerID", k8sContainerID), helpers.String("file", file))
+		}
+	})
 }
 
 func (rm *RelevancyManager) SetContainerHandler(containerHandler containerwatcher.ContainerWatcher) {

pkg/sbom/sbom_test.go

@@ -2,6 +2,7 @@ package sbom
 
 import (
 	"context"
+	"math/rand"
 	"node-agent/pkg/storageclient"
 	"testing"
 
@@ -18,15 +19,28 @@ func TestGetSBOM(t *testing.T) {
 
 }
 
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+// generate a random string of given length
+func randStringBytes(n int) string {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))] //pick a random letter from the letterBytes
+	}
+	return string(b)
+}
+
 func TestFilterSBOM(t *testing.T) {
 	SBOMClient := CreateSBOMStorageClient(storageclient.CreateSBOMStorageHttpClientMock(), "", &instanceidhandler.InstanceID{}, afero.NewMemMapFs())
 	err := SBOMClient.GetSBOM(context.TODO(), storageclient.NGINX_IMAGE_TAG, storageclient.NGINX)
 	if err != nil {
 		t.Fatalf("fail to get sbom, %v", err)
 	}
-	err = SBOMClient.FilterSBOM(context.TODO(), map[string]bool{
-		"/usr/share/adduser/adduser.conf": true,
-	})
+	m := make(map[string]bool)
+	for i := 0; i < 10000; i++ {
+		m["/tmp/"+randStringBytes(10)] = true
+	}
+	err = SBOMClient.FilterSBOM(context.TODO(), m)
 	if err != nil {
 		t.Fatalf("fail to filter sbom, %v", err)
 	}

pkg/utils/utils.go

@@ -1,7 +1,6 @@
 package utils
 
 import (
-	"fmt"
 	"math/rand"
 	"path/filepath"
 	"runtime"
@@ -47,7 +46,7 @@ func CurrentDir() string {
 }
 
 func CreateK8sContainerID(namespaceName string, podName string, containerName string) string {
-	return fmt.Sprintf("%s/%s/%s", namespaceName, podName, containerName)
+	return strings.Join([]string{namespaceName, podName, containerName}, "/")
 }
 
 // RandomSleep sleeps between min and max seconds