Skip to content

fix #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
amirmalka wants to merge 2 commits into from
Closed

fix #102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5a5a721
fix
amirmalka Jul 24, 2023
95e53b4
fix
amirmalka Jul 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/cilium/ebpf v0.10.0
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
github.com/gammazero/workerpool v1.1.3
github.com/google/uuid v1.3.0
github.com/inspektor-gadget/inspektor-gadget v0.18.0
github.com/kubescape/go-logger v0.0.13
github.com/kubescape/k8s-interface v0.0.130
Expand Down Expand Up @@ -60,7 +61,6 @@ require (
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/imdario/mergo v0.3.15 // indirect
Expand Down
1 change: 1 addition & 0 deletions pkg/filehandler/file_handler_interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import "context"

type FileHandler interface {
AddFile(ctx context.Context, bucket, file string) error
AddFiles(ctx context.Context, bucket string, files map[string]bool) error
Close()
GetFiles(ctx context.Context, container string) (map[string]bool, error)
RemoveBucket(ctx context.Context, bucket string) error
Expand Down
31 changes: 31 additions & 0 deletions pkg/filehandler/v1/file_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ func (b BoltFileHandler) AddFile(ctx context.Context, bucket, file string) error
})
}

func (b BoltFileHandler) AddFiles(ctx context.Context, bucket string, files map[string]bool) error {
panic("not implemented")
}

func (b BoltFileHandler) Close() {
_ = b.fileDB.Close()
}
Expand Down Expand Up @@ -135,6 +139,33 @@ func (f *FsFileHandler) AddFile(ctx context.Context, bucket, file string) error
return nil
}

func (f *FsFileHandler) AddFiles(ctx context.Context, bucket string, files map[string]bool) error {
// Normalize the bucket name
bucket = NormalizeBucketName(bucket)

// Check if bucket exists in the map
f.Lock.Lock()
fileObject, ok := f.fileMap[bucket]
if !ok {
// If not, create a new file object and add it to the map
fileOsObject, err := os.OpenFile(f.GetBucketFileName(bucket), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
f.Lock.Unlock()
return err
}
fileObject = &File{FileObject: fileOsObject, Lock: &sync.Mutex{}}
f.fileMap[bucket] = fileObject
}
f.Lock.Unlock()
// Lock the file
fileObject.Lock.Lock()
for file := range files {
fileObject.FileObject.WriteString(file + "\n")
}
defer fileObject.Lock.Unlock()
return nil
}

func (f *FsFileHandler) Close() {
f.Lock.Lock()
for _, file := range f.fileMap {
Expand Down
99 changes: 56 additions & 43 deletions pkg/relevancymanager/v1/relevancy_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,53 +80,66 @@ func (rm *RelevancyManager) afterTimerActions(ctx context.Context) error {

if rm.cfg.EnableRelevancy && afterTimerActionsData.service == RelevantCVEsService {
fileList, err := rm.fileHandler.GetFiles(ctx, containerData.k8sContainerID)
if err != nil {
logger.L().Debug("failed to get file list", helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
continue
}
logger.L().Debug("fileList generated", helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.String("file list", fmt.Sprintf("%v", fileList)))
ctxPostSBOM, spanPostSBOM := otel.Tracer("").Start(ctx, "PostFilterSBOM")
if err = <-containerData.syncChannel[StepGetSBOM]; err != nil {
logger.L().Debug("failed to get SBOM", helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
continue
}
if containerData.sbomClient == nil {
// it is possible that the sbom client was not created yet
logger.L().Debug("sbom client not yet created", helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
continue
}
if err = containerData.sbomClient.ValidateSBOM(ctx); err != nil {
ctx, span := otel.Tracer("").Start(ctxPostSBOM, "ValidateSBOM")
logger.L().Ctx(ctx).Warning("SBOM is incomplete", helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
containerData.syncChannel[StepValidateSBOM] <- err
span.End()
}
if err = containerData.sbomClient.FilterSBOM(ctx, fileList); err != nil {
ctx, span := otel.Tracer("").Start(ctxPostSBOM, "FilterSBOM")
logger.L().Ctx(ctx).Warning("failed to filter SBOM", helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
span.End()
continue
}
filterSBOMKey, err := containerData.instanceID.GetSlug()
if err != nil {
ctx, span := otel.Tracer("").Start(ctxPostSBOM, "filterSBOMKey")
logger.L().Ctx(ctx).Warning("failed to get filterSBOMKey for store filter SBOM", helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
span.End()
continue
}
// it is safe to use containerData.imageID directly since we needed it to retrieve the SBOM
if err = containerData.sbomClient.StoreFilterSBOM(ctx, containerData.imageID, filterSBOMKey); err != nil {
if !errors.Is(err, sbom.IsAlreadyExist()) {
ctx, span := otel.Tracer("").Start(ctxPostSBOM, "StoreFilterSBOM")
logger.L().Ctx(ctx).Error("failed to store filtered SBOM", helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
span.End()
if err == nil {
if err := storeFilteredSBOM(ctx, containerData, afterTimerActionsData.containerID, fileList); err == nil {
continue
}
continue

logger.L().Debug("failed to store filtered sbom", helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
} else {
logger.L().Debug("failed to get file list", helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
}
logger.L().Info("filtered SBOM has been stored successfully", helpers.String("containerID", afterTimerActionsData.containerID), helpers.String("k8s workload", containerData.k8sContainerID))
spanPostSBOM.End()

// TODO recover
rm.fileHandler.AddFiles(ctx, containerData.k8sContainerID, fileList)
}
}
}

func storeFilteredSBOM(ctx context.Context, containerData watchedContainerData, containerID string, fileList map[string]bool) (err error) {
logger.L().Debug("fileList generated", helpers.String("container ID", containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.String("file list", fmt.Sprintf("%v", fileList)))
ctxPostSBOM, spanPostSBOM := otel.Tracer("").Start(ctx, "PostFilterSBOM")
if err = <-containerData.syncChannel[StepGetSBOM]; err != nil {
logger.L().Debug("failed to get SBOM", helpers.String("container ID", containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
return err
}
if containerData.sbomClient == nil {
// it is possible that the sbom client was not created yet
logger.L().Debug("sbom client not yet created", helpers.String("container ID", containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
return err
}
if err = containerData.sbomClient.ValidateSBOM(ctx); err != nil {
ctx, span := otel.Tracer("").Start(ctxPostSBOM, "ValidateSBOM")
logger.L().Ctx(ctx).Warning("SBOM is incomplete", helpers.String("container ID", containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
containerData.syncChannel[StepValidateSBOM] <- err
span.End()
return err
}
if err = containerData.sbomClient.FilterSBOM(ctx, fileList); err != nil {
ctx, span := otel.Tracer("").Start(ctxPostSBOM, "FilterSBOM")
logger.L().Ctx(ctx).Warning("failed to filter SBOM", helpers.String("container ID", containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
span.End()
return err
}
filterSBOMKey, err := containerData.instanceID.GetSlug()
if err != nil {
ctx, span := otel.Tracer("").Start(ctxPostSBOM, "filterSBOMKey")
logger.L().Ctx(ctx).Warning("failed to get filterSBOMKey for store filter SBOM", helpers.String("container ID", containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
span.End()
return err
}
// it is safe to use containerData.imageID directly since we needed it to retrieve the SBOM
if err = containerData.sbomClient.StoreFilterSBOM(ctx, containerData.imageID, filterSBOMKey); err != nil {
if !errors.Is(err, sbom.IsAlreadyExist()) {
ctx, span := otel.Tracer("").Start(ctxPostSBOM, "StoreFilterSBOM")
logger.L().Ctx(ctx).Error("failed to store filtered SBOM", helpers.String("container ID", containerID), helpers.String("k8s workload", containerData.k8sContainerID), helpers.Error(err))
span.End()
}
return err
}
logger.L().Info("filtered SBOM has been stored successfully", helpers.String("containerID", containerID), helpers.String("k8s workload", containerData.k8sContainerID))
spanPostSBOM.End()
return nil
}

func (rm *RelevancyManager) deleteResources(watchedContainer watchedContainerData, containerID string) {
Expand Down