fix(sbom): avoid marking editable Python packages as absent

pkg/build/sbom_installed_packages_converge_test.go

@@ -34,7 +34,7 @@ import (
 )
 
 var _ = Describe("Installed inventory convergence", func() {
-	It("replaces a declaration-only cached BOM without any packages directive, then reuses the new inventory", func(specCtx SpecContext) {
+	DescribeTable("replaces an outdated cached BOM without any packages directive, then reuses the new inventory", func(specCtx SpecContext, oldVersion string) {
 		ctx := logging.WithLogger(specCtx)
 		Expect(werf.Init(GinkgoT().TempDir(), "")).To(Succeed())
 		Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed())
@@ -66,7 +66,7 @@ var _ = Describe("Installed inventory convergence", func() {
 		opts.Commands[0].SourcePath = info.Name
 		mergeOpts := cyclonedxutil.MergeOpts{}
 		oldChecksum := util.Sha256Hash(
-			"14", "scan", opts.Checksum(), "merge", mergeOpts.Checksum(),
+			oldVersion, "scan", opts.Checksum(), "merge", mergeOpts.Checksum(),
 			"gost_attack_surface", mergeOpts.Gost.AttackSurface.String(),
 			"gost_security_function", mergeOpts.Gost.SecurityFunction.String(),
 			"signer", "", "platform", "", "produced", sbomProvenanceChecksum(sbomStageProvenance{}),
@@ -109,5 +109,8 @@ var _ = Describe("Installed inventory convergence", func() {
 				}
 			}
 		}
-	})
+	},
+		Entry("declaration-only inventory", "14"),
+		Entry("inventory before editable uncertainty", "15"),
+	)
 })

pkg/build/sbom_step.go

@@ -599,7 +599,7 @@ func (step *sbomStep) recordDeclaredPackages(ctx context.Context, bom *cdx.BOM,
 	return nil
 }
 
-const sbomArtifactFormatVersion = "15"
+const sbomArtifactFormatVersion = "16"
 
 // calculateStableChecksum computes the SBOM artifact cache checksum. Together with the
 // parent stage digest it forms the cache key: a previously attached SBOM is reused only

pkg/sbom/shippedscope/fileset.go

@@ -33,6 +33,9 @@ type Fileset struct {
 	// nor absent by its files, which is a different thing from one whose declared files are
 	// all gone.
 	DeclaresFiles bool
+	// Incomplete means the manifest can omit payload, such as an editable Python
+	// install whose RECORD lists hooks rather than the external source tree.
+	Incomplete bool
 }
 
 type Filesets struct {
@@ -97,6 +100,7 @@ func CollectFilesets(ctx context.Context, img v1.Image) (*Filesets, error) {
 func isFilesetManifest(file string) bool {
 	return file == pmFilesPath ||
 		strings.HasSuffix(file, distInfoRecordSuffix) ||
+		strings.HasSuffix(file, ".dist-info/direct_url.json") ||
 		isNodePackageManifest(file) ||
 		path.Base(file) == rockManifestName
 }

pkg/sbom/shippedscope/occurrences.go

@@ -17,8 +17,8 @@ import (
 // one or the other, so a reader can tell werf's own evidence from an occurrence that
 // arrived in a merged document and means something else entirely.
 //
-// A component no fileset names is left as it was, and so is one whose filesets all declare
-// no files at all. Nothing is then known about its files — its ecosystem records no file
+// A component no fileset names is left as it was, and so is one whose complete filesets
+// all declare no files at all. Nothing is then known about its files — its ecosystem records no file
 // lists, it was installed in a way none of them reads, or it is a metapackage that ships
 // nothing — and an empty occurrence list would instead claim its files were looked for and
 // none were found. That is what keeps os-pm components unmarked when the image was
@@ -34,7 +34,12 @@ import (
 // "its own file list names files it does not hold" is what an image that installed a
 // package and then deleted it looks like.
 //
-// Returns the number of components given occurrences.
+// An incomplete manifest records unknown instead of treating its listed files as complete
+// evidence. Existing occurrences are retained for provenance but ignored for the verdict.
+// Unknown survives merging and later complete installs of the same component: those
+// installs cannot establish completeness for an inherited origin.
+//
+// Returns the number of components whose file evidence was recorded or marked unknown.
 func RecordOccurrences(bom *cdx.BOM, filesets []Fileset) int {
 	if bom == nil || len(filesets) == 0 {
 		return 0
@@ -58,7 +63,7 @@ func RecordOccurrences(bom *cdx.BOM, filesets []Fileset) int {
 		}
 
 		sets, ok := owned[key]
-		if !ok || !lo.SomeBy(sets, func(fileset Fileset) bool { return fileset.DeclaresFiles }) {
+		if !ok || !lo.SomeBy(sets, func(fileset Fileset) bool { return fileset.DeclaresFiles || fileset.Incomplete }) {
 			return
 		}
 
@@ -170,13 +175,15 @@ func canonicalName(ecosystem Ecosystem, name string) string {
 	}
 }
 
-// PropertyFiles is written on a component whose files werf enumerated, and says what the
-// image holds of them. It exists because the answer "none" cannot be written as evidence:
-// an empty occurrence list says it in memory, and serialization drops the list along with
-// what it meant, leaving a component that reads as one nothing was ever looked for.
+// PropertyFiles records file evidence completeness. "none" preserves a checked-empty
+// list across serialization; "unknown" means an origin's manifest could omit payload.
+// Unknown survives merges even when another origin contributes a complete list.
 const PropertyFiles = "werf:files"
 
-const propertyFilesNone = "none"
+const (
+	propertyFilesNone    = "none"
+	propertyFilesUnknown = "unknown"
+)
 
 // symlinkContext marks an occurrence as a symlink, which has a path in the image but no
 // content of its own to compare.
@@ -190,6 +197,9 @@ const digestPrefix = "sha256:"
 // carrying none of werf's own is one nothing is known about, not one whose files were
 // looked for and missed.
 func recordedOccurrences(comp *cdx.Component) []Occurrence {
+	if incompleteFileEvidence(comp) {
+		return nil
+	}
 	if comp.Evidence == nil || comp.Evidence.Occurrences == nil {
 		if propertyValue(comp, PropertyFiles) == propertyFilesNone {
 			return []Occurrence{}
@@ -220,6 +230,12 @@ func recordedOccurrences(comp *cdx.Component) []Occurrence {
 // package more than once under different paths, and each copy is a fileset of its own;
 // what the component ships is all of them. Reports whether anything was recorded.
 func setOccurrences(comp *cdx.Component, sets []Fileset) bool {
+	if incompleteFileEvidence(comp) || lo.SomeBy(sets, func(fileset Fileset) bool { return fileset.Incomplete }) {
+		removeProperty(comp, PropertyFiles)
+		setProperty(comp, PropertyFiles, propertyFilesUnknown)
+		return true
+	}
+
 	contexts := make(map[string]string)
 	for _, fileset := range sets {
 		for file, entry := range fileset.Files {
@@ -256,6 +272,12 @@ func setOccurrences(comp *cdx.Component, sets []Fileset) bool {
 	return true
 }
 
+func incompleteFileEvidence(comp *cdx.Component) bool {
+	return lo.SomeBy(lo.FromPtr(comp.Properties), func(property cdx.Property) bool {
+		return property.Name == PropertyFiles && property.Value == propertyFilesUnknown
+	})
+}
+
 func walkComponents(bom *cdx.BOM, visit func(comp *cdx.Component)) {
 	var walk func(components []cdx.Component)
 	walk = func(components []cdx.Component) {

pkg/sbom/shippedscope/python_editable_test.go

@@ -0,0 +1,139 @@
+package shippedscope
+
+import (
+	"encoding/json"
+
+	cdx "github.com/CycloneDX/cyclonedx-go"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/werf/werf/v3/pkg/sbom/cyclonedxutil"
+)
+
+var _ = Describe("Editable Python evidence", func() {
+	DescribeTable("recognizes editable or unclassifiable origin metadata without executing hooks",
+		func(ctx SpecContext, origin, record string, wantUnknown bool) {
+			entries := []tarEntry{
+				regularFile("site/pkg-1.0.dist-info/RECORD", record),
+				regularFile("site/pkg.py", "payload"),
+				regularFile("site/generic.pth", "import arbitrary_code\n"),
+			}
+			if origin != "" {
+				entries = append(entries, regularFile("site/pkg-1.0.dist-info/direct_url.json", origin))
+			}
+			image, err := imageOf(entries)
+			Expect(err).NotTo(HaveOccurred())
+			filesets, err := CollectFilesets(ctx, image)
+			Expect(err).NotTo(HaveOccurred())
+			set := filesetOf(filesets, EcosystemPython, "pkg", "1.0")
+			Expect(set).NotTo(BeNil())
+			Expect(set.Incomplete).To(Equal(wantUnknown))
+			bom := cyclonedxutil.NewBOM()
+			bom.Components = &[]cdx.Component{{PackageURL: "pkg:pypi/pkg@1.0"}}
+			RecordOccurrences(bom, filesets.Filesets)
+			Expect(incompleteFileEvidence(&(*bom.Components)[0])).To(Equal(wantUnknown))
+		},
+		Entry("PEP 660 without a pth", `{"dir_info":{"editable":true}}`, "pkg.py,,\n", true),
+		Entry("editable with an empty RECORD", `{"dir_info":{"editable":true}}`, "", true),
+		Entry("malformed origin", `{`, "pkg.py,,\n", true),
+		Entry("null origin", `null`, "pkg.py,,\n", true),
+		Entry("non-object origin", `[]`, "pkg.py,,\n", true),
+		Entry("non-object directory", `{"dir_info":true}`, "pkg.py,,\n", true),
+		Entry("null directory", `{"dir_info":null}`, "pkg.py,,\n", true),
+		Entry("string editable is not boolean false", `{"dir_info":{"editable":"false"}}`, "pkg.py,,\n", true),
+		Entry("null editable is not boolean false", `{"dir_info":{"editable":null}}`, "pkg.py,,\n", true),
+		Entry("ordinary install with a pth", `{"dir_info":{"editable":false}}`, "pkg.py,,\ngeneric.pth,,\n", false),
+		Entry("omitted editable defaults to false", `{"dir_info":{}}`, "pkg.py,,\n", false),
+		Entry("VCS origin is not editable", `{"vcs_info":{"vcs":"git","commit_id":"abc"}}`, "pkg.py,,\n", false),
+		Entry("normal wheel has no origin", "", "pkg.py,,\n", false),
+		Entry("setuptools hook without direct URL metadata", "", "__editable__.pkg-1.0.pth,,\n", true),
+		Entry("recorded legacy link without direct URL metadata", "", "pkg.egg-link,,\n", true),
+	)
+
+	It("does not interpret an uncaptured symlinked origin as a normal install", func(ctx SpecContext) {
+		image, err := imageOf([]tarEntry{
+			regularFile("site/pkg-1.0.dist-info/RECORD", "pkg.py,,\n"),
+			regularFile("site/pkg.py", "payload"),
+			regularFile("shared/origin", `{"dir_info":{"editable":true}}`),
+			symlink("site/pkg-1.0.dist-info/direct_url.json", "/shared/origin"),
+		})
+		Expect(err).NotTo(HaveOccurred())
+		filesets, err := CollectFilesets(ctx, image)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(filesetOf(filesets, EcosystemPython, "pkg", "1.0").Incomplete).To(BeTrue())
+	})
+
+	DescribeTable("preserves uncertainty alongside another copy or origin of the same package",
+		func(ctx SpecContext, reverse, previousOccurrences bool) {
+			image, err := imageOf([]tarEntry{
+				regularFile("site/pkg-1.0.dist-info/RECORD", "__editable__.pkg-1.0.pth,,\n"),
+				regularFile("other/pkg-1.0.dist-info/RECORD", "pkg.py,,\n"),
+			})
+			Expect(err).NotTo(HaveOccurred())
+			filesets, err := CollectFilesets(ctx, image)
+			Expect(err).NotTo(HaveOccurred())
+			bom := cyclonedxutil.NewBOM()
+			bom.Components = &[]cdx.Component{{Name: "pkg", Version: "1.0", BOMRef: "pkg:pypi/pkg@1.0", PackageURL: "pkg:pypi/pkg@1.0"}}
+			known, err := cyclonedxutil.CloneBOM(bom)
+			Expect(err).NotTo(HaveOccurred())
+			if previousOccurrences {
+				(*known.Components)[0].Evidence = &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{{Location: "/gone/pkg.py", AdditionalContext: digestOf("gone")}}}
+			} else {
+				setProperty(&(*known.Components)[0], PropertyFiles, propertyFilesNone)
+			}
+			RecordOccurrences(bom, filesets.Filesets)
+			Expect(incompleteFileEvidence(&(*bom.Components)[0])).To(BeTrue(), "a regular copy cannot make an editable copy's manifest complete")
+			if reverse {
+				bom, known = known, bom
+			}
+			merged, err := cyclonedxutil.MergeBOMs(ctx, bom, cyclonedxutil.MergeOpts{BaseBOM: known})
+			Expect(err).NotTo(HaveOccurred())
+			for range 2 {
+				merged, err = cyclonedxutil.CloneBOM(merged)
+				Expect(err).NotTo(HaveOccurred())
+				RecordOccurrences(merged, []Fileset{{Ecosystem: EcosystemPython, Name: "pkg", Version: "1.0", DeclaresFiles: true}})
+				Expect(MarkNotShipped(ctx, merged, &Filesets{Index: emptyIndex()})).To(BeZero())
+				Expect(recordedOccurrences(&(*merged.Components)[0])).To(BeNil())
+			}
+		},
+		Entry("unknown before checked-empty", false, false),
+		Entry("checked-empty before unknown", true, false),
+		Entry("unknown before old occurrences", false, true),
+		Entry("old occurrences before unknown", true, true),
+	)
+
+	DescribeTable("does not treat editable installation scaffolding as complete payload evidence",
+		func(ctx SpecContext, imported []tarEntry) {
+			donorImage, err := imageOf([]tarEntry{
+				regularFile("site/editable-1.0.dist-info/RECORD", "editable.pth,,\neditable-1.0.dist-info/RECORD,,\neditable-1.0.dist-info/direct_url.json,,\n"),
+				regularFile("site/editable-1.0.dist-info/direct_url.json", `{"url":"file:///src/editable","dir_info":{"editable":true}}`),
+				regularFile("site/editable.pth", "/src/editable\n"),
+				regularFile("src/editable/module.py", "value = 42\n"),
+			})
+			Expect(err).NotTo(HaveOccurred())
+			donorFilesets, err := CollectFilesets(ctx, donorImage)
+			Expect(err).NotTo(HaveOccurred())
+			bom := &cdx.BOM{SpecVersion: cdx.SpecVersion1_6, Components: &[]cdx.Component{{
+				BOMRef: "pkg:pypi/editable@1.0", PackageURL: "pkg:pypi/editable@1.0", Name: "editable", Version: "1.0",
+			}}}
+			RecordOccurrences(bom, donorFilesets.Filesets)
+			Expect(MarkNotShipped(ctx, bom, donorFilesets)).To(BeZero())
+
+			body, err := json.Marshal(bom)
+			Expect(err).NotTo(HaveOccurred())
+			var inherited cdx.BOM
+			Expect(json.Unmarshal(body, &inherited)).To(Succeed())
+			image, err := imageOf(imported)
+			Expect(err).NotTo(HaveOccurred())
+			filesets, err := CollectFilesets(ctx, image)
+			Expect(err).NotTo(HaveOccurred())
+			RecordOccurrences(&inherited, filesets.Filesets)
+
+			Expect(MarkNotShipped(ctx, &inherited, filesets)).To(BeZero(), "missing editable scaffolding does not prove that the source was omitted")
+			Expect(recordedOccurrences(&(*inherited.Components)[0])).To(BeNil(), "neither copied metadata nor a .pth file is complete evidence of editable source files")
+		},
+		Entry("only source is copied", []tarEntry{regularFile("app/module.py", "value = 42\n")}),
+		Entry("only the path hook is copied", []tarEntry{regularFile("site/editable.pth", "/src/editable\n")}),
+		Entry("nothing is copied", []tarEntry{regularFile("app/unrelated.txt", "unrelated")}),
+	)
+})

pkg/sbom/shippedscope/python_fileset.go

@@ -3,6 +3,7 @@ package shippedscope
 import (
 	"context"
 	"encoding/csv"
+	"encoding/json"
 	"errors"
 	"io"
 	"maps"
@@ -39,10 +40,20 @@ func pythonFilesets(ctx context.Context, index *FileIndex, manifests map[string]
 		}
 
 		siteDir := path.Dir(distInfo)
+		incomplete := false
+		originPath := path.Join(distInfo, "direct_url.json")
+		if origin, exists := manifests[originPath]; exists {
+			incomplete = pythonOriginMayBeEditable(origin)
+		} else if _, linked := index.Symlinks[originPath]; linked {
+			incomplete = true
+		}
 
 		var declared []string
 		for _, record := range recordPaths(ctx, manifest, body) {
 			file := rootedJoin(siteDir, record)
+			if strings.HasPrefix(path.Base(file), "__editable__") || strings.HasSuffix(file, ".egg-link") {
+				incomplete = true
+			}
 			declared = append(declared, file)
 			declared = append(declared, pycache[file]...)
 		}
@@ -56,12 +67,30 @@ func pythonFilesets(ctx context.Context, index *FileIndex, manifests map[string]
 			Files:         files,
 			Symlinks:      symlinks,
 			DeclaresFiles: len(declared) > 0,
+			Incomplete:    incomplete,
 		})
 	}
 
 	return filesets
 }
 
+func pythonOriginMayBeEditable(body []byte) bool {
+	var origin map[string]json.RawMessage
+	if err := json.Unmarshal(body, &origin); err != nil || origin == nil {
+		return true
+	}
+	raw, exists := origin["dir_info"]
+	if !exists {
+		return false
+	}
+	var directory map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &directory); err != nil || directory == nil {
+		return true
+	}
+	editable, exists := directory["editable"]
+	return exists && strings.TrimSpace(string(editable)) != "false"
+}
+
 func recordPaths(ctx context.Context, manifest string, body []byte) []string {
 	reader := csv.NewReader(strings.NewReader(string(body)))
 	reader.FieldsPerRecord = -1

test/e2e/sbom/_fixtures/inject/editable_python/requirements.txt

@@ -0,0 +1,2 @@
+-e ./werfeditable
+./werfordinary

test/e2e/sbom/_fixtures/inject/editable_python/werf.yaml

@@ -0,0 +1,47 @@
+project: werf-test-e2e-sbom-python-pip-editable
+configVersion: 1
+build:
+  sbom:
+    enable: true
+    standard: cyclonedx@1.6
+---
+image: donor
+from: registry.deckhouse.io/container-factory@sha256:7aac8d97a91ac233b19111aaad9d7bba12b975e620cfa9287b97e0003e4d0a71
+git:
+  - add: /
+    to: /app
+    stageDependencies:
+      packages:
+        - requirements.txt
+        - werfeditable/**
+        - werfordinary/**
+packages:
+  - type: os-pm
+    spec:
+      - python==3.12.12
+  - type: python-pip
+    workdir: /app
+---
+image: source
+from: registry.werf.io/werf/scratch:latest
+import:
+  - image: donor
+    add: /app/werfeditable
+    to: /app/werfeditable
+    before: setup
+---
+image: grandchild
+from: registry.werf.io/werf/scratch:latest
+import:
+  - image: source
+    add: /app/werfeditable
+    to: /app/werfeditable
+    before: setup
+---
+image: empty
+from: registry.werf.io/werf/scratch:latest
+import:
+  - image: donor
+    add: /app/requirements.txt
+    to: /app/requirements.txt
+    before: setup