diff --git a/docs/src/content/docs/status.mdx b/docs/src/content/docs/status.mdx index 1371fa10263..334f70bfe81 100644 --- a/docs/src/content/docs/status.mdx +++ b/docs/src/content/docs/status.mdx @@ -15,7 +15,7 @@ Status of all agentic workflows. [Browse source files](https://github.com/github | [Basic Research Agent](https://github.com/githubnext/gh-aw/blob/main/.github/workflows/research.md) | copilot | [![Basic Research Agent](https://github.com/githubnext/gh-aw/actions/workflows/research.lock.yml/badge.svg)](https://github.com/githubnext/gh-aw/actions/workflows/research.lock.yml) | - | - | | [Blog Auditor](https://github.com/githubnext/gh-aw/blob/main/.github/workflows/blog-auditor.md) | claude | [![Blog Auditor](https://github.com/githubnext/gh-aw/actions/workflows/blog-auditor.lock.yml/badge.svg)](https://github.com/githubnext/gh-aw/actions/workflows/blog-auditor.lock.yml) | `0 12 * * 3` | - | | [Brave Web Search Agent](https://github.com/githubnext/gh-aw/blob/main/.github/workflows/brave.md) | copilot | [![Brave Web Search Agent](https://github.com/githubnext/gh-aw/actions/workflows/brave.lock.yml/badge.svg)](https://github.com/githubnext/gh-aw/actions/workflows/brave.lock.yml) | - | `/brave` | -| [Changeset Generator](https://github.com/githubnext/gh-aw/blob/main/.github/workflows/changeset.md) | copilot | [![Changeset Generator](https://github.com/githubnext/gh-aw/actions/workflows/changeset.lock.yml/badge.svg)](https://github.com/githubnext/gh-aw/actions/workflows/changeset.lock.yml) | - | - | +| [Changeset Generator](https://github.com/githubnext/gh-aw/blob/main/.github/workflows/changeset.md) | copilot | [![Changeset Generator](https://github.com/githubnext/gh-aw/actions/workflows/changeset.lock.yml/badge.svg)](https://github.com/githubnext/gh-aw/actions/workflows/changeset.lock.yml) | `0 */2 * * *` | - | | [CI Failure Doctor](https://github.com/githubnext/gh-aw/blob/main/.github/workflows/ci-doctor.md) | copilot | [![CI Failure Doctor](https://github.com/githubnext/gh-aw/actions/workflows/ci-doctor.lock.yml/badge.svg)](https://github.com/githubnext/gh-aw/actions/workflows/ci-doctor.lock.yml) | - | - | | [CLI Consistency Checker](https://github.com/githubnext/gh-aw/blob/main/.github/workflows/cli-consistency-checker.md) | copilot | [![CLI Consistency Checker](https://github.com/githubnext/gh-aw/actions/workflows/cli-consistency-checker.lock.yml/badge.svg)](https://github.com/githubnext/gh-aw/actions/workflows/cli-consistency-checker.lock.yml) | `0 13 * * 1-5` | - | | [CLI Version Checker](https://github.com/githubnext/gh-aw/blob/main/.github/workflows/cli-version-checker.md) | copilot | [![CLI Version Checker](https://github.com/githubnext/gh-aw/actions/workflows/cli-version-checker.lock.yml/badge.svg)](https://github.com/githubnext/gh-aw/actions/workflows/cli-version-checker.lock.yml) | `0 15 * * *` | - | diff --git a/pkg/workflow/dependabot.go b/pkg/workflow/dependabot.go index 5c433e3100e..dc1f6a0b406 100644 --- a/pkg/workflow/dependabot.go +++ b/pkg/workflow/dependabot.go @@ -603,29 +603,22 @@ func extractGoPackages(workflowData *WorkflowData) []string { // extractGoFromCommands extracts Go package paths from command strings func extractGoFromCommands(commands string) []string { var packages []string - lines := strings.Split(commands, "\n") - - for _, line := range lines { - // Look for "go install " or "go get " patterns - words := strings.Fields(line) - for i, word := range words { - if word == "go" && i+1 < len(words) { - cmd := words[i+1] - if cmd == "install" || cmd == "get" { - // Find the package path - for j := i + 2; j < len(words); j++ { - pkg := words[j] - pkg = strings.TrimRight(pkg, "&|;") - // Skip flags (start with - or --) - if !strings.HasPrefix(pkg, "-") { - packages = append(packages, pkg) - break - } - } - } - } - } + + // Extract "go install " pattern + installExtractor := PackageExtractor{ + CommandNames: []string{"go"}, + RequiredSubcommand: "install", + TrimSuffixes: "&|;", + } + packages = append(packages, installExtractor.ExtractPackages(commands)...) + + // Extract "go get " pattern + getExtractor := PackageExtractor{ + CommandNames: []string{"go"}, + RequiredSubcommand: "get", + TrimSuffixes: "&|;", } + packages = append(packages, getExtractor.ExtractPackages(commands)...) return packages } diff --git a/pkg/workflow/npm.go b/pkg/workflow/npm.go index fc410b9073b..9b03f6483a0 100644 --- a/pkg/workflow/npm.go +++ b/pkg/workflow/npm.go @@ -15,10 +15,6 @@ // For detailed documentation, see specs/validation-architecture.md package workflow -import ( - "strings" -) - // extractNpxPackages extracts npx package names from workflow data func extractNpxPackages(workflowData *WorkflowData) []string { return collectPackagesFromWorkflow(workflowData, extractNpxFromCommands, "npx") @@ -26,27 +22,10 @@ func extractNpxPackages(workflowData *WorkflowData) []string { // extractNpxFromCommands extracts npx package names from command strings func extractNpxFromCommands(commands string) []string { - var packages []string - lines := strings.Split(commands, "\n") - - for _, line := range lines { - // Look for "npx " pattern - words := strings.Fields(line) - for i, word := range words { - if word == "npx" && i+1 < len(words) { - // Skip flags and find the first package name - for j := i + 1; j < len(words); j++ { - pkg := words[j] - pkg = strings.TrimRight(pkg, "&|;") - // Skip flags (start with - or --) - if !strings.HasPrefix(pkg, "-") { - packages = append(packages, pkg) - break - } - } - } - } + extractor := PackageExtractor{ + CommandNames: []string{"npx"}, + RequiredSubcommand: "", + TrimSuffixes: "&|;", } - - return packages + return extractor.ExtractPackages(commands) } diff --git a/pkg/workflow/package_extraction.go b/pkg/workflow/package_extraction.go new file mode 100644 index 00000000000..854fd68fdcd --- /dev/null +++ b/pkg/workflow/package_extraction.go @@ -0,0 +1,154 @@ +// Package workflow provides generic package extraction utilities for agentic workflows. +// +// # Package Extraction Framework +// +// This file provides a generic framework for extracting package names from command strings. +// The PackageExtractor type can be configured to handle different package managers +// (npm, pip, uv, go, etc.) with minimal code duplication. +// +// # Usage Example +// +// extractor := PackageExtractor{ +// CommandNames: []string{"pip", "pip3"}, +// RequiredSubcommand: "install", +// TrimSuffixes: "&|;", +// } +// packages := extractor.ExtractPackages("pip install requests") +// // Returns: []string{"requests"} +// +// For package-specific extraction, see npm.go, pip.go, and dependabot.go. +// For validation, see validation.go. +package workflow + +import ( + "strings" +) + +// PackageExtractor provides a configurable framework for extracting package names +// from command-line strings. It can be configured to handle different package +// managers (npm, pip, uv, go) by setting the appropriate command names and options. +type PackageExtractor struct { + // CommandNames is the list of command names to look for (e.g., ["pip", "pip3"]) + CommandNames []string + + // RequiredSubcommand is the subcommand that must follow the command name + // (e.g., "install" for pip). If empty, the package name is expected immediately + // after the command name (e.g., "npx "). + RequiredSubcommand string + + // TrimSuffixes is a string of characters to trim from the end of package names + // (e.g., "&|;" for shell operators) + TrimSuffixes string +} + +// ExtractPackages extracts package names from command strings using the configured +// extraction rules. It processes multi-line command strings and returns all found +// package names. +// +// The extraction process: +// 1. Split commands by newlines +// 2. Split each line into words +// 3. Find command name matches +// 4. If RequiredSubcommand is set, look for that subcommand +// 5. Skip flags (words starting with -) +// 6. Extract package name and trim configured suffixes +// 7. Return first package found per command invocation +// +// Example usage: +// +// extractor := PackageExtractor{ +// CommandNames: []string{"pip", "pip3"}, +// RequiredSubcommand: "install", +// TrimSuffixes: "&|;", +// } +// packages := extractor.ExtractPackages("pip install requests==2.28.0") +// // Returns: []string{"requests==2.28.0"} +func (pe *PackageExtractor) ExtractPackages(commands string) []string { + var packages []string + lines := strings.Split(commands, "\n") + + for _, line := range lines { + words := strings.Fields(line) + for i, word := range words { + // Check if this word matches one of our command names + if !pe.isCommandName(word) { + continue + } + + // If we have a required subcommand, find it first + if pe.RequiredSubcommand != "" { + pkg := pe.extractWithSubcommand(words, i) + if pkg != "" { + packages = append(packages, pkg) + } + } else { + // No subcommand required - package comes directly after command + pkg := pe.extractDirectPackage(words, i) + if pkg != "" { + packages = append(packages, pkg) + } + } + } + } + + return packages +} + +// isCommandName checks if the given word matches any of the configured command names +func (pe *PackageExtractor) isCommandName(word string) bool { + for _, cmdName := range pe.CommandNames { + if word == cmdName { + return true + } + } + return false +} + +// extractWithSubcommand extracts a package name when a required subcommand must be present +// (e.g., "pip install ") +func (pe *PackageExtractor) extractWithSubcommand(words []string, commandIndex int) string { + // Look for the required subcommand after the command name + for j := commandIndex + 1; j < len(words); j++ { + if words[j] == pe.RequiredSubcommand { + // Found the subcommand - now find the package name + return pe.findPackageName(words, j+1) + } + } + return "" +} + +// extractDirectPackage extracts a package name that comes directly after the command +// (e.g., "npx ") +func (pe *PackageExtractor) extractDirectPackage(words []string, commandIndex int) string { + if commandIndex+1 >= len(words) { + return "" + } + return pe.findPackageName(words, commandIndex+1) +} + +// findPackageName finds and processes the package name starting at the given index. +// It skips flags (words starting with -) and returns the first non-flag word, +// trimming configured suffixes. +// +// This method is exported to allow special-case extraction patterns (like uv) +// to reuse the package finding logic. +func (pe *PackageExtractor) FindPackageName(words []string, startIndex int) string { + return pe.findPackageName(words, startIndex) +} + +// findPackageName is the internal implementation of FindPackageName +func (pe *PackageExtractor) findPackageName(words []string, startIndex int) string { + for i := startIndex; i < len(words); i++ { + pkg := words[i] + // Skip flags (start with - or --) + if strings.HasPrefix(pkg, "-") { + continue + } + // Trim configured suffixes (e.g., shell operators) + if pe.TrimSuffixes != "" { + pkg = strings.TrimRight(pkg, pe.TrimSuffixes) + } + return pkg + } + return "" +} diff --git a/pkg/workflow/package_extraction_test.go b/pkg/workflow/package_extraction_test.go new file mode 100644 index 00000000000..f41763654ea --- /dev/null +++ b/pkg/workflow/package_extraction_test.go @@ -0,0 +1,388 @@ +package workflow + +import ( + "reflect" + "testing" +) + +func TestPackageExtractor_ExtractPackages_NpxPattern(t *testing.T) { + // Test npx pattern (no required subcommand) + extractor := PackageExtractor{ + CommandNames: []string{"npx"}, + RequiredSubcommand: "", + TrimSuffixes: "&|;", + } + + tests := []struct { + name string + commands string + want []string + }{ + { + name: "simple npx command", + commands: "npx playwright", + want: []string{"playwright"}, + }, + { + name: "npx with version", + commands: "npx @playwright/mcp@latest", + want: []string{"@playwright/mcp@latest"}, + }, + { + name: "npx with flags", + commands: "npx --yes playwright", + want: []string{"playwright"}, + }, + { + name: "npx with semicolon", + commands: "npx playwright;", + want: []string{"playwright"}, + }, + { + name: "npx with ampersand", + commands: "npx playwright&", + want: []string{"playwright"}, + }, + { + name: "npx with pipe", + commands: "npx playwright|", + want: []string{"playwright"}, + }, + { + name: "multiple npx commands", + commands: `npx playwright +npx typescript`, + want: []string{"playwright", "typescript"}, + }, + { + name: "no npx command", + commands: "echo 'hello'", + want: nil, + }, + { + name: "empty command", + commands: "", + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractor.ExtractPackages(tt.commands) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ExtractPackages() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPackageExtractor_ExtractPackages_PipPattern(t *testing.T) { + // Test pip pattern (required "install" subcommand) + extractor := PackageExtractor{ + CommandNames: []string{"pip", "pip3"}, + RequiredSubcommand: "install", + TrimSuffixes: "&|;", + } + + tests := []struct { + name string + commands string + want []string + }{ + { + name: "simple pip install", + commands: "pip install requests", + want: []string{"requests"}, + }, + { + name: "pip3 install", + commands: "pip3 install numpy", + want: []string{"numpy"}, + }, + { + name: "pip install with version", + commands: "pip install requests==2.28.0", + want: []string{"requests==2.28.0"}, + }, + { + name: "pip install with flags", + commands: "pip install --upgrade pip", + want: []string{"pip"}, + }, + { + name: "pip install with multiple flags", + commands: "pip install --no-cache-dir --upgrade requests", + want: []string{"requests"}, + }, + { + name: "multiple pip commands", + commands: `pip install requests +pip3 install numpy`, + want: []string{"requests", "numpy"}, + }, + { + name: "pip without install", + commands: "pip list", + want: nil, + }, + { + name: "pip install only flags", + commands: "pip install --upgrade --no-deps", + want: nil, + }, + { + name: "pip command with semicolon", + commands: "pip install requests;", + want: []string{"requests"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractor.ExtractPackages(tt.commands) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ExtractPackages() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPackageExtractor_ExtractPackages_GoPattern(t *testing.T) { + // Test go pattern (required "install" or "get" subcommand) + // Note: This requires handling multiple possible subcommands, + // which we'll test separately for "install" and "get" + extractorInstall := PackageExtractor{ + CommandNames: []string{"go"}, + RequiredSubcommand: "install", + TrimSuffixes: "&|;", + } + + extractorGet := PackageExtractor{ + CommandNames: []string{"go"}, + RequiredSubcommand: "get", + TrimSuffixes: "&|;", + } + + tests := []struct { + name string + extractor PackageExtractor + commands string + want []string + }{ + { + name: "go install", + extractor: extractorInstall, + commands: "go install github.com/user/tool@v1.0.0", + want: []string{"github.com/user/tool@v1.0.0"}, + }, + { + name: "go get", + extractor: extractorGet, + commands: "go get golang.org/x/tools@latest", + want: []string{"golang.org/x/tools@latest"}, + }, + { + name: "go install with flags", + extractor: extractorInstall, + commands: "go install -v github.com/user/tool", + want: []string{"github.com/user/tool"}, + }, + { + name: "go without install", + extractor: extractorInstall, + commands: "go build main.go", + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.extractor.ExtractPackages(tt.commands) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ExtractPackages() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPackageExtractor_isCommandName(t *testing.T) { + extractor := PackageExtractor{ + CommandNames: []string{"pip", "pip3"}, + } + + tests := []struct { + name string + word string + want bool + }{ + { + name: "matches pip", + word: "pip", + want: true, + }, + { + name: "matches pip3", + word: "pip3", + want: true, + }, + { + name: "does not match npm", + word: "npm", + want: false, + }, + { + name: "does not match empty", + word: "", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractor.isCommandName(tt.word) + if got != tt.want { + t.Errorf("isCommandName() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPackageExtractor_findPackageName(t *testing.T) { + extractor := PackageExtractor{ + TrimSuffixes: "&|;", + } + + tests := []struct { + name string + words []string + startIndex int + want string + }{ + { + name: "finds package name", + words: []string{"install", "requests"}, + startIndex: 1, + want: "requests", + }, + { + name: "skips flags", + words: []string{"install", "--upgrade", "requests"}, + startIndex: 1, + want: "requests", + }, + { + name: "trims semicolon", + words: []string{"install", "requests;"}, + startIndex: 1, + want: "requests", + }, + { + name: "trims ampersand", + words: []string{"install", "requests&"}, + startIndex: 1, + want: "requests", + }, + { + name: "trims pipe", + words: []string{"install", "requests|"}, + startIndex: 1, + want: "requests", + }, + { + name: "no package found", + words: []string{"install", "--upgrade"}, + startIndex: 1, + want: "", + }, + { + name: "start index out of bounds", + words: []string{"install"}, + startIndex: 5, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractor.findPackageName(tt.words, tt.startIndex) + if got != tt.want { + t.Errorf("findPackageName() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPackageExtractor_ComplexScenarios(t *testing.T) { + tests := []struct { + name string + extractor PackageExtractor + commands string + want []string + }{ + { + name: "mixed commands with pip", + extractor: PackageExtractor{ + CommandNames: []string{"pip", "pip3"}, + RequiredSubcommand: "install", + TrimSuffixes: "&|;", + }, + commands: `apt-get update +pip install requests +apt-get install python3-dev +pip3 install numpy`, + want: []string{"requests", "numpy"}, + }, + { + name: "script block with pip", + extractor: PackageExtractor{ + CommandNames: []string{"pip"}, + RequiredSubcommand: "install", + TrimSuffixes: "&|;", + }, + commands: `#!/bin/bash +set -e +pip install --upgrade pip +pip install requests==2.28.0`, + want: []string{"pip", "requests==2.28.0"}, + }, + { + name: "multiple npx on same line", + extractor: PackageExtractor{ + CommandNames: []string{"npx"}, + RequiredSubcommand: "", + TrimSuffixes: "&|;", + }, + commands: "npx black && npx ruff", + want: []string{"black", "ruff"}, + }, + { + name: "package with special characters", + extractor: PackageExtractor{ + CommandNames: []string{"pip"}, + RequiredSubcommand: "install", + TrimSuffixes: "&|;", + }, + commands: "pip install Flask-CORS", + want: []string{"Flask-CORS"}, + }, + { + name: "package in quotes", + extractor: PackageExtractor{ + CommandNames: []string{"pip"}, + RequiredSubcommand: "install", + TrimSuffixes: "&|;", + }, + commands: `pip install "requests[security]"`, + want: []string{`"requests[security]"`}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.extractor.ExtractPackages(tt.commands) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ExtractPackages() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/workflow/pip.go b/pkg/workflow/pip.go index 0f96b6c382d..fea2c565277 100644 --- a/pkg/workflow/pip.go +++ b/pkg/workflow/pip.go @@ -37,35 +37,12 @@ func extractPipPackages(workflowData *WorkflowData) []string { // extractPipFromCommands extracts pip package names from command strings func extractPipFromCommands(commands string) []string { - var packages []string - lines := strings.Split(commands, "\n") - - for _, line := range lines { - // Look for "pip install " or "pip3 install " patterns - words := strings.Fields(line) - for i, word := range words { - if (word == "pip" || word == "pip3") && i+1 < len(words) { - // Look for install command - for j := i + 1; j < len(words); j++ { - if words[j] == "install" { - // Skip flags and find the first package name - for k := j + 1; k < len(words); k++ { - pkg := words[k] - pkg = strings.TrimRight(pkg, "&|;") - // Skip flags (start with - or --) - if !strings.HasPrefix(pkg, "-") { - packages = append(packages, pkg) - break - } - } - break - } - } - } - } + extractor := PackageExtractor{ + CommandNames: []string{"pip", "pip3"}, + RequiredSubcommand: "install", + TrimSuffixes: "&|;", } - - return packages + return extractor.ExtractPackages(commands) } // extractUvPackages extracts uv package names from workflow data @@ -78,27 +55,30 @@ func extractUvFromCommands(commands string) []string { var packages []string lines := strings.Split(commands, "\n") + uvxExtractor := PackageExtractor{ + CommandNames: []string{"uvx"}, + RequiredSubcommand: "", + TrimSuffixes: "&|;", + } + + uvPipHelper := PackageExtractor{TrimSuffixes: "&|;"} + for _, line := range lines { - // Look for "uv pip install " or "uvx " patterns words := strings.Fields(line) for i, word := range words { + // Check for "uvx " pattern if word == "uvx" && i+1 < len(words) { - pkg := words[i+1] - pkg = strings.TrimRight(pkg, "&|;") - packages = append(packages, pkg) + pkg := uvxExtractor.FindPackageName(words, i+1) + if pkg != "" { + packages = append(packages, pkg) + } } else if word == "uv" && i+2 < len(words) && words[i+1] == "pip" { - // Look for install command + // Check for "uv pip install " pattern for j := i + 2; j < len(words); j++ { if words[j] == "install" { - // Skip flags and find the first package name - for k := j + 1; k < len(words); k++ { - pkg := words[k] - pkg = strings.TrimRight(pkg, "&|;") - // Skip flags (start with - or --) - if !strings.HasPrefix(pkg, "-") { - packages = append(packages, pkg) - break - } + pkg := uvPipHelper.FindPackageName(words, j+1) + if pkg != "" { + packages = append(packages, pkg) } break }