diff --git a/.github/workflows/downstream-compatibility.yml b/.github/workflows/downstream-compatibility.yml index 18d3636..9bf1115 100644 --- a/.github/workflows/downstream-compatibility.yml +++ b/.github/workflows/downstream-compatibility.yml @@ -53,7 +53,10 @@ jobs: repository: DataDog/dd-trace-py runtime: python submodule: tests/openfeature/ffe-system-test-data - setup: ":" + # ddtest runs as uid 1000 in Docker while the Actions checkout is uid 1001. + setup: >- + mkdir -p .cache/cython .cache/pip .cache/sccache .riot && + chmod -R a+rwX . evaluation: >- scripts/ddtest riot -v run --pass-env -s 13c4b39 -- -vv tests/openfeature/test_provider_fixtures.py @@ -81,7 +84,7 @@ jobs: repository: DataDog/dd-trace-php runtime: php submodule: tests/FeatureFlags/ffe-system-test-data - setup: ":" + setup: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev evaluation: >- make test_c TESTS=tests/ext/ffe/system_test_data_evaluate.phpt MAX_TEST_PARALLELISM=1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dd2e408..191745f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,39 @@ configuration, and return the caller default with `ERROR` / `PARSE_ERROR` when those keys are evaluated. Reserve `ERROR` / `FLAG_NOT_FOUND` for keys absent from both the active and rejected maps. Replace both maps atomically on refresh. +### Updating Targeting Regex Conformance + +The standalone targeting regex contract lives in +`regex-conformance/targeting-regex-conformance.json`. Do not put it in +`evaluation-cases/`; consumers parse every JSON file there as a full UFC +evaluation case. + +When changing the contract: + +1. Bump `schemaVersion` only for an incompatible schema change and bump + `contractVersion` when portable behavior changes. +2. Keep case IDs stable. Add a new ID instead of changing the meaning of an + existing case. +3. Include every required case field: `id`, `description`, `category`, + `contract`, `rawPattern`, `normalizedPattern`, `expectedCompile`, `input`, + and `expectedMatch`. +4. Use `contract` for the portable authoring decision. Use `expectedCompile` + and `expectedMatch` only for common native-engine observations. Set a common + field to `null` and add `engineExpectations` when native engines differ. +5. Verify accepted cases in every shipped SDK evaluator. Rejected cases may be + accepted by a native compiler; that does not make them part of the portable + contract or require an SDK production change. +6. Recompute `targeting-regex-conformance.sha256` from the exact JSON bytes. + +Validate JSON syntax, unique case IDs, required field types, and the hash: + +```bash +jq empty ufc-config.json evaluation-cases/*.json regex-conformance/targeting-regex-conformance.json +jq -e -f regex-conformance/validate-targeting-regex-conformance.jq regex-conformance/targeting-regex-conformance.json +regex-conformance/test-validate-targeting-regex-conformance.sh +(cd regex-conformance && shasum -a 256 -c targeting-regex-conformance.sha256) +``` + ### Modifying Flag Configuration When adding or modifying flags in `ufc-config.json`: diff --git a/README.md b/README.md index a6c7465..1778e01 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,13 @@ This repository contains the canonical set of flag configurations and evaluation ``` ffe-system-test-data/ ├── ufc-config.json # Master flag configuration (UFC format) -└── evaluation-cases/ - └── test-*.json # Evaluation test case files +├── evaluation-cases/ +│ └── test-*.json # Evaluation test case files +└── regex-conformance/ + ├── targeting-regex-conformance.json # FFE authoring and matching contract + ├── targeting-regex-conformance.sha256 # SHA-256 of the JSON bytes + ├── validate-targeting-regex-conformance.jq # Canonical schema validator + └── test-validate-targeting-regex-conformance.sh # Validator regression tests ``` ## Usage @@ -106,6 +111,29 @@ The shared fixtures intentionally exclude SDK-specific fields such as `variant` - **variant**: Derive from the flag configuration in `ufc-config.json` by matching the result value - **flagMetadata**: Extract from the flag's metadata field in `ufc-config.json` +### Targeting Regex Conformance + +`regex-conformance/targeting-regex-conformance.json` is a standalone, versioned +contract for authoring targeting regular expressions in FFE. It is intentionally +outside `evaluation-cases/`; consumers of that directory parse every JSON file +as a complete UFC evaluation case. + +The fixture records native observations for four implementations: Go `regexp`, +RE2JS, the Rust rules-based evaluator, and the Rust rkyv evaluator. The accepted +authoring subset is narrower: accepted cases must also evaluate consistently in +the shipped Java, JavaScript, and .NET SDK evaluators. Several SDKs share the +Rust evaluator, so agreement across those SDKs is not evidence from independent +regex engines. + +Each regex case has a stable ID, an FFE authoring `contract`, raw and normalized +patterns, native compile observations, an input, and an unanchored match +observation when the modeled engines agree. Go and RE2JS consumers compile +`normalizedPattern`; Rust consumers compile `rawPattern`. Cases with differences +between modeled engines include per-engine expectations. Downstream SDK checks +must require consistent behavior for accepted cases. A native engine accepting +rejected syntax does not change the authoring contract. The adjacent SHA-256 +file lets downstream tests detect fixture drift. + ## Automated Validation Pull requests run a blocking static validation check over `ufc-config.json` diff --git a/ci/run-downstream-conformance.sh b/ci/run-downstream-conformance.sh index 92929fb..bd079cb 100755 --- a/ci/run-downstream-conformance.sh +++ b/ci/run-downstream-conformance.sh @@ -119,7 +119,11 @@ if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then } >>"$GITHUB_STEP_SUMMARY" fi -# Reflect the proposed fixture result in the check conclusion. Branch protection -# decides whether this advisory check blocks merging; the check itself stays red -# until the downstream implementation accepts the proposed fixtures. -exit "$head_status" +# Fail only when the pull request introduces a downstream regression. Existing +# drift remains visible in the job summary and artifacts without blocking an +# unrelated fixture change. +if [[ $classification == new-regression ]]; then + exit 1 +fi + +exit 0 diff --git a/evaluation-cases/test-case-regex-flag.json b/evaluation-cases/test-case-regex-flag.json index 13ea0c3..b01f6f1 100644 --- a/evaluation-cases/test-case-regex-flag.json +++ b/evaluation-cases/test-case-regex-flag.json @@ -1,4 +1,43 @@ [ + { + "attributes": { + "comma_regex_case": "xxaaay" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "comma-quantifier" + }, + "targetingKey": "comma-quantifier", + "variationType": "STRING" + }, + { + "attributes": { + "whitespace_regex_case": "prefix exact suffix" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "literal-whitespace" + }, + "targetingKey": "literal-whitespace", + "variationType": "STRING" + }, + { + "attributes": { + "whitespace_regex_case": "exact" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "trimmed-whitespace-does-not-match", + "variationType": "STRING" + }, { "attributes": { "email": "user.name+tag@capture.example" diff --git a/regex-conformance/targeting-regex-conformance.json b/regex-conformance/targeting-regex-conformance.json new file mode 100644 index 0000000..64cd10b --- /dev/null +++ b/regex-conformance/targeting-regex-conformance.json @@ -0,0 +1,1065 @@ +{ + "schema": "datadog.ffe.targeting-regex-conformance/v1", + "schemaVersion": 1, + "contractVersion": "targeting-regex-v2", + "semantics": { + "acceptedSyntax": "The portable intersection exercised by accepted cases, not every construct accepted by an individual engine.", + "engines": [ + "RE2JS 1.2.2 using normalizedPattern", + "Go regexp using normalizedPattern", + "Rust regex 1.12 using rawPattern", + "Rust regex-syntax and regex-automata using rawPattern" + ], + "matchMode": "unanchored search unless the pattern contains anchors", + "nativeCompileFailureEvaluationResult": false, + "normalization": "For the inline flag groups exercised below, remove only the u flag and preserve the tested i, m, and s flags. Remove an empty global group and replace an empty scoped group with a non-capturing group.", + "contract": "accepted or rejected is the portable authoring decision. It is separate from native engine compilation.", + "expectedCompile": "The common native compile result. It is null when engines disagree; engineExpectations then records each result.", + "expectedMatch": "The common unanchored evaluation result. It is null when engines disagree or there is no portable match result.", + "engineExpectations": "When present, records native Go, RE2JS, Rust rules-based, and Rust rkyv compile and match observations. Match is null when that engine cannot compile the pattern." + }, + "portableSyntax": { + "accepted": [ + "literals and escaped literals", + "dot", + "start and end anchors", + "alternation", + "capturing and non-capturing groups", + "positive and negated character classes without a leading literal closing bracket", + "star, plus, optional, and counted quantifiers", + "Unicode literals", + "case-insensitive, multiline, dot-all, and standalone normalized Unicode inline flags" + ], + "rejected": [ + "malformed expressions", + "positive and negative lookahead", + "positive and negative lookbehind", + "numeric and named backreferences", + "octal escapes and quoted-literal escapes that are not accepted consistently", + "Perl shorthand classes and word boundaries with ASCII versus Unicode semantic differences", + "unsupported byte escapes", + "Rust-only character class set operators and nested non-POSIX character classes", + "inline Unicode flag forms whose raw Rust meaning differs from normalized Go and browser syntax", + "named capture groups", + "POSIX character classes", + "Unicode property character classes", + "combined or scoped Unicode inline flags", + "character classes with a leading literal closing bracket", + "atomic groups", + "possessive quantifiers" + ] + }, + "cases": [ + { + "id": "accepted-literal-unanchored-match", + "description": "A literal searches within a larger string.", + "category": "literal", + "contract": "accepted", + "rawPattern": "datadog", + "normalizedPattern": "datadog", + "expectedCompile": true, + "input": "visit datadoghq.com", + "expectedMatch": true + }, + { + "id": "accepted-literal-unanchored-no-match", + "description": "A valid literal can compile without matching the input.", + "category": "literal", + "contract": "accepted", + "rawPattern": "datadog", + "normalizedPattern": "datadog", + "expectedCompile": true, + "input": "example.com", + "expectedMatch": false + }, + { + "id": "accepted-escaped-literal-dot-no-match", + "description": "An escaped dot is a literal dot, not any character.", + "category": "escape", + "contract": "accepted", + "rawPattern": "datadoghq\\.com", + "normalizedPattern": "datadoghq\\.com", + "expectedCompile": true, + "input": "datadoghqXcom", + "expectedMatch": false + }, + { + "id": "accepted-dot-match", + "description": "Dot matches one non-newline character.", + "category": "dot", + "contract": "accepted", + "rawPattern": "d.g", + "normalizedPattern": "d.g", + "expectedCompile": true, + "input": "dog", + "expectedMatch": true + }, + { + "id": "accepted-anchors-full-match", + "description": "Start and end anchors require a full-string match.", + "category": "anchors", + "contract": "accepted", + "rawPattern": "^foo$", + "normalizedPattern": "^foo$", + "expectedCompile": true, + "input": "foo", + "expectedMatch": true + }, + { + "id": "accepted-anchors-reject-substring", + "description": "Anchors prevent a substring-only match.", + "category": "anchors", + "contract": "accepted", + "rawPattern": "^foo$", + "normalizedPattern": "^foo$", + "expectedCompile": true, + "input": "prefix foo suffix", + "expectedMatch": false + }, + { + "id": "accepted-alternation-capturing-group", + "description": "Alternation inside a capturing group is portable.", + "category": "alternation", + "contract": "accepted", + "rawPattern": "^(cat|dog)$", + "normalizedPattern": "^(cat|dog)$", + "expectedCompile": true, + "input": "dog", + "expectedMatch": true + }, + { + "id": "accepted-non-capturing-group", + "description": "A non-capturing group is portable.", + "category": "group", + "contract": "accepted", + "rawPattern": "^(?:cat|dog)$", + "normalizedPattern": "^(?:cat|dog)$", + "expectedCompile": true, + "input": "cat", + "expectedMatch": true + }, + { + "id": "rejected-python-style-named-group", + "description": "Python-style named captures are not accepted consistently by shipped SDK evaluators.", + "category": "named-group", + "contract": "rejected", + "rawPattern": "^(?P[[:alpha:]]+)$", + "normalizedPattern": "^(?P[[:alpha:]]+)$", + "expectedCompile": true, + "input": "Dog", + "expectedMatch": true + }, + { + "id": "rejected-js-style-named-group", + "description": "JS-style named captures are not evaluated consistently by shipped SDK evaluators.", + "category": "named-group", + "contract": "rejected", + "rawPattern": "^(?[[:alpha:]]+)$", + "normalizedPattern": "^(?[[:alpha:]]+)$", + "expectedCompile": true, + "input": "Dog", + "expectedMatch": true + }, + { + "id": "accepted-escaped-octal-like-literal", + "description": "A doubled backslash followed by digits is literal text, not an octal escape.", + "category": "literal-escape", + "contract": "accepted", + "rawPattern": "^\\\\123$", + "normalizedPattern": "^\\\\123$", + "expectedCompile": true, + "input": "\\123", + "expectedMatch": true + }, + { + "id": "accepted-escaped-shorthand-like-literal", + "description": "A doubled backslash followed by d is literal text, not a digit shorthand.", + "category": "literal-escape", + "contract": "accepted", + "rawPattern": "^\\\\d$", + "normalizedPattern": "^\\\\d$", + "expectedCompile": true, + "input": "\\d", + "expectedMatch": true + }, + { + "id": "accepted-escaped-quoted-literal-like-text", + "description": "A doubled backslash before Q is literal text, not the start of a quoted-literal escape.", + "category": "literal-escape", + "contract": "accepted", + "rawPattern": "^\\\\Qfoo$", + "normalizedPattern": "^\\\\Qfoo$", + "expectedCompile": true, + "input": "\\Qfoo", + "expectedMatch": true + }, + { + "id": "accepted-escaped-named-group-punctuation", + "description": "Escaped punctuation that spells (?< is literal text, not a named capture.", + "category": "literal-escape", + "contract": "accepted", + "rawPattern": "^\\(\\?<$", + "normalizedPattern": "^\\(\\?<$", + "expectedCompile": true, + "input": "(?<", + "expectedMatch": true + }, + { + "id": "accepted-named-group-punctuation-in-character-class", + "description": "The characters (?< inside a character class are literals.", + "category": "character-class", + "contract": "accepted", + "rawPattern": "^[(?<]+$", + "normalizedPattern": "^[(?<]+$", + "expectedCompile": true, + "input": "(?<", + "expectedMatch": true + }, + { + "id": "accepted-escaped-shorthand-in-character-class", + "description": "A doubled backslash before d inside a character class is literal syntax, not a digit shorthand.", + "category": "character-class", + "contract": "accepted", + "rawPattern": "^[\\\\d]$", + "normalizedPattern": "^[\\\\d]$", + "expectedCompile": true, + "input": "d", + "expectedMatch": true + }, + { + "id": "accepted-escaped-boundary-in-character-class", + "description": "A doubled backslash before b inside a character class is literal syntax, not a word boundary.", + "category": "character-class", + "contract": "accepted", + "rawPattern": "^[\\\\b]$", + "normalizedPattern": "^[\\\\b]$", + "expectedCompile": true, + "input": "b", + "expectedMatch": true + }, + { + "id": "accepted-positive-character-class", + "description": "A positive character class is portable.", + "category": "character-class", + "contract": "accepted", + "rawPattern": "^[A-Z][0-9]$", + "normalizedPattern": "^[A-Z][0-9]$", + "expectedCompile": true, + "input": "Q7", + "expectedMatch": true + }, + { + "id": "accepted-negated-character-class", + "description": "A negated character class is portable.", + "category": "character-class", + "contract": "accepted", + "rawPattern": "^[^0-9]+$", + "normalizedPattern": "^[^0-9]+$", + "expectedCompile": true, + "input": "alpha", + "expectedMatch": true + }, + { + "id": "accepted-star-quantifier", + "description": "The star quantifier accepts zero repetitions.", + "category": "quantifier", + "contract": "accepted", + "rawPattern": "^ab*c$", + "normalizedPattern": "^ab*c$", + "expectedCompile": true, + "input": "ac", + "expectedMatch": true + }, + { + "id": "accepted-plus-quantifier-no-match", + "description": "The plus quantifier requires one or more repetitions.", + "category": "quantifier", + "contract": "accepted", + "rawPattern": "^ab+c$", + "normalizedPattern": "^ab+c$", + "expectedCompile": true, + "input": "ac", + "expectedMatch": false + }, + { + "id": "accepted-optional-quantifier", + "description": "The optional quantifier accepts a missing character.", + "category": "quantifier", + "contract": "accepted", + "rawPattern": "^colou?r$", + "normalizedPattern": "^colou?r$", + "expectedCompile": true, + "input": "color", + "expectedMatch": true + }, + { + "id": "accepted-comma-counted-quantifier", + "description": "A comma inside a counted quantifier remains part of one regex value.", + "category": "quantifier", + "contract": "accepted", + "rawPattern": "a{1,3}", + "normalizedPattern": "a{1,3}", + "expectedCompile": true, + "input": "caaad", + "expectedMatch": true + }, + { + "id": "rejected-posix-alpha-class", + "description": "POSIX character classes are not evaluated consistently by shipped SDK evaluators.", + "category": "posix-class", + "contract": "rejected", + "rawPattern": "^[[:alpha:]]+$", + "normalizedPattern": "^[[:alpha:]]+$", + "expectedCompile": true, + "input": "Alpha", + "expectedMatch": true + }, + { + "id": "rejected-unicode-letter-class", + "description": "Unicode property character classes are not evaluated consistently by shipped SDK evaluators.", + "category": "unicode-class", + "contract": "rejected", + "rawPattern": "^\\p{L}+$", + "normalizedPattern": "^\\p{L}+$", + "expectedCompile": true, + "input": "mañana", + "expectedMatch": true + }, + { + "id": "accepted-unicode-literal-unanchored", + "description": "Unicode literal text is portable with unanchored search.", + "category": "unicode-literal", + "contract": "accepted", + "rawPattern": "mañana", + "normalizedPattern": "mañana", + "expectedCompile": true, + "input": "hasta mañana!", + "expectedMatch": true + }, + { + "id": "accepted-inline-case-insensitive", + "description": "The case-insensitive inline flag is portable.", + "category": "inline-flag", + "contract": "accepted", + "rawPattern": "(?i)^alice$", + "normalizedPattern": "(?i)^alice$", + "expectedCompile": true, + "input": "ALICE", + "expectedMatch": true + }, + { + "id": "accepted-inline-multiline", + "description": "The multiline inline flag changes anchors to line boundaries.", + "category": "inline-flag", + "contract": "accepted", + "rawPattern": "(?m)^second$", + "normalizedPattern": "(?m)^second$", + "expectedCompile": true, + "input": "first\nsecond\nthird", + "expectedMatch": true + }, + { + "id": "accepted-inline-dot-all", + "description": "The dot-all inline flag lets dot match a newline.", + "category": "inline-flag", + "contract": "accepted", + "rawPattern": "(?s)^a.b$", + "normalizedPattern": "(?s)^a.b$", + "expectedCompile": true, + "input": "a\nb", + "expectedMatch": true + }, + { + "id": "accepted-inline-unicode-global-normalized", + "description": "A global Unicode flag is redundant and removed for Go and RE2JS.", + "category": "normalization", + "contract": "accepted", + "rawPattern": "(?u)foo", + "normalizedPattern": "foo", + "expectedCompile": true, + "input": "seafood", + "expectedMatch": true + }, + { + "id": "rejected-inline-unicode-combined-normalized", + "description": "Combined Unicode inline flags are not normalized consistently by shipped SDK evaluators.", + "category": "normalization", + "contract": "rejected", + "rawPattern": "(?iu)^alice$", + "normalizedPattern": "(?i)^alice$", + "expectedCompile": true, + "input": "ALICE", + "expectedMatch": true + }, + { + "id": "rejected-inline-unicode-scoped-normalized", + "description": "Scoped Unicode inline flags are not normalized consistently by shipped SDK evaluators.", + "category": "normalization", + "contract": "rejected", + "rawPattern": "(?u:^mañana$)", + "normalizedPattern": "(?:^mañana$)", + "expectedCompile": true, + "input": "mañana", + "expectedMatch": true + }, + { + "id": "rejected-inline-unicode-scoped-combined-normalized", + "description": "Combined scoped Unicode inline flags are not normalized consistently by shipped SDK evaluators.", + "category": "normalization", + "contract": "rejected", + "rawPattern": "(?iu:^alice$)", + "normalizedPattern": "(?i:^alice$)", + "expectedCompile": true, + "input": "ALICE", + "expectedMatch": true + }, + { + "id": "accepted-single-ampersand-in-character-class", + "description": "A single ampersand in an ordinary character class is literal text, not a Rust set-intersection operator.", + "category": "character-class", + "contract": "accepted", + "rawPattern": "^[a&]+$", + "normalizedPattern": "^[a&]+$", + "expectedCompile": true, + "input": "&", + "expectedMatch": true + }, + { + "id": "accepted-single-tilde-in-character-class", + "description": "A single tilde in an ordinary character class is literal text, not a Rust set-symmetric-difference operator.", + "category": "character-class", + "contract": "accepted", + "rawPattern": "^[a~]+$", + "normalizedPattern": "^[a~]+$", + "expectedCompile": true, + "input": "~", + "expectedMatch": true + }, + { + "id": "accepted-single-hyphen-in-character-class", + "description": "A trailing single hyphen in a character class is literal text, not a Rust set-difference operator.", + "category": "character-class", + "contract": "accepted", + "rawPattern": "^[a-]+$", + "normalizedPattern": "^[a-]+$", + "expectedCompile": true, + "input": "-", + "expectedMatch": true + }, + { + "id": "accepted-range-in-character-class", + "description": "An ordinary character range remains part of the portable contract.", + "category": "character-class", + "contract": "accepted", + "rawPattern": "^[a-z]+$", + "normalizedPattern": "^[a-z]+$", + "expectedCompile": true, + "input": "range", + "expectedMatch": true + }, + { + "id": "rejected-leading-closing-bracket-in-character-class", + "description": "A leading literal closing bracket in a character class is not evaluated consistently by shipped SDK evaluators.", + "category": "character-class", + "contract": "rejected", + "rawPattern": "^[]a]+$", + "normalizedPattern": "^[]a]+$", + "expectedCompile": true, + "input": "]", + "expectedMatch": true + }, + { + "id": "rejected-leading-closing-bracket-in-negated-character-class", + "description": "A leading literal closing bracket in a negated character class is not evaluated consistently by shipped SDK evaluators.", + "category": "character-class", + "contract": "rejected", + "rawPattern": "^[^]a]+$", + "normalizedPattern": "^[^]a]+$", + "expectedCompile": true, + "input": "b", + "expectedMatch": true + }, + { + "id": "rejected-rust-character-class-intersection", + "description": "Double ampersand is a set-intersection operator in Rust but literal class text in Go and RE2JS.", + "category": "character-class-dialect", + "contract": "rejected", + "rawPattern": "^[a-z&&b-z]+$", + "normalizedPattern": "^[a-z&&b-z]+$", + "expectedCompile": true, + "input": "&", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": true, "match": false }, + "rustRkyv": { "compile": true, "match": false } + } + }, + { + "id": "rejected-rust-character-class-intersection-after-leading-closing-bracket", + "description": "A leading closing bracket stays inside the class; the later double ampersand is a Rust intersection operator but literal class text in Go and RE2JS.", + "category": "character-class-dialect", + "contract": "rejected", + "rawPattern": "^[]a&&b]+$", + "normalizedPattern": "^[]a&&b]+$", + "expectedCompile": true, + "input": "&", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": true, "match": false }, + "rustRkyv": { "compile": true, "match": false } + } + }, + { + "id": "rejected-rust-character-class-intersection-after-leading-closing-bracket-in-negated-class", + "description": "A leading closing bracket after negation stays inside the class; Rust complements an empty intersection while Go and RE2JS treat ampersands as excluded literals.", + "category": "character-class-dialect", + "contract": "rejected", + "rawPattern": "^[^]a&&b]+$", + "normalizedPattern": "^[^]a&&b]+$", + "expectedCompile": true, + "input": "&", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": false }, + "re2js": { "compile": true, "match": false }, + "rustRulesBased": { "compile": true, "match": true }, + "rustRkyv": { "compile": true, "match": true } + } + }, + { + "id": "rejected-rust-character-class-symmetric-difference", + "description": "Double tilde is a set-symmetric-difference operator in Rust but literal class text in Go and RE2JS.", + "category": "character-class-dialect", + "contract": "rejected", + "rawPattern": "^[a-z~~b-z]+$", + "normalizedPattern": "^[a-z~~b-z]+$", + "expectedCompile": true, + "input": "~", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": true, "match": false }, + "rustRkyv": { "compile": true, "match": false } + } + }, + { + "id": "rejected-rust-character-class-difference", + "description": "Double hyphen is a set-difference operator in Rust but literal class text in Go and RE2JS.", + "category": "character-class-dialect", + "contract": "rejected", + "rawPattern": "^[a-z--b-z]+$", + "normalizedPattern": "^[a-z--b-z]+$", + "expectedCompile": true, + "input": "-", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": true, "match": false }, + "rustRkyv": { "compile": true, "match": false } + } + }, + { + "id": "rejected-rust-nested-character-class", + "description": "Rust treats a nested class as a union, while Go and RE2JS parse the same bytes with different match behavior.", + "category": "character-class-dialect", + "contract": "rejected", + "rawPattern": "^[a-z[0-9]]+$", + "normalizedPattern": "^[a-z[0-9]]+$", + "expectedCompile": true, + "input": "5", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": false }, + "re2js": { "compile": true, "match": false }, + "rustRulesBased": { "compile": true, "match": true }, + "rustRkyv": { "compile": true, "match": true } + } + }, + { + "id": "rejected-inline-flag-token-in-character-class", + "description": "The raw class contains a literal u, but current Go and browser normalization removes it as if it were an inline flag.", + "category": "normalization-dialect", + "contract": "rejected", + "rawPattern": "^[(?iu)]+$", + "normalizedPattern": "^[(?i)]+$", + "expectedCompile": true, + "input": "u", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": false }, + "re2js": { "compile": true, "match": false }, + "rustRulesBased": { "compile": true, "match": true }, + "rustRkyv": { "compile": true, "match": true } + } + }, + { + "id": "rejected-malformed-inline-unicode-flags", + "description": "The raw Rust flag group is malformed, while current Go and browser normalization removes it and accepts the remaining literal.", + "category": "normalization-dialect", + "contract": "rejected", + "rawPattern": "(?u-)foo", + "normalizedPattern": "foo", + "expectedCompile": null, + "input": "foo", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": false, "match": null }, + "rustRkyv": { "compile": false, "match": null } + } + }, + { + "id": "rejected-disabled-unicode-scoped-class", + "description": "Rust rejects a Unicode class inside a Unicode-disabled scope, while normalization removes the disabling flag and accepts it.", + "category": "normalization-dialect", + "contract": "rejected", + "rawPattern": "(?-u:\\p{L}+)", + "normalizedPattern": "(?:\\p{L}+)", + "expectedCompile": null, + "input": "ñ", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": false, "match": null }, + "rustRkyv": { "compile": false, "match": null } + } + }, + { + "id": "rejected-disabled-unicode-global-class", + "description": "Rust rejects a Unicode class after Unicode is disabled globally, while normalization removes the disabling flag and accepts it.", + "category": "normalization-dialect", + "contract": "rejected", + "rawPattern": "(?-u)\\p{L}+", + "normalizedPattern": "\\p{L}+", + "expectedCompile": null, + "input": "ñ", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": false, "match": null }, + "rustRkyv": { "compile": false, "match": null } + } + }, + { + "id": "rejected-disabled-unicode-negated-class", + "description": "Rust rejects a broad negated class inside a Unicode-disabled scope, while normalization removes the disabling flag and accepts it.", + "category": "normalization-dialect", + "contract": "rejected", + "rawPattern": "(?-u:[^a])", + "normalizedPattern": "(?:[^a])", + "expectedCompile": null, + "input": "ñ", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": false, "match": null }, + "rustRkyv": { "compile": false, "match": null } + } + }, + { + "id": "rejected-octal-escape", + "description": "Octal escapes are accepted by Go and RE2JS but disabled by default in Rust, so they are outside the portable contract.", + "category": "dialect-escape", + "contract": "rejected", + "rawPattern": "\\123", + "normalizedPattern": "\\123", + "expectedCompile": null, + "input": "S", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": false, "match": null }, + "rustRkyv": { "compile": false, "match": null } + } + }, + { + "id": "rejected-quoted-literal-escape", + "description": "Quoted-literal escapes are accepted by Go and RE2JS but rejected by Rust, so they are outside the portable contract.", + "category": "dialect-escape", + "contract": "rejected", + "rawPattern": "\\Qfoo.bar\\E", + "normalizedPattern": "\\Qfoo.bar\\E", + "expectedCompile": null, + "input": "foo.bar", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": false, "match": null }, + "rustRkyv": { "compile": false, "match": null } + } + }, + { + "id": "rejected-byte-escape", + "description": "The byte escape is unsupported by the targeting regex engines and falls through to native compiler rejection.", + "category": "dialect-escape", + "contract": "rejected", + "rawPattern": "\\C", + "normalizedPattern": "\\C", + "expectedCompile": false, + "input": "A", + "expectedMatch": false + }, + { + "id": "rejected-digit-shorthand-unicode-divergence", + "description": "Digit shorthand is ASCII-oriented in Go and RE2JS but Unicode-aware in Rust.", + "category": "semantic-shorthand", + "contract": "rejected", + "rawPattern": "^\\d+$", + "normalizedPattern": "^\\d+$", + "expectedCompile": true, + "input": "٣", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": false }, + "re2js": { "compile": true, "match": false }, + "rustRulesBased": { "compile": true, "match": true }, + "rustRkyv": { "compile": true, "match": true } + } + }, + { + "id": "rejected-negated-digit-shorthand-unicode-divergence", + "description": "Negated digit shorthand is ASCII-oriented in Go and RE2JS but Unicode-aware in Rust.", + "category": "semantic-shorthand", + "contract": "rejected", + "rawPattern": "^\\D+$", + "normalizedPattern": "^\\D+$", + "expectedCompile": true, + "input": "٣", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": true, "match": false }, + "rustRkyv": { "compile": true, "match": false } + } + }, + { + "id": "rejected-space-shorthand-unicode-divergence", + "description": "Space shorthand excludes non-breaking space in Go and RE2JS but includes it in Rust.", + "category": "semantic-shorthand", + "contract": "rejected", + "rawPattern": "^\\s+$", + "normalizedPattern": "^\\s+$", + "expectedCompile": true, + "input": " ", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": false }, + "re2js": { "compile": true, "match": false }, + "rustRulesBased": { "compile": true, "match": true }, + "rustRkyv": { "compile": true, "match": true } + } + }, + { + "id": "rejected-negated-space-shorthand-unicode-divergence", + "description": "Negated space shorthand includes non-breaking space in Go and RE2JS but excludes it in Rust.", + "category": "semantic-shorthand", + "contract": "rejected", + "rawPattern": "^\\S+$", + "normalizedPattern": "^\\S+$", + "expectedCompile": true, + "input": " ", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": true, "match": false }, + "rustRkyv": { "compile": true, "match": false } + } + }, + { + "id": "rejected-word-shorthand-unicode-divergence", + "description": "Word shorthand excludes a non-ASCII letter in Go and RE2JS but includes it in Rust.", + "category": "semantic-shorthand", + "contract": "rejected", + "rawPattern": "^\\w+$", + "normalizedPattern": "^\\w+$", + "expectedCompile": true, + "input": "ñ", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": false }, + "re2js": { "compile": true, "match": false }, + "rustRulesBased": { "compile": true, "match": true }, + "rustRkyv": { "compile": true, "match": true } + } + }, + { + "id": "rejected-negated-word-shorthand-unicode-divergence", + "description": "Negated word shorthand includes a non-ASCII letter in Go and RE2JS but excludes it in Rust.", + "category": "semantic-shorthand", + "contract": "rejected", + "rawPattern": "^\\W+$", + "normalizedPattern": "^\\W+$", + "expectedCompile": true, + "input": "ñ", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": true, "match": false }, + "rustRkyv": { "compile": true, "match": false } + } + }, + { + "id": "rejected-word-boundary-unicode-divergence", + "description": "Word boundary is ASCII-oriented in Go and RE2JS but Unicode-aware in Rust.", + "category": "semantic-boundary", + "contract": "rejected", + "rawPattern": "\\bñ\\b", + "normalizedPattern": "\\bñ\\b", + "expectedCompile": true, + "input": "ñ", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": false }, + "re2js": { "compile": true, "match": false }, + "rustRulesBased": { "compile": true, "match": true }, + "rustRkyv": { "compile": true, "match": true } + } + }, + { + "id": "rejected-negated-word-boundary-unicode-divergence", + "description": "Negated word boundary is ASCII-oriented in Go and RE2JS but Unicode-aware in Rust.", + "category": "semantic-boundary", + "contract": "rejected", + "rawPattern": "^\\Bñ\\B$", + "normalizedPattern": "^\\Bñ\\B$", + "expectedCompile": true, + "input": "ñ", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": true }, + "re2js": { "compile": true, "match": true }, + "rustRulesBased": { "compile": true, "match": false }, + "rustRkyv": { "compile": true, "match": false } + } + }, + { + "id": "rejected-digit-shorthand-in-character-class", + "description": "Digit shorthand keeps the same ASCII versus Unicode semantic difference inside a character class.", + "category": "semantic-shorthand", + "contract": "rejected", + "rawPattern": "^[\\d]$", + "normalizedPattern": "^[\\d]$", + "expectedCompile": true, + "input": "٣", + "expectedMatch": null, + "engineExpectations": { + "go": { "compile": true, "match": false }, + "re2js": { "compile": true, "match": false }, + "rustRulesBased": { "compile": true, "match": true }, + "rustRkyv": { "compile": true, "match": true } + } + }, + { + "id": "rejected-boundary-escape-in-character-class", + "description": "The boundary escape is not accepted consistently inside a character class.", + "category": "semantic-boundary", + "contract": "rejected", + "rawPattern": "^[\\b]$", + "normalizedPattern": "^[\\b]$", + "expectedCompile": false, + "input": "b", + "expectedMatch": false + }, + { + "id": "rejected-unclosed-character-class", + "description": "Malformed character classes are not authorable and fail closed.", + "category": "malformed", + "contract": "rejected", + "rawPattern": "[", + "normalizedPattern": "[", + "expectedCompile": false, + "input": "anything", + "expectedMatch": false + }, + { + "id": "rejected-dangling-quantifier", + "description": "A quantifier without an expression is malformed and fails closed.", + "category": "malformed", + "contract": "rejected", + "rawPattern": "*foo", + "normalizedPattern": "*foo", + "expectedCompile": false, + "input": "foo", + "expectedMatch": false + }, + { + "id": "rejected-positive-lookahead", + "description": "Positive lookahead is outside the portable syntax contract.", + "category": "lookaround", + "contract": "rejected", + "rawPattern": "foo(?=bar)", + "normalizedPattern": "foo(?=bar)", + "expectedCompile": false, + "input": "foobar", + "expectedMatch": false + }, + { + "id": "rejected-negative-lookahead", + "description": "Negative lookahead is outside the portable syntax contract.", + "category": "lookaround", + "contract": "rejected", + "rawPattern": "foo(?!bar)", + "normalizedPattern": "foo(?!bar)", + "expectedCompile": false, + "input": "foobaz", + "expectedMatch": false + }, + { + "id": "rejected-positive-lookbehind", + "description": "Positive lookbehind is outside the portable syntax contract.", + "category": "lookaround", + "contract": "rejected", + "rawPattern": "(?<=foo)bar", + "normalizedPattern": "(?<=foo)bar", + "expectedCompile": false, + "input": "foobar", + "expectedMatch": false + }, + { + "id": "rejected-negative-lookbehind", + "description": "Negative lookbehind is outside the portable syntax contract.", + "category": "lookaround", + "contract": "rejected", + "rawPattern": "(?a)$", + "normalizedPattern": "^(?P<1>a)$", + "expectedCompile": null, + "input": "a", + "expectedMatch": null, + "engineExpectations": { + "go": { + "compile": true, + "match": true + }, + "re2js": { + "compile": true, + "match": true + }, + "rustRulesBased": { + "compile": false, + "match": null + }, + "rustRkyv": { + "compile": false, + "match": null + } + } + }, + { + "id": "rejected-duplicate-named-group", + "description": "Go accepts a duplicate capture name, but RE2JS and both Rust evaluators reject it.", + "category": "named-group-dialect", + "contract": "rejected", + "rawPattern": "^(?Pa)(?b)$", + "normalizedPattern": "^(?Pa)(?b)$", + "expectedCompile": null, + "input": "ab", + "expectedMatch": null, + "engineExpectations": { + "go": { + "compile": true, + "match": true + }, + "re2js": { + "compile": false, + "match": null + }, + "rustRulesBased": { + "compile": false, + "match": null + }, + "rustRkyv": { + "compile": false, + "match": null + } + } + }, + { + "id": "rejected-named-backreference", + "description": "Named backreferences and named capture groups are outside the portable syntax contract.", + "category": "backreference", + "contract": "rejected", + "rawPattern": "(?Pa)\\k", + "normalizedPattern": "(?Pa)\\k", + "expectedCompile": false, + "input": "aa", + "expectedMatch": false + }, + { + "id": "rejected-atomic-group", + "description": "Atomic groups are outside the portable syntax contract.", + "category": "group", + "contract": "rejected", + "rawPattern": "(?>foo)", + "normalizedPattern": "(?>foo)", + "expectedCompile": false, + "input": "foo", + "expectedMatch": false + }, + { + "id": "rejected-possessive-quantifier", + "description": "Possessive quantifiers are outside the portable syntax contract.", + "category": "quantifier", + "contract": "rejected", + "rawPattern": "a++", + "normalizedPattern": "a++", + "expectedCompile": null, + "input": "aaa", + "expectedMatch": null, + "engineExpectations": { + "go": { + "compile": false, + "match": null + }, + "re2js": { + "compile": false, + "match": null + }, + "rustRulesBased": { + "compile": true, + "match": true + }, + "rustRkyv": { + "compile": true, + "match": true + } + } + } + ] +} diff --git a/regex-conformance/targeting-regex-conformance.sha256 b/regex-conformance/targeting-regex-conformance.sha256 new file mode 100644 index 0000000..52f81f6 --- /dev/null +++ b/regex-conformance/targeting-regex-conformance.sha256 @@ -0,0 +1 @@ +4f8ef95faec9faa3c2b0756ac61afe862e35c0646cba5375532a85335f209e8f targeting-regex-conformance.json diff --git a/regex-conformance/test-validate-targeting-regex-conformance.sh b/regex-conformance/test-validate-targeting-regex-conformance.sh new file mode 100755 index 0000000..d1480b5 --- /dev/null +++ b/regex-conformance/test-validate-targeting-regex-conformance.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +fixture="$script_dir/targeting-regex-conformance.json" +validator="$script_dir/validate-targeting-regex-conformance.jq" +temporary_file=$(mktemp) + +cleanup() { + rm -f "$temporary_file" +} + +handle_signal() { + signal=$1 + trap - "$signal" + cleanup + kill -s "$signal" "$$" +} + +trap cleanup 0 +trap 'handle_signal HUP' HUP +trap 'handle_signal INT' INT +trap 'handle_signal TERM' TERM + +expect_invalid() { + description=$1 + mutation=$2 + + jq "$mutation" "$fixture" >"$temporary_file" + if jq -e -f "$validator" "$temporary_file" >/dev/null; then + echo "validator accepted invalid fixture: $description" >&2 + exit 1 + fi +} + +jq -e -f "$validator" "$fixture" >/dev/null + +expect_invalid \ + "compile failure with a true match result" \ + '.cases |= map(if .id == "rejected-byte-escape" then .expectedMatch = true else . end)' + +expect_invalid \ + "empty semantics" \ + '.semantics = {}' + +expect_invalid \ + "empty portable syntax contract" \ + '.portableSyntax.accepted = [] | .portableSyntax.rejected = []' + +echo "targeting regex conformance validator tests passed" diff --git a/regex-conformance/validate-targeting-regex-conformance.jq b/regex-conformance/validate-targeting-regex-conformance.jq new file mode 100644 index 0000000..78a24e3 --- /dev/null +++ b/regex-conformance/validate-targeting-regex-conformance.jq @@ -0,0 +1,98 @@ +def nonempty_string: + type == "string" and length > 0; + +def nullable_boolean: + type == "boolean" or . == null; + +def nonempty_string_array: + type == "array" and + length > 0 and + all(.[]; nonempty_string) and + length == (unique | length); + +def valid_semantics: + type == "object" and + (.acceptedSyntax | nonempty_string) and + (.engines | nonempty_string_array) and + (.matchMode | nonempty_string) and + .nativeCompileFailureEvaluationResult == false and + (.normalization | nonempty_string) and + (.contract | nonempty_string) and + (.expectedCompile | nonempty_string) and + (.expectedMatch | nonempty_string) and + (.engineExpectations | nonempty_string); + +def valid_portable_syntax: + type == "object" and + (.accepted | nonempty_string_array) and + (.rejected | nonempty_string_array) and + ([.accepted[], .rejected[]] | length) == + ([.accepted[], .rejected[]] | unique | length); + +def expected_engine_keys: + ["go", "re2js", "rustRkyv", "rustRulesBased"]; + +def valid_engine_expectation: + has("compile") and + has("match") and + (.compile | type) == "boolean" and + (.match | nullable_boolean) and + (if .compile then (.match | type) == "boolean" else .match == null end); + +def valid_engine_expectations: + (.engineExpectations | keys | sort) == expected_engine_keys and + all(.engineExpectations[]; valid_engine_expectation) and + ([.engineExpectations[].compile] | unique) as $compile_values | + ([.engineExpectations[].match] | unique) as $match_values | + (if ($compile_values | length) == 1 + then .expectedCompile == $compile_values[0] + else .expectedCompile == null + end) and + (if ($match_values | length) == 1 + then .expectedMatch == $match_values[0] + else .expectedMatch == null + end); + +def valid_case: + . as $case | + has("id") and + has("description") and + has("category") and + has("contract") and + has("rawPattern") and + has("normalizedPattern") and + has("expectedCompile") and + has("input") and + has("expectedMatch") and + (.id | nonempty_string) and + (.description | nonempty_string) and + (.category | nonempty_string) and + (.contract == "accepted" or .contract == "rejected") and + (.id | startswith($case.contract + "-")) and + (.rawPattern | type) == "string" and + (.normalizedPattern | type) == "string" and + (.expectedCompile | nullable_boolean) and + (.input | type) == "string" and + (.expectedMatch | nullable_boolean) and + (if .contract == "accepted" + then .expectedCompile == true and + (.expectedMatch | type) == "boolean" and + has("engineExpectations") == false + else true + end) and + (if has("engineExpectations") + then valid_engine_expectations + else .expectedCompile != null and + .expectedMatch != null and + (if .expectedCompile == false then .expectedMatch == false else true end) + end); + +.schema == "datadog.ffe.targeting-regex-conformance/v1" and +.schemaVersion == 1 and +(.contractVersion | nonempty_string) and +(.semantics | valid_semantics) and +(.portableSyntax | valid_portable_syntax) and +(.cases | type) == "array" and +(.cases | length) > 0 and +([.cases[].id] | length) == ([.cases[].id] | unique | length) and +all(.cases[]; valid_case) diff --git a/ufc-config.json b/ufc-config.json index 52fd31d..59ca09a 100644 --- a/ufc-config.json +++ b/ufc-config.json @@ -89,9 +89,59 @@ "unicode-word-boundary": { "key": "unicode-word-boundary", "value": "unicode-word-boundary" + }, + "comma-quantifier": { + "key": "comma-quantifier", + "value": "comma-quantifier" + }, + "literal-whitespace": { + "key": "literal-whitespace", + "value": "literal-whitespace" } }, "allocations": [ + { + "key": "comma-quantifier", + "rules": [ + { + "conditions": [ + { + "attribute": "comma_regex_case", + "operator": "MATCHES", + "value": "a{1,3}" + } + ] + } + ], + "splits": [ + { + "variationKey": "comma-quantifier", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "literal-whitespace", + "rules": [ + { + "conditions": [ + { + "attribute": "whitespace_regex_case", + "operator": "MATCHES", + "value": " exact " + } + ] + } + ], + "splits": [ + { + "variationKey": "literal-whitespace", + "shards": [] + } + ], + "doLog": true + }, { "key": "capturing-groups", "rules": [