diff --git a/.changeset/patch-mention-filtering-refactor.md b/.changeset/patch-mention-filtering-refactor.md new file mode 100644 index 00000000000..5edc1757f53 --- /dev/null +++ b/.changeset/patch-mention-filtering-refactor.md @@ -0,0 +1,10 @@ +--- +"gh-aw": patch +--- + +Move mention filtering from incoming text processing to the agent output collector. + +This is an internal refactor and bugfix: sanitizers were modularized, mention +resolution was moved into the output collector, and a bug that prevented known +authors from being preserved in mentions was fixed. Tests were updated. + diff --git a/.github/workflows/ai-moderator.lock.yml b/.github/workflows/ai-moderator.lock.yml index 835578f0f34..eb12b0bbdad 100644 --- a/.github/workflows/ai-moderator.lock.yml +++ b/.github/workflows/ai-moderator.lock.yml @@ -3511,6 +3511,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3548,6 +3551,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3557,6 +3740,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3581,13 +3767,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3604,61 +3790,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3668,13 +3861,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3722,10 +3917,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3941,7 +4136,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4005,12 +4200,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4038,7 +4239,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4102,7 +4308,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4118,7 +4324,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4142,6 +4348,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4213,7 +4636,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4244,11 +4667,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4378,7 +4801,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/archie.lock.yml b/.github/workflows/archie.lock.yml index 0b9a8c7a04c..ce8c17fb219 100644 --- a/.github/workflows/archie.lock.yml +++ b/.github/workflows/archie.lock.yml @@ -418,6 +418,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -457,15 +460,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -490,13 +485,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -513,61 +508,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -577,14 +578,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -641,6 +640,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -942,16 +944,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -4734,6 +4727,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4771,6 +4767,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4780,6 +4956,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4804,13 +4983,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4827,61 +5006,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4891,13 +5077,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4945,10 +5133,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5164,7 +5352,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5228,12 +5416,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5261,7 +5455,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5325,7 +5524,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5341,7 +5540,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5365,6 +5564,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5436,7 +5852,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5467,11 +5883,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5601,7 +6017,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/artifacts-summary.lock.yml b/.github/workflows/artifacts-summary.lock.yml index 584ec64ce34..e07d691ad1d 100644 --- a/.github/workflows/artifacts-summary.lock.yml +++ b/.github/workflows/artifacts-summary.lock.yml @@ -2862,6 +2862,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2899,6 +2902,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2908,6 +3091,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -2932,13 +3118,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -2955,61 +3141,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3019,13 +3212,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3073,10 +3268,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3292,7 +3487,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3356,12 +3551,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3389,7 +3590,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3453,7 +3659,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3469,7 +3675,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3493,6 +3699,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3564,7 +3987,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3595,11 +4018,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3729,7 +4152,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/audit-workflows.lock.yml b/.github/workflows/audit-workflows.lock.yml index 93ddf6b1bc9..60a51e14976 100644 --- a/.github/workflows/audit-workflows.lock.yml +++ b/.github/workflows/audit-workflows.lock.yml @@ -4428,6 +4428,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4465,6 +4468,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4474,6 +4657,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4498,13 +4684,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4521,61 +4707,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4585,13 +4778,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4639,10 +4834,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4858,7 +5053,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4922,12 +5117,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4955,7 +5156,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5019,7 +5225,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5035,7 +5241,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5059,6 +5265,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5130,7 +5553,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5161,11 +5584,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5295,7 +5718,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/blog-auditor.lock.yml b/.github/workflows/blog-auditor.lock.yml index 9de967daa4b..a36048e3cef 100644 --- a/.github/workflows/blog-auditor.lock.yml +++ b/.github/workflows/blog-auditor.lock.yml @@ -3491,6 +3491,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3528,6 +3531,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3537,6 +3720,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3561,13 +3747,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3584,61 +3770,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3648,13 +3841,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3702,10 +3897,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3921,7 +4116,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3985,12 +4180,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4018,7 +4219,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4082,7 +4288,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4098,7 +4304,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4122,6 +4328,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4193,7 +4616,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4224,11 +4647,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4358,7 +4781,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/brave.lock.yml b/.github/workflows/brave.lock.yml index 51138d76454..7316700efec 100644 --- a/.github/workflows/brave.lock.yml +++ b/.github/workflows/brave.lock.yml @@ -316,6 +316,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -355,15 +358,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -388,13 +383,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -411,61 +406,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -475,14 +476,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -539,6 +538,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -840,16 +842,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -4525,6 +4518,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4562,6 +4558,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4571,6 +4747,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4595,13 +4774,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4618,61 +4797,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4682,13 +4868,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4736,10 +4924,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4955,7 +5143,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5019,12 +5207,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5052,7 +5246,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5116,7 +5315,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5132,7 +5331,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5156,6 +5355,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5227,7 +5643,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5258,11 +5674,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5392,7 +5808,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/breaking-change-checker.lock.yml b/.github/workflows/breaking-change-checker.lock.yml index 928d49f2b77..6cfab4adb2b 100644 --- a/.github/workflows/breaking-change-checker.lock.yml +++ b/.github/workflows/breaking-change-checker.lock.yml @@ -2945,6 +2945,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2982,6 +2985,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2991,6 +3174,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3015,13 +3201,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3038,61 +3224,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3102,13 +3295,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3156,10 +3351,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3375,7 +3570,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3439,12 +3634,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3472,7 +3673,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3536,7 +3742,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3552,7 +3758,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3576,6 +3782,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3647,7 +4070,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3678,11 +4101,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3812,7 +4235,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/changeset.lock.yml b/.github/workflows/changeset.lock.yml index 77e069c79f2..f21bed2f41c 100644 --- a/.github/workflows/changeset.lock.yml +++ b/.github/workflows/changeset.lock.yml @@ -464,6 +464,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -503,15 +506,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -536,13 +531,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -559,61 +554,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -623,14 +624,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -687,6 +686,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -988,16 +990,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -4042,6 +4035,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4079,6 +4075,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4088,6 +4264,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4112,13 +4291,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4135,61 +4314,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4199,13 +4385,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4253,10 +4441,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4472,7 +4660,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4536,12 +4724,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4569,7 +4763,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4633,7 +4832,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4649,7 +4848,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4673,6 +4872,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4744,7 +5160,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4775,11 +5191,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4909,7 +5325,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/ci-coach.lock.yml b/.github/workflows/ci-coach.lock.yml index 403d44a5461..f60e663b3a0 100644 --- a/.github/workflows/ci-coach.lock.yml +++ b/.github/workflows/ci-coach.lock.yml @@ -4171,6 +4171,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4208,6 +4211,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4217,6 +4400,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4241,13 +4427,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4264,61 +4450,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4328,13 +4521,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4382,10 +4577,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4601,7 +4796,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4665,12 +4860,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4698,7 +4899,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4762,7 +4968,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4778,7 +4984,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4802,6 +5008,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4873,7 +5296,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4904,11 +5327,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5038,7 +5461,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/ci-doctor.lock.yml b/.github/workflows/ci-doctor.lock.yml index 27c0d652a1d..1582bf6f0fb 100644 --- a/.github/workflows/ci-doctor.lock.yml +++ b/.github/workflows/ci-doctor.lock.yml @@ -3807,6 +3807,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3844,6 +3847,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3853,6 +4036,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3877,13 +4063,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3900,61 +4086,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3964,13 +4157,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4018,10 +4213,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4237,7 +4432,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4301,12 +4496,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4334,7 +4535,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4398,7 +4604,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4414,7 +4620,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4438,6 +4644,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4509,7 +4932,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4540,11 +4963,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4674,7 +5097,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/cli-consistency-checker.lock.yml b/.github/workflows/cli-consistency-checker.lock.yml index 25e120a2de5..e110764cdfc 100644 --- a/.github/workflows/cli-consistency-checker.lock.yml +++ b/.github/workflows/cli-consistency-checker.lock.yml @@ -2940,6 +2940,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2977,6 +2980,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2986,6 +3169,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3010,13 +3196,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3033,61 +3219,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3097,13 +3290,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3151,10 +3346,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3370,7 +3565,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3434,12 +3629,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3467,7 +3668,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3531,7 +3737,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3547,7 +3753,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3571,6 +3777,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3642,7 +4065,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3673,11 +4096,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3807,7 +4230,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/cli-version-checker.lock.yml b/.github/workflows/cli-version-checker.lock.yml index 7c100ab0794..1d37709d4bd 100644 --- a/.github/workflows/cli-version-checker.lock.yml +++ b/.github/workflows/cli-version-checker.lock.yml @@ -3474,6 +3474,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3511,6 +3514,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3520,6 +3703,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3544,13 +3730,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3567,61 +3753,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3631,13 +3824,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3685,10 +3880,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3904,7 +4099,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3968,12 +4163,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4001,7 +4202,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4065,7 +4271,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4081,7 +4287,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4105,6 +4311,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4176,7 +4599,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4207,11 +4630,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4341,7 +4764,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/cloclo.lock.yml b/.github/workflows/cloclo.lock.yml index fd6cc528056..99e70e7438a 100644 --- a/.github/workflows/cloclo.lock.yml +++ b/.github/workflows/cloclo.lock.yml @@ -524,6 +524,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -563,15 +566,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -596,13 +591,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -619,61 +614,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -683,14 +684,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -747,6 +746,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -1048,16 +1050,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -5278,6 +5271,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -5315,6 +5311,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -5324,6 +5500,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -5348,13 +5527,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -5371,61 +5550,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5435,13 +5621,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5489,10 +5677,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5708,7 +5896,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5772,12 +5960,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5805,7 +5999,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5869,7 +6068,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5885,7 +6084,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5909,6 +6108,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5980,7 +6396,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -6011,11 +6427,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -6145,7 +6561,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/close-old-discussions.lock.yml b/.github/workflows/close-old-discussions.lock.yml index 1ac5bf57413..d56da5319e4 100644 --- a/.github/workflows/close-old-discussions.lock.yml +++ b/.github/workflows/close-old-discussions.lock.yml @@ -3042,6 +3042,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3079,6 +3082,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3088,6 +3271,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3112,13 +3298,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3135,61 +3321,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3199,13 +3392,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3253,10 +3448,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3472,7 +3667,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3536,12 +3731,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3569,7 +3770,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3633,7 +3839,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3649,7 +3855,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3673,6 +3879,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3744,7 +4167,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3775,11 +4198,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3909,7 +4332,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/commit-changes-analyzer.lock.yml b/.github/workflows/commit-changes-analyzer.lock.yml index 93cc64c3048..d70a13f71ea 100644 --- a/.github/workflows/commit-changes-analyzer.lock.yml +++ b/.github/workflows/commit-changes-analyzer.lock.yml @@ -3369,6 +3369,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3406,6 +3409,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3415,6 +3598,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3439,13 +3625,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3462,61 +3648,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3526,13 +3719,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3580,10 +3775,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3799,7 +3994,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3863,12 +4058,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3896,7 +4097,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3960,7 +4166,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3976,7 +4182,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4000,6 +4206,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4071,7 +4494,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4102,11 +4525,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4236,7 +4659,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/copilot-agent-analysis.lock.yml b/.github/workflows/copilot-agent-analysis.lock.yml index 73c923c68fe..43c357ce349 100644 --- a/.github/workflows/copilot-agent-analysis.lock.yml +++ b/.github/workflows/copilot-agent-analysis.lock.yml @@ -4113,6 +4113,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4150,6 +4153,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4159,6 +4342,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4183,13 +4369,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4206,61 +4392,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4270,13 +4463,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4324,10 +4519,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4543,7 +4738,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4607,12 +4802,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4640,7 +4841,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4704,7 +4910,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4720,7 +4926,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4744,6 +4950,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4815,7 +5238,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4846,11 +5269,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4980,7 +5403,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/copilot-pr-merged-report.lock.yml b/.github/workflows/copilot-pr-merged-report.lock.yml index 0edbe29222b..0eb5b106b07 100644 --- a/.github/workflows/copilot-pr-merged-report.lock.yml +++ b/.github/workflows/copilot-pr-merged-report.lock.yml @@ -4385,6 +4385,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4422,6 +4425,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4431,6 +4614,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4455,13 +4641,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4478,61 +4664,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4542,13 +4735,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4596,10 +4791,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4815,7 +5010,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4879,12 +5074,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4912,7 +5113,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4976,7 +5182,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4992,7 +5198,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5016,6 +5222,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5087,7 +5510,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5118,11 +5541,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5252,7 +5675,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/copilot-pr-nlp-analysis.lock.yml b/.github/workflows/copilot-pr-nlp-analysis.lock.yml index f40bf94c066..b4afeadc285 100644 --- a/.github/workflows/copilot-pr-nlp-analysis.lock.yml +++ b/.github/workflows/copilot-pr-nlp-analysis.lock.yml @@ -4481,6 +4481,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4518,6 +4521,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4527,6 +4710,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4551,13 +4737,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4574,61 +4760,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4638,13 +4831,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4692,10 +4887,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4911,7 +5106,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4975,12 +5170,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5008,7 +5209,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5072,7 +5278,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5088,7 +5294,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5112,6 +5318,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5183,7 +5606,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5214,11 +5637,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5348,7 +5771,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/copilot-pr-prompt-analysis.lock.yml b/.github/workflows/copilot-pr-prompt-analysis.lock.yml index 5682c9b53bb..70324e83018 100644 --- a/.github/workflows/copilot-pr-prompt-analysis.lock.yml +++ b/.github/workflows/copilot-pr-prompt-analysis.lock.yml @@ -3504,6 +3504,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3541,6 +3544,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3550,6 +3733,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3574,13 +3760,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3597,61 +3783,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3661,13 +3854,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3715,10 +3910,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3934,7 +4129,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3998,12 +4193,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4031,7 +4232,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4095,7 +4301,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4111,7 +4317,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4135,6 +4341,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4206,7 +4629,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4237,11 +4660,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4371,7 +4794,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/copilot-session-insights.lock.yml b/.github/workflows/copilot-session-insights.lock.yml index c1431748d71..021d81b4371 100644 --- a/.github/workflows/copilot-session-insights.lock.yml +++ b/.github/workflows/copilot-session-insights.lock.yml @@ -5523,6 +5523,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -5560,6 +5563,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -5569,6 +5752,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -5593,13 +5779,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -5616,61 +5802,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5680,13 +5873,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5734,10 +5929,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5953,7 +6148,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -6017,12 +6212,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -6050,7 +6251,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -6114,7 +6320,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -6130,7 +6336,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -6154,6 +6360,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -6225,7 +6648,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -6256,11 +6679,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -6390,7 +6813,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/craft.lock.yml b/.github/workflows/craft.lock.yml index 3a10de3e434..502a7eedca5 100644 --- a/.github/workflows/craft.lock.yml +++ b/.github/workflows/craft.lock.yml @@ -474,6 +474,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -513,15 +516,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -546,13 +541,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -569,61 +564,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -633,14 +634,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -697,6 +696,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -998,16 +1000,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -4869,6 +4862,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4906,6 +4902,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4915,6 +5091,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4939,13 +5118,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4962,61 +5141,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5026,13 +5212,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5080,10 +5268,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5299,7 +5487,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5363,12 +5551,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5396,7 +5590,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5460,7 +5659,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5476,7 +5675,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5500,6 +5699,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5571,7 +5987,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5602,11 +6018,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5736,7 +6152,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-assign-issue-to-user.lock.yml b/.github/workflows/daily-assign-issue-to-user.lock.yml index 662921ccc89..ffcb7ab8781 100644 --- a/.github/workflows/daily-assign-issue-to-user.lock.yml +++ b/.github/workflows/daily-assign-issue-to-user.lock.yml @@ -3312,6 +3312,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3349,6 +3352,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3358,6 +3541,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3382,13 +3568,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3405,61 +3591,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3469,13 +3662,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3523,10 +3718,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3742,7 +3937,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3806,12 +4001,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3839,7 +4040,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3903,7 +4109,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3919,7 +4125,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3943,6 +4149,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4014,7 +4437,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4045,11 +4468,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4179,7 +4602,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-code-metrics.lock.yml b/.github/workflows/daily-code-metrics.lock.yml index 320b7133f6b..5f456af90b6 100644 --- a/.github/workflows/daily-code-metrics.lock.yml +++ b/.github/workflows/daily-code-metrics.lock.yml @@ -4569,6 +4569,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4606,6 +4609,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4615,6 +4798,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4639,13 +4825,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4662,61 +4848,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4726,13 +4919,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4780,10 +4975,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4999,7 +5194,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5063,12 +5258,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5096,7 +5297,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5160,7 +5366,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5176,7 +5382,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5200,6 +5406,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5271,7 +5694,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5302,11 +5725,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5436,7 +5859,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-copilot-token-report.lock.yml b/.github/workflows/daily-copilot-token-report.lock.yml index 16a45a4049b..549b3dee698 100644 --- a/.github/workflows/daily-copilot-token-report.lock.yml +++ b/.github/workflows/daily-copilot-token-report.lock.yml @@ -4649,6 +4649,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4686,6 +4689,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4695,6 +4878,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4719,13 +4905,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4742,61 +4928,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4806,13 +4999,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4860,10 +5055,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5079,7 +5274,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5143,12 +5338,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5176,7 +5377,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5240,7 +5446,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5256,7 +5462,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5280,6 +5486,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5351,7 +5774,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5382,11 +5805,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5516,7 +5939,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-doc-updater.lock.yml b/.github/workflows/daily-doc-updater.lock.yml index ece8ff59ada..edd09d54cd2 100644 --- a/.github/workflows/daily-doc-updater.lock.yml +++ b/.github/workflows/daily-doc-updater.lock.yml @@ -3165,6 +3165,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3202,6 +3205,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3211,6 +3394,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3235,13 +3421,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3258,61 +3444,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3322,13 +3515,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3376,10 +3571,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3595,7 +3790,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3659,12 +3854,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3692,7 +3893,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3756,7 +3962,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3772,7 +3978,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3796,6 +4002,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3867,7 +4290,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3898,11 +4321,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4032,7 +4455,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-fact.lock.yml b/.github/workflows/daily-fact.lock.yml index 1945a5d721e..6424c8e6653 100644 --- a/.github/workflows/daily-fact.lock.yml +++ b/.github/workflows/daily-fact.lock.yml @@ -3410,6 +3410,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3447,6 +3450,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3456,6 +3639,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3480,13 +3666,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3503,61 +3689,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3567,13 +3760,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3621,10 +3816,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3840,7 +4035,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3904,12 +4099,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3937,7 +4138,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4001,7 +4207,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4017,7 +4223,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4041,6 +4247,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4112,7 +4535,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4143,11 +4566,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4277,7 +4700,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-file-diet.lock.yml b/.github/workflows/daily-file-diet.lock.yml index 5c4285c02bd..96e33057430 100644 --- a/.github/workflows/daily-file-diet.lock.yml +++ b/.github/workflows/daily-file-diet.lock.yml @@ -4649,6 +4649,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4686,6 +4689,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4695,6 +4878,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4719,13 +4905,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4742,61 +4928,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4806,13 +4999,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4860,10 +5055,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5079,7 +5274,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5143,12 +5338,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5176,7 +5377,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5240,7 +5446,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5256,7 +5462,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5280,6 +5486,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5351,7 +5774,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5382,11 +5805,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5516,7 +5939,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-firewall-report.lock.yml b/.github/workflows/daily-firewall-report.lock.yml index 6b903374658..16ae8c30547 100644 --- a/.github/workflows/daily-firewall-report.lock.yml +++ b/.github/workflows/daily-firewall-report.lock.yml @@ -3934,6 +3934,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3971,6 +3974,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3980,6 +4163,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4004,13 +4190,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4027,61 +4213,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4091,13 +4284,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4145,10 +4340,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4364,7 +4559,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4428,12 +4623,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4461,7 +4662,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4525,7 +4731,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4541,7 +4747,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4565,6 +4771,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4636,7 +5059,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4667,11 +5090,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4801,7 +5224,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-issues-report.lock.yml b/.github/workflows/daily-issues-report.lock.yml index 8660d827142..bc1f1e4c020 100644 --- a/.github/workflows/daily-issues-report.lock.yml +++ b/.github/workflows/daily-issues-report.lock.yml @@ -4779,6 +4779,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4816,6 +4819,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4825,6 +5008,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4849,13 +5035,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4872,61 +5058,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4936,13 +5129,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4990,10 +5185,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5209,7 +5404,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5273,12 +5468,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5306,7 +5507,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5370,7 +5576,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5386,7 +5592,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5410,6 +5616,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5481,7 +5904,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5512,11 +5935,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5646,7 +6069,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-malicious-code-scan.lock.yml b/.github/workflows/daily-malicious-code-scan.lock.yml index ef2f46d3b67..d8506e16614 100644 --- a/.github/workflows/daily-malicious-code-scan.lock.yml +++ b/.github/workflows/daily-malicious-code-scan.lock.yml @@ -3179,6 +3179,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3216,6 +3219,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3225,6 +3408,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3249,13 +3435,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3272,61 +3458,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3336,13 +3529,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3390,10 +3585,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3609,7 +3804,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3673,12 +3868,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3706,7 +3907,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3770,7 +3976,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3786,7 +3992,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3810,6 +4016,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3881,7 +4304,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3912,11 +4335,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4046,7 +4469,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-multi-device-docs-tester.lock.yml b/.github/workflows/daily-multi-device-docs-tester.lock.yml index 7f9c8394a86..c4ff910cb1b 100644 --- a/.github/workflows/daily-multi-device-docs-tester.lock.yml +++ b/.github/workflows/daily-multi-device-docs-tester.lock.yml @@ -3075,6 +3075,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3112,6 +3115,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3121,6 +3304,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3145,13 +3331,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3168,61 +3354,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3232,13 +3425,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3286,10 +3481,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3505,7 +3700,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3569,12 +3764,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3602,7 +3803,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3666,7 +3872,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3682,7 +3888,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3706,6 +3912,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3777,7 +4200,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3808,11 +4231,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3942,7 +4365,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-news.lock.yml b/.github/workflows/daily-news.lock.yml index db44d4d3612..63436eb2570 100644 --- a/.github/workflows/daily-news.lock.yml +++ b/.github/workflows/daily-news.lock.yml @@ -4408,6 +4408,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4445,6 +4448,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4454,6 +4637,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4478,13 +4664,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4501,61 +4687,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4565,13 +4758,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4619,10 +4814,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4838,7 +5033,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4902,12 +5097,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4935,7 +5136,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4999,7 +5205,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5015,7 +5221,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5039,6 +5245,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5110,7 +5533,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5141,11 +5564,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5275,7 +5698,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-performance-summary.lock.yml b/.github/workflows/daily-performance-summary.lock.yml index 1da3392de44..ce674a6a29f 100644 --- a/.github/workflows/daily-performance-summary.lock.yml +++ b/.github/workflows/daily-performance-summary.lock.yml @@ -6012,6 +6012,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -6049,6 +6052,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -6058,6 +6241,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -6082,13 +6268,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -6105,61 +6291,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -6169,13 +6362,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -6223,10 +6418,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -6442,7 +6637,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -6506,12 +6701,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -6539,7 +6740,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -6603,7 +6809,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -6619,7 +6825,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -6643,6 +6849,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -6714,7 +7137,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -6745,11 +7168,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -6879,7 +7302,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-repo-chronicle.lock.yml b/.github/workflows/daily-repo-chronicle.lock.yml index c8c0a338cac..cf56c52f508 100644 --- a/.github/workflows/daily-repo-chronicle.lock.yml +++ b/.github/workflows/daily-repo-chronicle.lock.yml @@ -4083,6 +4083,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4120,6 +4123,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4129,6 +4312,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4153,13 +4339,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4176,61 +4362,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4240,13 +4433,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4294,10 +4489,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4513,7 +4708,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4577,12 +4772,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4610,7 +4811,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4674,7 +4880,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4690,7 +4896,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4714,6 +4920,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4785,7 +5208,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4816,11 +5239,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4950,7 +5373,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-team-status.lock.yml b/.github/workflows/daily-team-status.lock.yml index 12dbcbcaae5..9c387a9a988 100644 --- a/.github/workflows/daily-team-status.lock.yml +++ b/.github/workflows/daily-team-status.lock.yml @@ -2707,6 +2707,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2744,6 +2747,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2753,6 +2936,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -2777,13 +2963,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -2800,61 +2986,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -2864,13 +3057,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -2918,10 +3113,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3137,7 +3332,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3201,12 +3396,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3234,7 +3435,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3298,7 +3504,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3314,7 +3520,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3338,6 +3544,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3409,7 +3832,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3440,11 +3863,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3574,7 +3997,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/daily-workflow-updater.lock.yml b/.github/workflows/daily-workflow-updater.lock.yml index 27060830047..cdae398bbee 100644 --- a/.github/workflows/daily-workflow-updater.lock.yml +++ b/.github/workflows/daily-workflow-updater.lock.yml @@ -2871,6 +2871,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2908,6 +2911,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2917,6 +3100,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -2941,13 +3127,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -2964,61 +3150,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3028,13 +3221,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3082,10 +3277,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3301,7 +3496,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3365,12 +3560,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3398,7 +3599,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3462,7 +3668,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3478,7 +3684,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3502,6 +3708,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3573,7 +3996,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3604,11 +4027,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3738,7 +4161,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/deep-report.lock.yml b/.github/workflows/deep-report.lock.yml index 3a12513972d..6230d50f159 100644 --- a/.github/workflows/deep-report.lock.yml +++ b/.github/workflows/deep-report.lock.yml @@ -3653,6 +3653,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3690,6 +3693,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3699,6 +3882,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3723,13 +3909,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3746,61 +3932,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3810,13 +4003,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3864,10 +4059,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4083,7 +4278,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4147,12 +4342,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4180,7 +4381,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4244,7 +4450,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4260,7 +4466,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4284,6 +4490,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4355,7 +4778,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4386,11 +4809,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4520,7 +4943,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/dependabot-go-checker.lock.yml b/.github/workflows/dependabot-go-checker.lock.yml index a30d2d44a53..b6261756812 100644 --- a/.github/workflows/dependabot-go-checker.lock.yml +++ b/.github/workflows/dependabot-go-checker.lock.yml @@ -3472,6 +3472,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3509,6 +3512,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3518,6 +3701,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3542,13 +3728,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3565,61 +3751,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3629,13 +3822,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3683,10 +3878,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3902,7 +4097,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3966,12 +4161,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3999,7 +4200,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4063,7 +4269,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4079,7 +4285,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4103,6 +4309,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4174,7 +4597,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4205,11 +4628,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4339,7 +4762,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/dev-hawk.lock.yml b/.github/workflows/dev-hawk.lock.yml index 4026484c9a0..c5c06fb5760 100644 --- a/.github/workflows/dev-hawk.lock.yml +++ b/.github/workflows/dev-hawk.lock.yml @@ -3592,6 +3592,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3629,6 +3632,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3638,6 +3821,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3662,13 +3848,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3685,61 +3871,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3749,13 +3942,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3803,10 +3998,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4022,7 +4217,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4086,12 +4281,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4119,7 +4320,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4183,7 +4389,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4199,7 +4405,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4223,6 +4429,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4294,7 +4717,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4325,11 +4748,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4459,7 +4882,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/dev.lock.yml b/.github/workflows/dev.lock.yml index 23ea4fc0c12..ea3da0a5496 100644 --- a/.github/workflows/dev.lock.yml +++ b/.github/workflows/dev.lock.yml @@ -3572,6 +3572,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3609,6 +3612,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3618,6 +3801,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3642,13 +3828,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3665,61 +3851,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3729,13 +3922,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3783,10 +3978,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4002,7 +4197,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4066,12 +4261,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4099,7 +4300,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4163,7 +4369,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4179,7 +4385,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4203,6 +4409,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4274,7 +4697,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4305,11 +4728,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4439,7 +4862,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/developer-docs-consolidator.lock.yml b/.github/workflows/developer-docs-consolidator.lock.yml index b62c0dea210..ba14dc7679d 100644 --- a/.github/workflows/developer-docs-consolidator.lock.yml +++ b/.github/workflows/developer-docs-consolidator.lock.yml @@ -4315,6 +4315,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4352,6 +4355,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4361,6 +4544,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4385,13 +4571,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4408,61 +4594,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4472,13 +4665,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4526,10 +4721,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4745,7 +4940,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4809,12 +5004,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4842,7 +5043,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4906,7 +5112,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4922,7 +5128,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4946,6 +5152,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5017,7 +5440,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5048,11 +5471,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5182,7 +5605,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/dictation-prompt.lock.yml b/.github/workflows/dictation-prompt.lock.yml index 9f60d215316..939b1893a39 100644 --- a/.github/workflows/dictation-prompt.lock.yml +++ b/.github/workflows/dictation-prompt.lock.yml @@ -2818,6 +2818,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2855,6 +2858,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2864,6 +3047,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -2888,13 +3074,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -2911,61 +3097,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -2975,13 +3168,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3029,10 +3224,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3248,7 +3443,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3312,12 +3507,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3345,7 +3546,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3409,7 +3615,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3425,7 +3631,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3449,6 +3655,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3520,7 +3943,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3551,11 +3974,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3685,7 +4108,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/docs-noob-tester.lock.yml b/.github/workflows/docs-noob-tester.lock.yml index 8eee53cce99..f85d4476133 100644 --- a/.github/workflows/docs-noob-tester.lock.yml +++ b/.github/workflows/docs-noob-tester.lock.yml @@ -2950,6 +2950,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2987,6 +2990,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2996,6 +3179,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3020,13 +3206,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3043,61 +3229,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3107,13 +3300,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3161,10 +3356,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3380,7 +3575,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3444,12 +3639,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3477,7 +3678,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3541,7 +3747,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3557,7 +3763,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3581,6 +3787,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3652,7 +4075,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3683,11 +4106,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3817,7 +4240,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/duplicate-code-detector.lock.yml b/.github/workflows/duplicate-code-detector.lock.yml index 3dfef7253db..e2704ca2da8 100644 --- a/.github/workflows/duplicate-code-detector.lock.yml +++ b/.github/workflows/duplicate-code-detector.lock.yml @@ -3029,6 +3029,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3066,6 +3069,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3075,6 +3258,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3099,13 +3285,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3122,61 +3308,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3186,13 +3379,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3240,10 +3435,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3459,7 +3654,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3523,12 +3718,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3556,7 +3757,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3620,7 +3826,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3636,7 +3842,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3660,6 +3866,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3731,7 +4154,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3762,11 +4185,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3896,7 +4319,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/example-workflow-analyzer.lock.yml b/.github/workflows/example-workflow-analyzer.lock.yml index 694ce9f66b5..14e97663e72 100644 --- a/.github/workflows/example-workflow-analyzer.lock.yml +++ b/.github/workflows/example-workflow-analyzer.lock.yml @@ -2882,6 +2882,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2919,6 +2922,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2928,6 +3111,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -2952,13 +3138,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -2975,61 +3161,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3039,13 +3232,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3093,10 +3288,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3312,7 +3507,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3376,12 +3571,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3409,7 +3610,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3473,7 +3679,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3489,7 +3695,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3513,6 +3719,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3584,7 +4007,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3615,11 +4038,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3749,7 +4172,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/github-mcp-structural-analysis.lock.yml b/.github/workflows/github-mcp-structural-analysis.lock.yml index 03d65ccaac9..2919f0d61aa 100644 --- a/.github/workflows/github-mcp-structural-analysis.lock.yml +++ b/.github/workflows/github-mcp-structural-analysis.lock.yml @@ -4238,6 +4238,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4275,6 +4278,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4284,6 +4467,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4308,13 +4494,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4331,61 +4517,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4395,13 +4588,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4449,10 +4644,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4668,7 +4863,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4732,12 +4927,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4765,7 +4966,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4829,7 +5035,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4845,7 +5051,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4869,6 +5075,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4940,7 +5363,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4971,11 +5394,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5105,7 +5528,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/github-mcp-tools-report.lock.yml b/.github/workflows/github-mcp-tools-report.lock.yml index 95f5ccbea79..d30a490e433 100644 --- a/.github/workflows/github-mcp-tools-report.lock.yml +++ b/.github/workflows/github-mcp-tools-report.lock.yml @@ -4015,6 +4015,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4052,6 +4055,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4061,6 +4244,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4085,13 +4271,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4108,61 +4294,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4172,13 +4365,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4226,10 +4421,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4445,7 +4640,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4509,12 +4704,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4542,7 +4743,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4606,7 +4812,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4622,7 +4828,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4646,6 +4852,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4717,7 +5140,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4748,11 +5171,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4882,7 +5305,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/glossary-maintainer.lock.yml b/.github/workflows/glossary-maintainer.lock.yml index d73abbf39f3..f676de6c0e9 100644 --- a/.github/workflows/glossary-maintainer.lock.yml +++ b/.github/workflows/glossary-maintainer.lock.yml @@ -3971,6 +3971,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4008,6 +4011,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4017,6 +4200,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4041,13 +4227,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4064,61 +4250,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4128,13 +4321,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4182,10 +4377,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4401,7 +4596,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4465,12 +4660,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4498,7 +4699,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4562,7 +4768,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4578,7 +4784,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4602,6 +4808,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4673,7 +5096,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4704,11 +5127,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4838,7 +5261,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/go-fan.lock.yml b/.github/workflows/go-fan.lock.yml index cc6d6524ff7..39859967bdb 100644 --- a/.github/workflows/go-fan.lock.yml +++ b/.github/workflows/go-fan.lock.yml @@ -3590,6 +3590,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3627,6 +3630,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3636,6 +3819,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3660,13 +3846,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3683,61 +3869,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3747,13 +3940,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3801,10 +3996,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4020,7 +4215,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4084,12 +4279,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4117,7 +4318,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4181,7 +4387,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4197,7 +4403,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4221,6 +4427,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4292,7 +4715,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4323,11 +4746,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4457,7 +4880,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/go-file-size-reduction.campaign.g.lock.yml b/.github/workflows/go-file-size-reduction.campaign.g.lock.yml index 4780682badd..35afdbd2b86 100644 --- a/.github/workflows/go-file-size-reduction.campaign.g.lock.yml +++ b/.github/workflows/go-file-size-reduction.campaign.g.lock.yml @@ -3340,6 +3340,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3377,6 +3380,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3386,6 +3569,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3410,13 +3596,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3433,61 +3619,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3497,13 +3690,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3551,10 +3746,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3770,7 +3965,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3834,12 +4029,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3867,7 +4068,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3931,7 +4137,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3947,7 +4153,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3971,6 +4177,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4042,7 +4465,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4073,11 +4496,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4207,7 +4630,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/go-logger.lock.yml b/.github/workflows/go-logger.lock.yml index c185a2f4162..7a11f325215 100644 --- a/.github/workflows/go-logger.lock.yml +++ b/.github/workflows/go-logger.lock.yml @@ -3324,6 +3324,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3361,6 +3364,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3370,6 +3553,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3394,13 +3580,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3417,61 +3603,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3481,13 +3674,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3535,10 +3730,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3754,7 +3949,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3818,12 +4013,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3851,7 +4052,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3915,7 +4121,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3931,7 +4137,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3955,6 +4161,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4026,7 +4449,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4057,11 +4480,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4191,7 +4614,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/go-pattern-detector.lock.yml b/.github/workflows/go-pattern-detector.lock.yml index 72f4ae85e41..7c16de36f32 100644 --- a/.github/workflows/go-pattern-detector.lock.yml +++ b/.github/workflows/go-pattern-detector.lock.yml @@ -3075,6 +3075,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3112,6 +3115,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3121,6 +3304,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3145,13 +3331,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3168,61 +3354,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3232,13 +3425,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3286,10 +3481,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3505,7 +3700,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3569,12 +3764,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3602,7 +3803,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3666,7 +3872,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3682,7 +3888,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3706,6 +3912,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3777,7 +4200,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3808,11 +4231,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3942,7 +4365,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/grumpy-reviewer.lock.yml b/.github/workflows/grumpy-reviewer.lock.yml index c423c39be48..824f8985af6 100644 --- a/.github/workflows/grumpy-reviewer.lock.yml +++ b/.github/workflows/grumpy-reviewer.lock.yml @@ -356,6 +356,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -395,15 +398,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -428,13 +423,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -451,61 +446,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -515,14 +516,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -579,6 +578,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -880,16 +882,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -4675,6 +4668,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4712,6 +4708,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4721,6 +4897,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4745,13 +4924,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4768,61 +4947,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4832,13 +5018,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4886,10 +5074,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5105,7 +5293,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5169,12 +5357,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5202,7 +5396,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5266,7 +5465,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5282,7 +5481,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5306,6 +5505,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5377,7 +5793,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5408,11 +5824,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5542,7 +5958,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/hourly-ci-cleaner.lock.yml b/.github/workflows/hourly-ci-cleaner.lock.yml index bdc83437e11..8f22f3c34c0 100644 --- a/.github/workflows/hourly-ci-cleaner.lock.yml +++ b/.github/workflows/hourly-ci-cleaner.lock.yml @@ -3326,6 +3326,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3363,6 +3366,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3372,6 +3555,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3396,13 +3582,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3419,61 +3605,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3483,13 +3676,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3537,10 +3732,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3756,7 +3951,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3820,12 +4015,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3853,7 +4054,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3917,7 +4123,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3933,7 +4139,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3957,6 +4163,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4028,7 +4451,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4059,11 +4482,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4193,7 +4616,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/human-ai-collaboration.lock.yml b/.github/workflows/human-ai-collaboration.lock.yml index 4ade6a0dd2b..c9cc2a794c8 100644 --- a/.github/workflows/human-ai-collaboration.lock.yml +++ b/.github/workflows/human-ai-collaboration.lock.yml @@ -3536,6 +3536,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3573,6 +3576,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3582,6 +3765,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3606,13 +3792,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3629,61 +3815,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3693,13 +3886,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3747,10 +3942,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3966,7 +4161,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4030,12 +4225,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4063,7 +4264,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4127,7 +4333,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4143,7 +4349,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4167,6 +4373,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4238,7 +4661,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4269,11 +4692,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4403,7 +4826,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/incident-response.lock.yml b/.github/workflows/incident-response.lock.yml index d4f81f7d5e5..14bba583d5f 100644 --- a/.github/workflows/incident-response.lock.yml +++ b/.github/workflows/incident-response.lock.yml @@ -4997,6 +4997,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -5034,6 +5037,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -5043,6 +5226,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -5067,13 +5253,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -5090,61 +5276,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5154,13 +5347,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5208,10 +5403,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5427,7 +5622,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5491,12 +5686,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5524,7 +5725,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5588,7 +5794,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5604,7 +5810,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5628,6 +5834,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5699,7 +6122,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5730,11 +6153,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5864,7 +6287,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/instructions-janitor.lock.yml b/.github/workflows/instructions-janitor.lock.yml index 139b844e42c..8e814623ee5 100644 --- a/.github/workflows/instructions-janitor.lock.yml +++ b/.github/workflows/instructions-janitor.lock.yml @@ -3089,6 +3089,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3126,6 +3129,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3135,6 +3318,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3159,13 +3345,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3182,61 +3368,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3246,13 +3439,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3300,10 +3495,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3519,7 +3714,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3583,12 +3778,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3616,7 +3817,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3680,7 +3886,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3696,7 +3902,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3720,6 +3926,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3791,7 +4214,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3822,11 +4245,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3956,7 +4379,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/intelligence.lock.yml b/.github/workflows/intelligence.lock.yml index 8d3d03a0d65..a17e9a1ec75 100644 --- a/.github/workflows/intelligence.lock.yml +++ b/.github/workflows/intelligence.lock.yml @@ -4800,6 +4800,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4837,6 +4840,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4846,6 +5029,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4870,13 +5056,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4893,61 +5079,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4957,13 +5150,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5011,10 +5206,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5230,7 +5425,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5294,12 +5489,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5327,7 +5528,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5391,7 +5597,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5407,7 +5613,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5431,6 +5637,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5502,7 +5925,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5533,11 +5956,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5667,7 +6090,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/issue-arborist.lock.yml b/.github/workflows/issue-arborist.lock.yml index 2be2d622720..35ae7828b4e 100644 --- a/.github/workflows/issue-arborist.lock.yml +++ b/.github/workflows/issue-arborist.lock.yml @@ -3149,6 +3149,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3186,6 +3189,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3195,6 +3378,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3219,13 +3405,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3242,61 +3428,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3306,13 +3499,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3360,10 +3555,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3579,7 +3774,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3643,12 +3838,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3676,7 +3877,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3740,7 +3946,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3756,7 +3962,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3780,6 +3986,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3851,7 +4274,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3882,11 +4305,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4016,7 +4439,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/issue-classifier.lock.yml b/.github/workflows/issue-classifier.lock.yml index f7281e4eac7..4835e8a47c9 100644 --- a/.github/workflows/issue-classifier.lock.yml +++ b/.github/workflows/issue-classifier.lock.yml @@ -246,6 +246,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -285,15 +288,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -318,13 +313,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -341,61 +336,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -405,14 +406,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -469,6 +468,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -770,16 +772,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -4090,6 +4083,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4127,6 +4123,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4136,6 +4312,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4160,13 +4339,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4183,61 +4362,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4247,13 +4433,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4301,10 +4489,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4520,7 +4708,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4584,12 +4772,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4617,7 +4811,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4681,7 +4880,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4697,7 +4896,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4721,6 +4920,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4792,7 +5208,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4823,11 +5239,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4957,7 +5373,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/issue-monster.lock.yml b/.github/workflows/issue-monster.lock.yml index a28938fe650..97098f6d6bd 100644 --- a/.github/workflows/issue-monster.lock.yml +++ b/.github/workflows/issue-monster.lock.yml @@ -3751,6 +3751,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3788,6 +3791,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3797,6 +3980,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3821,13 +4007,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3844,61 +4030,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3908,13 +4101,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3962,10 +4157,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4181,7 +4376,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4245,12 +4440,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4278,7 +4479,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4342,7 +4548,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4358,7 +4564,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4382,6 +4588,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4453,7 +4876,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4484,11 +4907,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4618,7 +5041,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/issue-triage-agent.lock.yml b/.github/workflows/issue-triage-agent.lock.yml index 13d87b3f1df..1bc9391a7fa 100644 --- a/.github/workflows/issue-triage-agent.lock.yml +++ b/.github/workflows/issue-triage-agent.lock.yml @@ -3861,6 +3861,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3898,6 +3901,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3907,6 +4090,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3931,13 +4117,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3954,61 +4140,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4018,13 +4211,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4072,10 +4267,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4291,7 +4486,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4355,12 +4550,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4388,7 +4589,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4452,7 +4658,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4468,7 +4674,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4492,6 +4698,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4563,7 +4986,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4594,11 +5017,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4728,7 +5151,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/layout-spec-maintainer.lock.yml b/.github/workflows/layout-spec-maintainer.lock.yml index f60c6e0d4a9..a1bbf888c85 100644 --- a/.github/workflows/layout-spec-maintainer.lock.yml +++ b/.github/workflows/layout-spec-maintainer.lock.yml @@ -3108,6 +3108,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3145,6 +3148,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3154,6 +3337,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3178,13 +3364,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3201,61 +3387,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3265,13 +3458,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3319,10 +3514,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3538,7 +3733,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3602,12 +3797,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3635,7 +3836,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3699,7 +3905,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3715,7 +3921,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3739,6 +3945,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3810,7 +4233,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3841,11 +4264,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3975,7 +4398,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/lockfile-stats.lock.yml b/.github/workflows/lockfile-stats.lock.yml index 05884ade6b1..696516a566b 100644 --- a/.github/workflows/lockfile-stats.lock.yml +++ b/.github/workflows/lockfile-stats.lock.yml @@ -3602,6 +3602,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3639,6 +3642,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3648,6 +3831,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3672,13 +3858,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3695,61 +3881,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3759,13 +3952,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3813,10 +4008,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4032,7 +4227,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4096,12 +4291,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4129,7 +4330,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4193,7 +4399,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4209,7 +4415,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4233,6 +4439,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4304,7 +4727,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4335,11 +4758,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4469,7 +4892,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/mcp-inspector.lock.yml b/.github/workflows/mcp-inspector.lock.yml index ac5cd777215..ed513b48d9b 100644 --- a/.github/workflows/mcp-inspector.lock.yml +++ b/.github/workflows/mcp-inspector.lock.yml @@ -3487,6 +3487,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3524,6 +3527,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3533,6 +3716,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3557,13 +3743,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3580,61 +3766,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3644,13 +3837,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3698,10 +3893,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3917,7 +4112,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3981,12 +4176,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4014,7 +4215,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4078,7 +4284,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4094,7 +4300,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4118,6 +4324,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4189,7 +4612,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4220,11 +4643,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4354,7 +4777,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/mergefest.lock.yml b/.github/workflows/mergefest.lock.yml index 98c825a6c91..5646a605016 100644 --- a/.github/workflows/mergefest.lock.yml +++ b/.github/workflows/mergefest.lock.yml @@ -3661,6 +3661,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3698,6 +3701,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3707,6 +3890,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3731,13 +3917,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3754,61 +3940,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3818,13 +4011,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3872,10 +4067,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4091,7 +4286,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4155,12 +4350,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4188,7 +4389,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4252,7 +4458,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4268,7 +4474,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4292,6 +4498,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4363,7 +4786,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4394,11 +4817,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4528,7 +4951,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/notion-issue-summary.lock.yml b/.github/workflows/notion-issue-summary.lock.yml index ffc7a7530d8..4148d1b1b83 100644 --- a/.github/workflows/notion-issue-summary.lock.yml +++ b/.github/workflows/notion-issue-summary.lock.yml @@ -2549,6 +2549,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2586,6 +2589,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2595,6 +2778,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -2619,13 +2805,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -2642,61 +2828,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -2706,13 +2899,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -2760,10 +2955,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -2979,7 +3174,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3043,12 +3238,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3076,7 +3277,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3140,7 +3346,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3156,7 +3362,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3180,6 +3386,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3251,7 +3674,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3282,11 +3705,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3416,7 +3839,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/org-health-report.lock.yml b/.github/workflows/org-health-report.lock.yml index 8bd3022a42d..2120101cfc0 100644 --- a/.github/workflows/org-health-report.lock.yml +++ b/.github/workflows/org-health-report.lock.yml @@ -4342,6 +4342,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4379,6 +4382,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4388,6 +4571,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4412,13 +4598,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4435,61 +4621,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4499,13 +4692,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4553,10 +4748,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4772,7 +4967,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4836,12 +5031,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4869,7 +5070,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4933,7 +5139,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4949,7 +5155,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4973,6 +5179,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5044,7 +5467,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5075,11 +5498,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5209,7 +5632,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/org-wide-rollout.lock.yml b/.github/workflows/org-wide-rollout.lock.yml index 810c8f78e5b..5d2d73d91d4 100644 --- a/.github/workflows/org-wide-rollout.lock.yml +++ b/.github/workflows/org-wide-rollout.lock.yml @@ -5049,6 +5049,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -5086,6 +5089,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -5095,6 +5278,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -5119,13 +5305,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -5142,61 +5328,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5206,13 +5399,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5260,10 +5455,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5479,7 +5674,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5543,12 +5738,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5576,7 +5777,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5640,7 +5846,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5656,7 +5862,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5680,6 +5886,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5751,7 +6174,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5782,11 +6205,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5916,7 +6339,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/pdf-summary.lock.yml b/.github/workflows/pdf-summary.lock.yml index 8c7c2a2555a..92a51c3a9d0 100644 --- a/.github/workflows/pdf-summary.lock.yml +++ b/.github/workflows/pdf-summary.lock.yml @@ -407,6 +407,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -446,15 +449,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -479,13 +474,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -502,61 +497,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -566,14 +567,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -630,6 +629,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -931,16 +933,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -4699,6 +4692,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4736,6 +4732,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4745,6 +4921,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4769,13 +4948,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4792,61 +4971,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4856,13 +5042,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4910,10 +5098,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5129,7 +5317,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5193,12 +5381,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5226,7 +5420,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5290,7 +5489,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5306,7 +5505,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5330,6 +5529,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5401,7 +5817,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5432,11 +5848,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5566,7 +5982,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/plan.lock.yml b/.github/workflows/plan.lock.yml index 57f5f364c76..e842c449f1d 100644 --- a/.github/workflows/plan.lock.yml +++ b/.github/workflows/plan.lock.yml @@ -394,6 +394,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -433,15 +436,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -466,13 +461,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -489,61 +484,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -553,14 +554,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -617,6 +616,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -918,16 +920,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -3985,6 +3978,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4022,6 +4018,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4031,6 +4207,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4055,13 +4234,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4078,61 +4257,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4142,13 +4328,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4196,10 +4384,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4415,7 +4603,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4479,12 +4667,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4512,7 +4706,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4576,7 +4775,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4592,7 +4791,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4616,6 +4815,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4687,7 +5103,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4718,11 +5134,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4852,7 +5268,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/poem-bot.lock.yml b/.github/workflows/poem-bot.lock.yml index 066fef7e661..26a01196c9d 100644 --- a/.github/workflows/poem-bot.lock.yml +++ b/.github/workflows/poem-bot.lock.yml @@ -435,6 +435,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -474,15 +477,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -507,13 +502,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -530,61 +525,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -594,14 +595,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -658,6 +657,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -959,16 +961,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -5751,6 +5744,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -5788,6 +5784,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -5797,6 +5973,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -5821,13 +6000,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -5844,61 +6023,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5908,13 +6094,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5962,10 +6150,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -6181,7 +6369,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -6245,12 +6433,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -6278,7 +6472,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -6342,7 +6541,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -6358,7 +6557,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -6382,6 +6581,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -6453,7 +6869,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -6484,11 +6900,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -6618,7 +7034,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index a10fcaba814..3949bbb250e 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -4668,6 +4668,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4705,6 +4708,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4714,6 +4897,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4738,13 +4924,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4761,61 +4947,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4825,13 +5018,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4879,10 +5074,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5098,7 +5293,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5162,12 +5357,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5195,7 +5396,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5259,7 +5465,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5275,7 +5481,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5299,6 +5505,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5370,7 +5793,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5401,11 +5824,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5535,7 +5958,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/pr-nitpick-reviewer.lock.yml b/.github/workflows/pr-nitpick-reviewer.lock.yml index 8800314fd63..e57f68b94e2 100644 --- a/.github/workflows/pr-nitpick-reviewer.lock.yml +++ b/.github/workflows/pr-nitpick-reviewer.lock.yml @@ -4830,6 +4830,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4867,6 +4870,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4876,6 +5059,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4900,13 +5086,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4923,61 +5109,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4987,13 +5180,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5041,10 +5236,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5260,7 +5455,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5324,12 +5519,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5357,7 +5558,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5421,7 +5627,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5437,7 +5643,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5461,6 +5667,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5532,7 +5955,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5563,11 +5986,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5697,7 +6120,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/prompt-clustering-analysis.lock.yml b/.github/workflows/prompt-clustering-analysis.lock.yml index 665c6a3bb0d..b5deb357c00 100644 --- a/.github/workflows/prompt-clustering-analysis.lock.yml +++ b/.github/workflows/prompt-clustering-analysis.lock.yml @@ -4878,6 +4878,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4915,6 +4918,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4924,6 +5107,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4948,13 +5134,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4971,61 +5157,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5035,13 +5228,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5089,10 +5284,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5308,7 +5503,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5372,12 +5567,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5405,7 +5606,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5469,7 +5675,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5485,7 +5691,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5509,6 +5715,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5580,7 +6003,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5611,11 +6034,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5745,7 +6168,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/python-data-charts.lock.yml b/.github/workflows/python-data-charts.lock.yml index 40be48dc79f..59ff7dba92b 100644 --- a/.github/workflows/python-data-charts.lock.yml +++ b/.github/workflows/python-data-charts.lock.yml @@ -4716,6 +4716,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4753,6 +4756,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4762,6 +4945,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4786,13 +4972,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4809,61 +4995,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4873,13 +5066,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4927,10 +5122,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5146,7 +5341,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5210,12 +5405,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5243,7 +5444,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5307,7 +5513,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5323,7 +5529,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5347,6 +5553,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5418,7 +5841,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5449,11 +5872,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5583,7 +6006,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/q.lock.yml b/.github/workflows/q.lock.yml index a15c0a30a21..65cf791c427 100644 --- a/.github/workflows/q.lock.yml +++ b/.github/workflows/q.lock.yml @@ -644,6 +644,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -683,15 +686,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -716,13 +711,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -739,61 +734,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -803,14 +804,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -867,6 +866,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -1168,16 +1170,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -5282,6 +5275,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -5319,6 +5315,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -5328,6 +5504,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -5352,13 +5531,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -5375,61 +5554,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5439,13 +5625,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5493,10 +5681,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5712,7 +5900,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5776,12 +5964,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5809,7 +6003,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5873,7 +6072,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5889,7 +6088,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5913,6 +6112,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5984,7 +6400,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -6015,11 +6431,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -6149,7 +6565,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/release.lock.yml b/.github/workflows/release.lock.yml index 30c03941b15..fdb82c6ff0a 100644 --- a/.github/workflows/release.lock.yml +++ b/.github/workflows/release.lock.yml @@ -3020,6 +3020,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3057,6 +3060,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3066,6 +3249,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3090,13 +3276,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3113,61 +3299,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3177,13 +3370,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3231,10 +3426,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3450,7 +3645,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3514,12 +3709,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3547,7 +3748,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3611,7 +3817,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3627,7 +3833,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3651,6 +3857,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3722,7 +4145,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3753,11 +4176,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3887,7 +4310,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/repo-tree-map.lock.yml b/.github/workflows/repo-tree-map.lock.yml index b87b10abaf0..78c3a8d537b 100644 --- a/.github/workflows/repo-tree-map.lock.yml +++ b/.github/workflows/repo-tree-map.lock.yml @@ -2888,6 +2888,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2925,6 +2928,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2934,6 +3117,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -2958,13 +3144,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -2981,61 +3167,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3045,13 +3238,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3099,10 +3294,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3318,7 +3513,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3382,12 +3577,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3415,7 +3616,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3479,7 +3685,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3495,7 +3701,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3519,6 +3725,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3590,7 +4013,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3621,11 +4044,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3755,7 +4178,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/repository-quality-improver.lock.yml b/.github/workflows/repository-quality-improver.lock.yml index d8b10d2df8f..df66259e51a 100644 --- a/.github/workflows/repository-quality-improver.lock.yml +++ b/.github/workflows/repository-quality-improver.lock.yml @@ -3924,6 +3924,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3961,6 +3964,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3970,6 +4153,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3994,13 +4180,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4017,61 +4203,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4081,13 +4274,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4135,10 +4330,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4354,7 +4549,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4418,12 +4613,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4451,7 +4652,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4515,7 +4721,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4531,7 +4737,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4555,6 +4761,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4626,7 +5049,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4657,11 +5080,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4791,7 +5214,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/research.lock.yml b/.github/workflows/research.lock.yml index a5b5ee9d8f3..bfae32a39b2 100644 --- a/.github/workflows/research.lock.yml +++ b/.github/workflows/research.lock.yml @@ -2803,6 +2803,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2840,6 +2843,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2849,6 +3032,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -2873,13 +3059,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -2896,61 +3082,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -2960,13 +3153,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3014,10 +3209,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3233,7 +3428,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3297,12 +3492,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3330,7 +3531,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3394,7 +3600,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3410,7 +3616,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3434,6 +3640,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3505,7 +3928,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3536,11 +3959,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3670,7 +4093,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/safe-output-health.lock.yml b/.github/workflows/safe-output-health.lock.yml index d105c79692a..07258f999e5 100644 --- a/.github/workflows/safe-output-health.lock.yml +++ b/.github/workflows/safe-output-health.lock.yml @@ -3900,6 +3900,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3937,6 +3940,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3946,6 +4129,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3970,13 +4156,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3993,61 +4179,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4057,13 +4250,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4111,10 +4306,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4330,7 +4525,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4394,12 +4589,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4427,7 +4628,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4491,7 +4697,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4507,7 +4713,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4531,6 +4737,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4602,7 +5025,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4633,11 +5056,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4767,7 +5190,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/schema-consistency-checker.lock.yml b/.github/workflows/schema-consistency-checker.lock.yml index 5fb05d4756f..b0a8b4d0e98 100644 --- a/.github/workflows/schema-consistency-checker.lock.yml +++ b/.github/workflows/schema-consistency-checker.lock.yml @@ -3546,6 +3546,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3583,6 +3586,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3592,6 +3775,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3616,13 +3802,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3639,61 +3825,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3703,13 +3896,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3757,10 +3952,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3976,7 +4171,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4040,12 +4235,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4073,7 +4274,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4137,7 +4343,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4153,7 +4359,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4177,6 +4383,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4248,7 +4671,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4279,11 +4702,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4413,7 +4836,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/scout.lock.yml b/.github/workflows/scout.lock.yml index 569eabed123..7d4794cfe3c 100644 --- a/.github/workflows/scout.lock.yml +++ b/.github/workflows/scout.lock.yml @@ -608,6 +608,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -647,15 +650,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -680,13 +675,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -703,61 +698,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -767,14 +768,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -831,6 +830,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -1132,16 +1134,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -5324,6 +5317,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -5361,6 +5357,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -5370,6 +5546,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -5394,13 +5573,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -5417,61 +5596,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5481,13 +5667,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5535,10 +5723,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5754,7 +5942,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5818,12 +6006,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5851,7 +6045,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5915,7 +6114,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5931,7 +6130,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5955,6 +6154,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -6026,7 +6442,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -6057,11 +6473,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -6191,7 +6607,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/security-compliance.lock.yml b/.github/workflows/security-compliance.lock.yml index d620cf6e72b..1f839493b32 100644 --- a/.github/workflows/security-compliance.lock.yml +++ b/.github/workflows/security-compliance.lock.yml @@ -3173,6 +3173,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3210,6 +3213,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3219,6 +3402,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3243,13 +3429,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3266,61 +3452,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3330,13 +3523,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3384,10 +3579,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3603,7 +3798,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3667,12 +3862,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3700,7 +3901,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3764,7 +3970,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3780,7 +3986,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3804,6 +4010,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3875,7 +4298,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3906,11 +4329,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4040,7 +4463,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/security-fix-pr.lock.yml b/.github/workflows/security-fix-pr.lock.yml index ee7524236d5..702a42a416d 100644 --- a/.github/workflows/security-fix-pr.lock.yml +++ b/.github/workflows/security-fix-pr.lock.yml @@ -3097,6 +3097,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3134,6 +3137,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3143,6 +3326,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3167,13 +3353,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3190,61 +3376,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3254,13 +3447,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3308,10 +3503,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3527,7 +3722,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3591,12 +3786,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3624,7 +3825,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3688,7 +3894,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3704,7 +3910,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3728,6 +3934,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3799,7 +4222,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3830,11 +4253,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3964,7 +4387,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/semantic-function-refactor.lock.yml b/.github/workflows/semantic-function-refactor.lock.yml index 4dcc01853d8..9ac6f9e29c8 100644 --- a/.github/workflows/semantic-function-refactor.lock.yml +++ b/.github/workflows/semantic-function-refactor.lock.yml @@ -3935,6 +3935,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3972,6 +3975,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3981,6 +4164,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4005,13 +4191,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4028,61 +4214,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4092,13 +4285,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4146,10 +4341,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4365,7 +4560,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4429,12 +4624,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4462,7 +4663,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4526,7 +4732,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4542,7 +4748,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4566,6 +4772,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4637,7 +5060,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4668,11 +5091,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4802,7 +5225,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/smoke-claude.lock.yml b/.github/workflows/smoke-claude.lock.yml index 650bed25aa0..087f92b8811 100644 --- a/.github/workflows/smoke-claude.lock.yml +++ b/.github/workflows/smoke-claude.lock.yml @@ -5021,6 +5021,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -5058,6 +5061,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -5067,6 +5250,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -5091,13 +5277,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -5114,61 +5300,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5178,13 +5371,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5232,10 +5427,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5451,7 +5646,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5515,12 +5710,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5548,7 +5749,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5612,7 +5818,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5628,7 +5834,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5652,6 +5858,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5723,7 +6146,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5754,11 +6177,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5888,7 +6311,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/smoke-codex.lock.yml b/.github/workflows/smoke-codex.lock.yml index 5f4018d2869..8e0c1be0121 100644 --- a/.github/workflows/smoke-codex.lock.yml +++ b/.github/workflows/smoke-codex.lock.yml @@ -4602,6 +4602,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4639,6 +4642,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4648,6 +4831,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4672,13 +4858,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4695,61 +4881,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4759,13 +4952,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4813,10 +5008,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5032,7 +5227,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5096,12 +5291,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5129,7 +5330,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5193,7 +5399,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5209,7 +5415,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5233,6 +5439,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5304,7 +5727,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5335,11 +5758,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5469,7 +5892,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/smoke-copilot-no-firewall.lock.yml b/.github/workflows/smoke-copilot-no-firewall.lock.yml index 50bfd5199f3..f852fa9c2f2 100644 --- a/.github/workflows/smoke-copilot-no-firewall.lock.yml +++ b/.github/workflows/smoke-copilot-no-firewall.lock.yml @@ -6002,6 +6002,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -6039,6 +6042,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -6048,6 +6231,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -6072,13 +6258,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -6095,61 +6281,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -6159,13 +6352,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -6213,10 +6408,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -6432,7 +6627,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -6496,12 +6691,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -6529,7 +6730,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -6593,7 +6799,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -6609,7 +6815,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -6633,6 +6839,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -6704,7 +7127,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -6735,11 +7158,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -6869,7 +7292,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/smoke-copilot-playwright.lock.yml b/.github/workflows/smoke-copilot-playwright.lock.yml index a32c0723654..c89d9361cb2 100644 --- a/.github/workflows/smoke-copilot-playwright.lock.yml +++ b/.github/workflows/smoke-copilot-playwright.lock.yml @@ -5981,6 +5981,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -6018,6 +6021,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -6027,6 +6210,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -6051,13 +6237,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -6074,61 +6260,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -6138,13 +6331,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -6192,10 +6387,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -6411,7 +6606,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -6475,12 +6670,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -6508,7 +6709,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -6572,7 +6778,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -6588,7 +6794,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -6612,6 +6818,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -6683,7 +7106,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -6714,11 +7137,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -6848,7 +7271,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/smoke-copilot-safe-inputs.lock.yml b/.github/workflows/smoke-copilot-safe-inputs.lock.yml index 137c297c30b..fb04770dba2 100644 --- a/.github/workflows/smoke-copilot-safe-inputs.lock.yml +++ b/.github/workflows/smoke-copilot-safe-inputs.lock.yml @@ -5705,6 +5705,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -5742,6 +5745,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -5751,6 +5934,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -5775,13 +5961,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -5798,61 +5984,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5862,13 +6055,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5916,10 +6111,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -6135,7 +6330,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -6199,12 +6394,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -6232,7 +6433,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -6296,7 +6502,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -6312,7 +6518,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -6336,6 +6542,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -6407,7 +6830,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -6438,11 +6861,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -6572,7 +6995,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index 8600c236dc9..c209ee78215 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -4529,6 +4529,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4566,6 +4569,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4575,6 +4758,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4599,13 +4785,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4622,61 +4808,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4686,13 +4879,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4740,10 +4935,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4959,7 +5154,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5023,12 +5218,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5056,7 +5257,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5120,7 +5326,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5136,7 +5342,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5160,6 +5366,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5231,7 +5654,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5262,11 +5685,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5396,7 +5819,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/smoke-detector.lock.yml b/.github/workflows/smoke-detector.lock.yml index a4fafd7fe7a..3377b5f4837 100644 --- a/.github/workflows/smoke-detector.lock.yml +++ b/.github/workflows/smoke-detector.lock.yml @@ -4763,6 +4763,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4800,6 +4803,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4809,6 +4992,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4833,13 +5019,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4856,61 +5042,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4920,13 +5113,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4974,10 +5169,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5193,7 +5388,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5257,12 +5452,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5290,7 +5491,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5354,7 +5560,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5370,7 +5576,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5394,6 +5600,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5465,7 +5888,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5496,11 +5919,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5630,7 +6053,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/smoke-srt.lock.yml b/.github/workflows/smoke-srt.lock.yml index 54961b61a29..08a899fc8c8 100644 --- a/.github/workflows/smoke-srt.lock.yml +++ b/.github/workflows/smoke-srt.lock.yml @@ -2704,6 +2704,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2741,6 +2744,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2750,6 +2933,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -2774,13 +2960,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -2797,61 +2983,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -2861,13 +3054,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -2915,10 +3110,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3134,7 +3329,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3198,12 +3393,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3231,7 +3432,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3295,7 +3501,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3311,7 +3517,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3335,6 +3541,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3406,7 +3829,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3437,11 +3860,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3571,7 +3994,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/spec-kit-execute.lock.yml b/.github/workflows/spec-kit-execute.lock.yml index 4967f7ad694..30ff36bdda3 100644 --- a/.github/workflows/spec-kit-execute.lock.yml +++ b/.github/workflows/spec-kit-execute.lock.yml @@ -3416,6 +3416,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3453,6 +3456,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3462,6 +3645,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3486,13 +3672,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3509,61 +3695,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3573,13 +3766,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3627,10 +3822,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3846,7 +4041,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3910,12 +4105,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3943,7 +4144,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4007,7 +4213,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4023,7 +4229,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4047,6 +4253,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4118,7 +4541,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4149,11 +4572,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4283,7 +4706,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/spec-kit-executor.lock.yml b/.github/workflows/spec-kit-executor.lock.yml index ba49d9b293f..0c0867f594d 100644 --- a/.github/workflows/spec-kit-executor.lock.yml +++ b/.github/workflows/spec-kit-executor.lock.yml @@ -3106,6 +3106,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3143,6 +3146,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3152,6 +3335,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3176,13 +3362,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3199,61 +3385,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3263,13 +3456,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3317,10 +3512,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3536,7 +3731,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3600,12 +3795,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3633,7 +3834,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3697,7 +3903,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3713,7 +3919,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3737,6 +3943,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3808,7 +4231,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3839,11 +4262,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3973,7 +4396,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/speckit-dispatcher.lock.yml b/.github/workflows/speckit-dispatcher.lock.yml index 49d92c14ab3..7f2d3037dfa 100644 --- a/.github/workflows/speckit-dispatcher.lock.yml +++ b/.github/workflows/speckit-dispatcher.lock.yml @@ -622,6 +622,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -661,15 +664,7 @@ jobs: return []; } } - function sanitizeContent(content, maxLengthOrOptions) { - let maxLength; - let allowedAliasesLowercase = []; - if (typeof maxLengthOrOptions === "number") { - maxLength = maxLengthOrOptions; - } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { - maxLength = maxLengthOrOptions.maxLength; - allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); - } + function sanitizeContentCore(content, maxLength) { if (!content || typeof content !== "string") { return ""; } @@ -694,13 +689,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeAllMentions(sanitized); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -717,61 +712,67 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -781,14 +782,12 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { - return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); - if (isAllowed) { - return `${p1}@${p2}`; + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); } - core.info(`Escaped mention: @${p2} (not in allowed list)`); - return `${p1}\`@${p2}\``; + return `${p1}\`@${p2}\``; }); } function removeXmlComments(s) { @@ -845,6 +844,9 @@ jobs: return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } + function sanitizeIncomingText(content, maxLength) { + return sanitizeContentCore(content, maxLength); + } function extractMentions(text) { if (!text || typeof text !== "string") { return []; @@ -1146,16 +1148,7 @@ jobs: text = ""; break; } - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + const sanitizedText = sanitizeIncomingText(text); core.info(`text: ${sanitizedText}`); core.setOutput("text", sanitizedText); const logPath = writeRedactedDomainsLog(); @@ -5194,6 +5187,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -5231,6 +5227,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -5240,6 +5416,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -5264,13 +5443,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -5287,61 +5466,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5351,13 +5537,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5405,10 +5593,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5624,7 +5812,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5688,12 +5876,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5721,7 +5915,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5785,7 +5984,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5801,7 +6000,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5825,6 +6024,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5896,7 +6312,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5927,11 +6343,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -6061,7 +6477,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/stale-repo-identifier.lock.yml b/.github/workflows/stale-repo-identifier.lock.yml index 7bec9933369..191a62d675a 100644 --- a/.github/workflows/stale-repo-identifier.lock.yml +++ b/.github/workflows/stale-repo-identifier.lock.yml @@ -4580,6 +4580,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4617,6 +4620,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4626,6 +4809,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4650,13 +4836,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4673,61 +4859,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4737,13 +4930,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4791,10 +4986,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5010,7 +5205,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5074,12 +5269,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5107,7 +5308,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5171,7 +5377,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5187,7 +5393,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5211,6 +5417,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5282,7 +5705,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5313,11 +5736,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5447,7 +5870,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/static-analysis-report.lock.yml b/.github/workflows/static-analysis-report.lock.yml index 0f2cdb047e4..3b48da84ed3 100644 --- a/.github/workflows/static-analysis-report.lock.yml +++ b/.github/workflows/static-analysis-report.lock.yml @@ -3639,6 +3639,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3676,6 +3679,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3685,6 +3868,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3709,13 +3895,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3732,61 +3918,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3796,13 +3989,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3850,10 +4045,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4069,7 +4264,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4133,12 +4328,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4166,7 +4367,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4230,7 +4436,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4246,7 +4452,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4270,6 +4476,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4341,7 +4764,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4372,11 +4795,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4506,7 +4929,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/super-linter.lock.yml b/.github/workflows/super-linter.lock.yml index 8854d90ce98..c2775a3c4c4 100644 --- a/.github/workflows/super-linter.lock.yml +++ b/.github/workflows/super-linter.lock.yml @@ -3102,6 +3102,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3139,6 +3142,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3148,6 +3331,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3172,13 +3358,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3195,61 +3381,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3259,13 +3452,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3313,10 +3508,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3532,7 +3727,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3596,12 +3791,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3629,7 +3830,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3693,7 +3899,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3709,7 +3915,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3733,6 +3939,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3804,7 +4227,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3835,11 +4258,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3969,7 +4392,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/technical-doc-writer.lock.yml b/.github/workflows/technical-doc-writer.lock.yml index 6badd7437f6..e9a6fb5feb9 100644 --- a/.github/workflows/technical-doc-writer.lock.yml +++ b/.github/workflows/technical-doc-writer.lock.yml @@ -4330,6 +4330,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4367,6 +4370,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4376,6 +4559,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4400,13 +4586,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4423,61 +4609,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4487,13 +4680,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4541,10 +4736,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4760,7 +4955,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4824,12 +5019,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4857,7 +5058,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4921,7 +5127,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4937,7 +5143,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4961,6 +5167,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5032,7 +5455,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5063,11 +5486,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5197,7 +5620,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/test-discussion-expires.lock.yml b/.github/workflows/test-discussion-expires.lock.yml index 2f386f7c202..19eb04bf689 100644 --- a/.github/workflows/test-discussion-expires.lock.yml +++ b/.github/workflows/test-discussion-expires.lock.yml @@ -2486,6 +2486,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -2523,6 +2526,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -2532,6 +2715,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -2556,13 +2742,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -2579,61 +2765,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -2643,13 +2836,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -2697,10 +2892,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -2916,7 +3111,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -2980,12 +3175,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3013,7 +3214,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3077,7 +3283,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3093,7 +3299,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3117,6 +3323,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3188,7 +3611,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3219,11 +3642,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -3353,7 +3776,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/test-hide-older-comments.lock.yml b/.github/workflows/test-hide-older-comments.lock.yml index e1bb78fa1e2..f26103d88bc 100644 --- a/.github/workflows/test-hide-older-comments.lock.yml +++ b/.github/workflows/test-hide-older-comments.lock.yml @@ -3261,6 +3261,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3298,6 +3301,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3307,6 +3490,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3331,13 +3517,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3354,61 +3540,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3418,13 +3611,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3472,10 +3667,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3691,7 +3886,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3755,12 +3950,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3788,7 +3989,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3852,7 +4058,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3868,7 +4074,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3892,6 +4098,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3963,7 +4386,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3994,11 +4417,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4128,7 +4551,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/test-python-safe-input.lock.yml b/.github/workflows/test-python-safe-input.lock.yml index 04a6004d4cf..426ef066770 100644 --- a/.github/workflows/test-python-safe-input.lock.yml +++ b/.github/workflows/test-python-safe-input.lock.yml @@ -4100,6 +4100,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4137,6 +4140,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4146,6 +4329,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4170,13 +4356,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4193,61 +4379,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4257,13 +4450,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4311,10 +4506,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4530,7 +4725,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4594,12 +4789,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4627,7 +4828,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4691,7 +4897,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4707,7 +4913,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4731,6 +4937,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4802,7 +5225,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4833,11 +5256,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4967,7 +5390,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/tidy.lock.yml b/.github/workflows/tidy.lock.yml index ce2239e0e33..1bce8001c7d 100644 --- a/.github/workflows/tidy.lock.yml +++ b/.github/workflows/tidy.lock.yml @@ -3236,6 +3236,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3273,6 +3276,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3282,6 +3465,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3306,13 +3492,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3329,61 +3515,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3393,13 +3586,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3447,10 +3642,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3666,7 +3861,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3730,12 +3925,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3763,7 +3964,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3827,7 +4033,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3843,7 +4049,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3867,6 +4073,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3938,7 +4361,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3969,11 +4392,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4103,7 +4526,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/typist.lock.yml b/.github/workflows/typist.lock.yml index 27e7969ce46..caf448c2119 100644 --- a/.github/workflows/typist.lock.yml +++ b/.github/workflows/typist.lock.yml @@ -3965,6 +3965,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4002,6 +4005,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4011,6 +4194,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4035,13 +4221,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4058,61 +4244,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4122,13 +4315,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4176,10 +4371,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4395,7 +4590,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4459,12 +4654,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4492,7 +4693,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4556,7 +4762,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4572,7 +4778,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4596,6 +4802,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4667,7 +5090,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4698,11 +5121,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4832,7 +5255,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/unbloat-docs.lock.yml b/.github/workflows/unbloat-docs.lock.yml index 5b20c235900..b8f07157e16 100644 --- a/.github/workflows/unbloat-docs.lock.yml +++ b/.github/workflows/unbloat-docs.lock.yml @@ -4883,6 +4883,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -4920,6 +4923,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -4929,6 +5112,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4953,13 +5139,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4976,61 +5162,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -5040,13 +5233,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -5094,10 +5289,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -5313,7 +5508,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -5377,12 +5572,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -5410,7 +5611,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -5474,7 +5680,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -5490,7 +5696,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -5514,6 +5720,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -5585,7 +6008,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -5616,11 +6039,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -5750,7 +6173,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/video-analyzer.lock.yml b/.github/workflows/video-analyzer.lock.yml index 32f963bde0b..3a09d9e43b5 100644 --- a/.github/workflows/video-analyzer.lock.yml +++ b/.github/workflows/video-analyzer.lock.yml @@ -3144,6 +3144,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3181,6 +3184,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3190,6 +3373,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -3214,13 +3400,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -3237,61 +3423,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -3301,13 +3494,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -3355,10 +3550,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -3574,7 +3769,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -3638,12 +3833,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -3671,7 +3872,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -3735,7 +3941,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -3751,7 +3957,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -3775,6 +3981,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -3846,7 +4269,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -3877,11 +4300,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4011,7 +4434,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/.github/workflows/weekly-issue-summary.lock.yml b/.github/workflows/weekly-issue-summary.lock.yml index ab16f3caf48..b6de77ce22a 100644 --- a/.github/workflows/weekly-issue-summary.lock.yml +++ b/.github/workflows/weekly-issue-summary.lock.yml @@ -3939,6 +3939,9 @@ jobs: function getRedactedDomains() { return [...redactedDomains]; } + function addRedactedDomain(domain) { + redactedDomains.push(domain); + } function clearRedactedDomains() { redactedDomains.length = 0; } @@ -3976,6 +3979,186 @@ jobs: return []; } } + function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + allowedDomains = [...new Set(allowedDomains)]; + let sanitized = content; + sanitized = neutralizeCommands(sanitized); + sanitized = neutralizeAllMentions(sanitized); + sanitized = removeXmlComments(sanitized); + sanitized = convertXmlTags(sanitized); + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + sanitized = neutralizeBotTriggers(sanitized); + return sanitized.trim(); + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); + }); + if (isAllowed) { + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + function sanitizeUrlProtocols(s) { + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + function neutralizeAllMentions(s) { + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + function removeXmlComments(s) { + return s.replace(//g, "").replace(//g, ""); + } + function convertXmlTags(s) { + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + s = s.replace(//g, (match, content) => { + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + return `(![CDATA[${convertedContent}]])`; + }); + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; + } + } + return `(${tagContent})`; + }); + } + function neutralizeBotTriggers(s) { + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } + } function sanitizeContent(content, maxLengthOrOptions) { let maxLength; let allowedAliasesLowercase = []; @@ -3985,6 +4168,9 @@ jobs: maxLength = maxLengthOrOptions.maxLength; allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } if (!content || typeof content !== "string") { return ""; } @@ -4009,13 +4195,13 @@ jobs: allowedDomains = [...new Set(allowedDomains)]; let sanitized = content; sanitized = neutralizeCommands(sanitized); - sanitized = neutralizeMentions(sanitized); + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); sanitized = removeXmlComments(sanitized); sanitized = convertXmlTags(sanitized); sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); sanitized = sanitizeUrlProtocols(sanitized); - sanitized = sanitizeUrlDomains(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; @@ -4032,61 +4218,68 @@ jobs: } sanitized = neutralizeBotTriggers(sanitized); return sanitized.trim(); - function sanitizeUrlDomains(s) { - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + if (hostname === normalizedAllowed) { + return true; + } + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; - } - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; - } else { - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); } + addRedactedDomain(hostname); + return "(redacted)"; } - return result; }); - return s; + return result; } function sanitizeUrlProtocols(s) { - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - if (protocol.toLowerCase() === "https") { - return match; - } - if (match.includes("::")) { - return match; - } - if (match.includes("://")) { - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - return match; - }); + ); } function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -4096,13 +4289,15 @@ jobs: const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } - function neutralizeMentions(s) { + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; } - core.info(`Escaped mention: @${p2} (not in allowed list)`); + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; }); } @@ -4150,10 +4345,10 @@ jobs: if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; + return match; } } - return `(${tagContent})`; + return `(${tagContent})`; }); } function neutralizeBotTriggers(s) { @@ -4369,7 +4564,7 @@ jobs: } return { isValid: true, normalizedValue: parsed, isTemporary: false }; } - function validateField(value, fieldName, validation, itemType, lineNum) { + function validateField(value, fieldName, validation, itemType, lineNum, options) { if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); } @@ -4433,12 +4628,18 @@ jobs: const matchIndex = normalizedEnum.indexOf(normalizedValue); let normalizedResult = validation.enum[matchIndex]; if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } return { isValid: true, normalizedValue: value }; @@ -4466,7 +4667,12 @@ jobs: } if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -4530,7 +4736,7 @@ jobs: } return null; } - function validateItem(item, itemType, lineNum) { + function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; if (!typeConfig) { @@ -4546,7 +4752,7 @@ jobs: } for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); } else if (result.normalizedValue !== undefined) { @@ -4570,6 +4776,223 @@ jobs: const validationConfig = loadValidationConfig(); return Object.keys(validationConfig); } + function extractMentions(text) { + if (!text || typeof text !== "string") { + return []; + } + const mentionRegex = /(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const username = match[2]; + const lowercaseUsername = username.toLowerCase(); + if (!seen.has(lowercaseUsername)) { + seen.add(lowercaseUsername); + mentions.push(username); + } + } + return mentions; + } + function isPayloadUserBot(user) { + return !!(user && user.type === "Bot"); + } + async function getRecentCollaborators(owner, repo, github, core) { + try { + const collaborators = await github.rest.repos.listCollaborators({ + owner: owner, + repo: repo, + affiliation: "direct", + per_page: 30, + }); + const allowedMap = new Map(); + for (const collaborator of collaborators.data) { + const lowercaseLogin = collaborator.login.toLowerCase(); + const isAllowed = collaborator.type !== "Bot"; + allowedMap.set(lowercaseLogin, isAllowed); + } + return allowedMap; + } catch (error) { + core.warning(`Failed to fetch recent collaborators: ${error instanceof Error ? error.message : String(error)}`); + return new Map(); + } + } + async function checkUserPermission(username, owner, repo, github, core) { + try { + const { data: user } = await github.rest.users.getByUsername({ + username: username, + }); + if (user.type === "Bot") { + return false; + } + const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: owner, + repo: repo, + username: username, + }); + return permissionData.permission !== "none"; + } catch (error) { + return false; + } + } + async function resolveMentionsLazily(text, knownAuthors, owner, repo, github, core) { + const mentions = extractMentions(text); + const totalMentions = mentions.length; + core.info(`Found ${totalMentions} unique mentions in text`); + const limitExceeded = totalMentions > 50; + const mentionsToProcess = limitExceeded ? mentions.slice(0, 50) : mentions; + if (limitExceeded) { + core.warning(`Mention limit exceeded: ${totalMentions} mentions found, processing only first 50`); + } + const knownAuthorsLowercase = new Set(knownAuthors.filter(a => a).map(a => a.toLowerCase())); + const collaboratorCache = await getRecentCollaborators(owner, repo, github, core); + core.info(`Cached ${collaboratorCache.size} recent collaborators for optimistic resolution`); + const allowedMentions = []; + let resolvedCount = 0; + for (const mention of mentionsToProcess) { + const lowerMention = mention.toLowerCase(); + if (knownAuthorsLowercase.has(lowerMention)) { + allowedMentions.push(mention); + continue; + } + if (collaboratorCache.has(lowerMention)) { + if (collaboratorCache.get(lowerMention)) { + allowedMentions.push(mention); + } + continue; + } + resolvedCount++; + const isAllowed = await checkUserPermission(mention, owner, repo, github, core); + if (isAllowed) { + allowedMentions.push(mention); + } + } + core.info(`Resolved ${resolvedCount} mentions via individual API calls`); + core.info(`Total allowed mentions: ${allowedMentions.length}`); + return { + allowedMentions, + totalMentions, + resolvedCount, + limitExceeded, + }; + } + async function resolveAllowedMentionsFromPayload(context, github, core) { + if (!context || !github || !core) { + return []; + } + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + case "workflow_dispatch": + knownAuthors.push(context.actor); + break; + default: + break; + } + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + return []; + } + } + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; try { if (fs.existsSync(validationConfigPath)) { @@ -4641,7 +5064,7 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -4672,11 +5095,11 @@ jobs: error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -4806,7 +5229,7 @@ jobs: } core.info(`Line ${i + 1}: type '${itemType}'`); if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/pkg/workflow/js.go b/pkg/workflow/js.go index f667ef1e7d3..70939712489 100644 --- a/pkg/workflow/js.go +++ b/pkg/workflow/js.go @@ -279,6 +279,15 @@ var safeOutputsToolsLoaderScript string //go:embed js/safe_outputs_bootstrap.cjs var safeOutputsBootstrapScript string +//go:embed js/resolve_mentions_from_payload.cjs +var resolveMentionsFromPayloadScript string + +//go:embed js/sanitize_incoming_text.cjs +var sanitizeIncomingTextScript string + +//go:embed js/sanitize_content_core.cjs +var sanitizeContentCoreScript string + // GetJavaScriptSources returns a map of all embedded JavaScript sources // The keys are the relative paths from the js directory func GetJavaScriptSources() map[string]string { @@ -344,6 +353,9 @@ func GetJavaScriptSources() map[string]string { "safe_outputs_tools.json": safeOutputsToolsJSON, "safe_outputs_bootstrap.cjs": safeOutputsBootstrapScript, "safe_outputs_mcp_server.cjs": safeOutputsMCPServerScriptSource, + "resolve_mentions_from_payload.cjs": resolveMentionsFromPayloadScript, + "sanitize_incoming_text.cjs": sanitizeIncomingTextScript, + "sanitize_content_core.cjs": sanitizeContentCoreScript, "add_copilot_reviewer.cjs": addCopilotReviewerScriptSource, "add_reaction_and_edit_comment.cjs": addReactionAndEditCommentScriptSource, "assign_issue.cjs": assignIssueScriptSource, diff --git a/pkg/workflow/js/collect_ndjson_output.cjs b/pkg/workflow/js/collect_ndjson_output.cjs index 9d23cc25add..a20067f6bf9 100644 --- a/pkg/workflow/js/collect_ndjson_output.cjs +++ b/pkg/workflow/js/collect_ndjson_output.cjs @@ -12,6 +12,11 @@ async function main() { MAX_BODY_LENGTH: maxBodyLength, resetValidationConfigCache, } = require("./safe_output_type_validator.cjs"); + const { resolveAllowedMentionsFromPayload } = require("./resolve_mentions_from_payload.cjs"); + + // Resolve allowed mentions for the output collector + // This determines which @mentions are allowed in the agent output + const allowedMentions = await resolveAllowedMentionsFromPayload(context, github, core); // Load validation config from file and set it in environment for the validator to read const validationConfigPath = process.env.GH_AW_VALIDATION_CONFIG_PATH || "/tmp/gh-aw/safeoutputs/validation.json"; @@ -87,7 +92,7 @@ async function main() { error: `Line ${lineNum}: ${fieldName} must be a string`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; case "boolean": if (typeof value !== "boolean") { @@ -118,11 +123,11 @@ async function main() { error: `Line ${lineNum}: ${fieldName} must be one of: ${inputSchema.options.join(", ")}`, }; } - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); break; default: if (typeof value === "string") { - normalizedValue = sanitizeContent(value); + normalizedValue = sanitizeContent(value, { allowedAliases: allowedMentions }); } break; } @@ -263,7 +268,7 @@ async function main() { // Use the validation engine to validate the item if (hasValidationConfig(itemType)) { - const validationResult = validateItem(item, itemType, i + 1); + const validationResult = validateItem(item, itemType, i + 1, { allowedAliases: allowedMentions }); if (!validationResult.isValid) { if (validationResult.error) { errors.push(validationResult.error); diff --git a/pkg/workflow/js/collect_ndjson_output.test.cjs b/pkg/workflow/js/collect_ndjson_output.test.cjs index 38c4a902b02..6ab4dd5be08 100644 --- a/pkg/workflow/js/collect_ndjson_output.test.cjs +++ b/pkg/workflow/js/collect_ndjson_output.test.cjs @@ -66,6 +66,28 @@ describe("collect_ndjson_output.cjs", () => { }; global.core = mockCore; + // Mock context and github for the helper function + global.context = { + eventName: "issues", + actor: "test-actor", + repo: { + owner: "test-owner", + repo: "test-repo", + }, + payload: {}, + }; + + global.github = { + rest: { + repos: { + listCollaborators: vi.fn().mockResolvedValue({ data: [] }), + }, + users: { + getByUsername: vi.fn(), + }, + }, + }; + // Read the script file const scriptPath = path.join(__dirname, "collect_ndjson_output.cjs"); collectScript = fs.readFileSync(scriptPath, "utf8"); diff --git a/pkg/workflow/js/compute_text.cjs b/pkg/workflow/js/compute_text.cjs index f1e429276f1..8e4bc830222 100644 --- a/pkg/workflow/js/compute_text.cjs +++ b/pkg/workflow/js/compute_text.cjs @@ -6,8 +6,8 @@ * @param {string} content - The content to sanitize * @returns {string} The sanitized content */ -const { sanitizeContent, writeRedactedDomainsLog } = require("./sanitize_content.cjs"); -const { isPayloadUserBot, resolveMentionsLazily } = require("./resolve_mentions.cjs"); +const { sanitizeIncomingText, writeRedactedDomainsLog } = require("./sanitize_incoming_text.cjs"); +const { isPayloadUserBot } = require("./resolve_mentions.cjs"); async function main() { let text = ""; @@ -267,23 +267,10 @@ async function main() { break; } - // Resolve mentions lazily using the new helper - const mentionResult = await resolveMentionsLazily(text, knownAuthors, owner, repo, github, core); - - // Log known authors for debugging - if (knownAuthors.length > 0) { - core.info(`Known authors (from payload): ${knownAuthors.join(", ")}`); - } - - // Log allowed mentions for documentation - if (mentionResult.allowedMentions.length > 0) { - core.info(`Allowed mentions (will not be escaped): ${mentionResult.allowedMentions.join(", ")}`); - } else { - core.info("No allowed mentions configured - all mentions will be escaped"); - } - - // Sanitize the text before output, passing the known authors - const sanitizedText = sanitizeContent(text, { allowedAliases: mentionResult.allowedMentions }); + // Sanitize the text before output using slimmed-down sanitizer + // Note: Mention filtering is NOT applied here - all mentions are escaped + // Mention filtering will be applied by the agent output collector + const sanitizedText = sanitizeIncomingText(text); // Display sanitized text in logs core.info(`text: ${sanitizedText}`); diff --git a/pkg/workflow/js/compute_text.test.cjs b/pkg/workflow/js/compute_text.test.cjs index 5720c585cb6..918fef16acd 100644 --- a/pkg/workflow/js/compute_text.test.cjs +++ b/pkg/workflow/js/compute_text.test.cjs @@ -91,11 +91,11 @@ describe("compute_text.cjs", () => { const scriptPath = path.join(process.cwd(), "compute_text.cjs"); computeTextScript = fs.readFileSync(scriptPath, "utf8"); - // Extract sanitizeContent function for unit testing + // Extract sanitizeIncomingText function for unit testing // We need to eval the script to get access to the function const scriptWithExport = computeTextScript.replace( "await main();", - "global.testSanitizeContent = sanitizeContent; global.testMain = main;" + "global.testSanitizeContent = sanitizeIncomingText; global.testMain = main;" ); eval(scriptWithExport); sanitizeContentFunction = global.testSanitizeContent; @@ -422,7 +422,7 @@ describe("compute_text.cjs", () => { expect(mockCore.setOutput).toHaveBeenCalledWith("text", ""); }); - it("should allow issue author mention without neutralization", async () => { + it("should neutralize all mentions including issue author", async () => { mockContext.eventName = "issues"; mockContext.payload = { issue: { @@ -435,14 +435,12 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // @issueAuthor should not be neutralized (allowed alias) - expect(outputCall[1]).toContain("@issueAuthor"); - expect(outputCall[1]).not.toContain("`@issueAuthor`"); - // @other should be neutralized + // All mentions should be neutralized (mention filtering is done by output collector, not compute_text) + expect(outputCall[1]).toContain("`@issueAuthor`"); expect(outputCall[1]).toContain("`@other`"); }); - it("should allow PR author mention without neutralization", async () => { + it("should neutralize all mentions including PR author", async () => { mockContext.eventName = "pull_request"; mockContext.payload = { pull_request: { @@ -455,11 +453,11 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - expect(outputCall[1]).toContain("@prAuthor"); - expect(outputCall[1]).not.toContain("`@prAuthor`"); + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@prAuthor`"); }); - it("should allow comment author mention without neutralization", async () => { + it("should neutralize all mentions including comment author", async () => { mockContext.eventName = "issue_comment"; mockContext.payload = { comment: { @@ -471,11 +469,11 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - expect(outputCall[1]).toContain("@commentAuthor"); - expect(outputCall[1]).not.toContain("`@commentAuthor`"); + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@commentAuthor`"); }); - it("should allow discussion author mention without neutralization", async () => { + it("should neutralize all mentions including discussion author", async () => { mockContext.eventName = "discussion"; mockContext.payload = { discussion: { @@ -488,11 +486,11 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - expect(outputCall[1]).toContain("@discussionAuthor"); - expect(outputCall[1]).not.toContain("`@discussionAuthor`"); + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@discussionAuthor`"); }); - it("should allow release author mention without neutralization", async () => { + it("should neutralize all mentions including release author", async () => { mockContext.eventName = "release"; mockContext.payload = { release: { @@ -505,11 +503,11 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - expect(outputCall[1]).toContain("@releaseAuthor"); - expect(outputCall[1]).not.toContain("`@releaseAuthor`"); + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@releaseAuthor`"); }); - it("should handle case-insensitive author matching", async () => { + it("should neutralize all mentions regardless of case", async () => { mockContext.eventName = "issues"; mockContext.payload = { issue: { @@ -522,14 +520,12 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // Both @AUTHOR and @author should be allowed - expect(outputCall[1]).toContain("@AUTHOR"); - expect(outputCall[1]).not.toContain("`@AUTHOR`"); - expect(outputCall[1]).toContain("@author"); - expect(outputCall[1]).not.toContain("`@author`"); + // All mentions should be neutralized regardless of case + expect(outputCall[1]).toContain("`@AUTHOR`"); + expect(outputCall[1]).toContain("`@author`"); }); - it("should allow both comment author and parent issue author for issue_comment", async () => { + it("should neutralize all mentions including comment and issue authors", async () => { mockContext.eventName = "issue_comment"; mockContext.payload = { comment: { @@ -544,16 +540,13 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // Both comment author and issue author should not be neutralized - expect(outputCall[1]).toContain("@commentAuthor"); - expect(outputCall[1]).not.toContain("`@commentAuthor`"); - expect(outputCall[1]).toContain("@issueAuthor"); - expect(outputCall[1]).not.toContain("`@issueAuthor`"); - // @other should be neutralized + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@commentAuthor`"); + expect(outputCall[1]).toContain("`@issueAuthor`"); expect(outputCall[1]).toContain("`@other`"); }); - it("should allow both comment author and parent PR author for pull_request_review_comment", async () => { + it("should neutralize all mentions including review comment and PR authors", async () => { mockContext.eventName = "pull_request_review_comment"; mockContext.payload = { comment: { @@ -568,16 +561,13 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // Both comment author and PR author should not be neutralized - expect(outputCall[1]).toContain("@reviewCommentAuthor"); - expect(outputCall[1]).not.toContain("`@reviewCommentAuthor`"); - expect(outputCall[1]).toContain("@prAuthor"); - expect(outputCall[1]).not.toContain("`@prAuthor`"); - // @other should be neutralized + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@reviewCommentAuthor`"); + expect(outputCall[1]).toContain("`@prAuthor`"); expect(outputCall[1]).toContain("`@other`"); }); - it("should allow both review author and parent PR author for pull_request_review", async () => { + it("should neutralize all mentions including review and PR authors", async () => { mockContext.eventName = "pull_request_review"; mockContext.payload = { review: { @@ -592,16 +582,13 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // Both review author and PR author should not be neutralized - expect(outputCall[1]).toContain("@reviewAuthor"); - expect(outputCall[1]).not.toContain("`@reviewAuthor`"); - expect(outputCall[1]).toContain("@prAuthor"); - expect(outputCall[1]).not.toContain("`@prAuthor`"); - // @other should be neutralized + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@reviewAuthor`"); + expect(outputCall[1]).toContain("`@prAuthor`"); expect(outputCall[1]).toContain("`@other`"); }); - it("should allow both comment author and parent discussion author for discussion_comment", async () => { + it("should neutralize all mentions including comment and discussion authors", async () => { mockContext.eventName = "discussion_comment"; mockContext.payload = { comment: { @@ -616,16 +603,13 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // Both comment author and discussion author should not be neutralized - expect(outputCall[1]).toContain("@commentAuthor"); - expect(outputCall[1]).not.toContain("`@commentAuthor`"); - expect(outputCall[1]).toContain("@discussionAuthor"); - expect(outputCall[1]).not.toContain("`@discussionAuthor`"); - // @other should be neutralized + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@commentAuthor`"); + expect(outputCall[1]).toContain("`@discussionAuthor`"); expect(outputCall[1]).toContain("`@other`"); }); - it("should allow workflow_dispatch actor mention without neutralization", async () => { + it("should neutralize all mentions including workflow_dispatch actor", async () => { // Set up actor and event mockContext.actor = "dispatchActor"; mockContext.eventName = "workflow_dispatch"; @@ -647,13 +631,9 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // @dispatchActor should not be neutralized (workflow_dispatch actor) - expect(outputCall[1]).toContain("@dispatchActor"); - expect(outputCall[1]).not.toContain("`@dispatchActor`"); - // @releaseAuthor should also not be neutralized - expect(outputCall[1]).toContain("@releaseAuthor"); - expect(outputCall[1]).not.toContain("`@releaseAuthor`"); - // @other should be neutralized + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@dispatchActor`"); + expect(outputCall[1]).toContain("`@releaseAuthor`"); expect(outputCall[1]).toContain("`@other`"); }); @@ -670,7 +650,7 @@ describe("compute_text.cjs", () => { expect(mockCore.setOutput).toHaveBeenCalledWith("text", ""); }); - it("should allow workflow_dispatch actor with release_url", async () => { + it("should neutralize all mentions in workflow_dispatch with release_url", async () => { mockContext.actor = "dispatchActor"; mockContext.eventName = "workflow_dispatch"; mockContext.payload = { @@ -691,12 +671,11 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // @dispatchActor should not be neutralized - expect(outputCall[1]).toContain("@dispatchActor"); - expect(outputCall[1]).not.toContain("`@dispatchActor`"); + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@dispatchActor`"); }); - it("should filter out bot authors from allowed mentions", async () => { + it("should neutralize bot authors like any other mention", async () => { mockContext.eventName = "issues"; mockContext.payload = { issue: { @@ -709,13 +688,11 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // @botUser should be neutralized (bot author) + // All mentions should be neutralized expect(outputCall[1]).toContain("`@botUser`"); - // Should not have @botUser without backticks (look for pattern without backtick before) - expect(outputCall[1]).not.toMatch(/[^`]@botUser/); }); - it("should allow team member mentions", async () => { + it("should neutralize all mentions including team members", async () => { mockContext.eventName = "issues"; mockContext.payload = { issue: { @@ -728,14 +705,12 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // Team members should not be neutralized - expect(outputCall[1]).toContain("@team-member-1"); - expect(outputCall[1]).not.toContain("`@team-member-1`"); - expect(outputCall[1]).toContain("@team-member-2"); - expect(outputCall[1]).not.toContain("`@team-member-2`"); + // All mentions should be neutralized + expect(outputCall[1]).toContain("`@team-member-1`"); + expect(outputCall[1]).toContain("`@team-member-2`"); }); - it("should not allow bot team members in mentions", async () => { + it("should neutralize bot team members like any other mention", async () => { mockContext.eventName = "issues"; mockContext.payload = { issue: { @@ -748,11 +723,11 @@ describe("compute_text.cjs", () => { await testMain(); const outputCall = mockCore.setOutput.mock.calls[0]; - // @dependabot should be neutralized (bot) + // All mentions should be neutralized expect(outputCall[1]).toContain("`@dependabot`"); }); - it("should log allowed mentions", async () => { + it("should not log allowed mentions (mentions not resolved in compute_text)", async () => { mockContext.eventName = "issues"; mockContext.payload = { issue: { @@ -764,17 +739,13 @@ describe("compute_text.cjs", () => { await testMain(); - // Check that allowed mentions were logged + // Check that allowed mentions were NOT logged (mention resolution moved to output collector) const infoCalls = mockCore.info.mock.calls.map(call => call[0]); const allowedMentionsLog = infoCalls.find(msg => msg.includes("Allowed mentions")); - expect(allowedMentionsLog).toBeDefined(); - // Should include team members and issue author that were actually mentioned in text - expect(allowedMentionsLog).toContain("team-member-1"); - expect(allowedMentionsLog).toContain("team-member-2"); - expect(allowedMentionsLog).toContain("issueAuthor"); + expect(allowedMentionsLog).toBeUndefined(); }); - it("should log known authors from payload", async () => { + it("should not log known authors from payload (not tracked in compute_text)", async () => { mockContext.eventName = "issues"; mockContext.payload = { issue: { @@ -790,13 +761,10 @@ describe("compute_text.cjs", () => { await testMain(); - // Check that known authors were logged + // Check that known authors were NOT logged (mention resolution moved to output collector) const infoCalls = mockCore.info.mock.calls.map(call => call[0]); const knownAuthorsLog = infoCalls.find(msg => msg.includes("Known authors (from payload)")); - expect(knownAuthorsLog).toBeDefined(); - expect(knownAuthorsLog).toContain("issueAuthor"); - expect(knownAuthorsLog).toContain("assignee1"); - expect(knownAuthorsLog).toContain("assignee2"); + expect(knownAuthorsLog).toBeUndefined(); }); it("should log escaped mentions", async () => { @@ -809,33 +777,19 @@ describe("compute_text.cjs", () => { }, }; - // First call: actor permission check (should be admin to allow processing) - // Second call: unknown-user permission check (should be none to escape mention) - mockGithub.rest.repos.getCollaboratorPermissionLevel - .mockResolvedValueOnce({ - data: { permission: "admin" }, - }) - .mockResolvedValueOnce({ - data: { permission: "none" }, - }); - - // Mock that unknown-user exists but is not a collaborator - mockGithub.rest.users.getByUsername.mockResolvedValueOnce({ - data: { login: "unknown-user", type: "User" }, - }); - await testMain(); - // Check that escaped mention was logged + // Check that escaped mentions were logged const infoCalls = mockCore.info.mock.calls.map(call => call[0]); - const escapedMentionLog = infoCalls.find(msg => msg.includes("Escaped mention")); - expect(escapedMentionLog).toBeDefined(); - expect(escapedMentionLog).toContain("@unknown-user"); + const escapedMentionLogs = infoCalls.filter(msg => msg.includes("Escaped mention")); + // Should have logged both mentions as escaped + expect(escapedMentionLogs.length).toBeGreaterThanOrEqual(2); + const allEscapedMentions = escapedMentionLogs.join(" "); + expect(allEscapedMentions).toContain("@unknown-user"); + expect(allEscapedMentions).toContain("@team-member-1"); }); - it("should handle team member fetch failure gracefully", async () => { - mockGithub.rest.repos.listCollaborators.mockRejectedValue(new Error("API error")); - + it("should not handle team member fetch (moved to output collector)", async () => { mockContext.eventName = "issues"; mockContext.payload = { issue: { @@ -847,10 +801,8 @@ describe("compute_text.cjs", () => { await testMain(); - // Should still work and log a warning - const warningCalls = mockCore.warning.mock.calls.map(call => call[0]); - const collaboratorWarning = warningCalls.find(msg => msg.includes("Failed to fetch recent collaborators")); - expect(collaboratorWarning).toBeDefined(); + // listCollaborators should NOT be called (mention resolution moved to output collector) + expect(mockGithub.rest.repos.listCollaborators).not.toHaveBeenCalled(); // Should still set output with issue text expect(mockCore.setOutput).toHaveBeenCalledWith("text", expect.any(String)); diff --git a/pkg/workflow/js/resolve_mentions_from_payload.cjs b/pkg/workflow/js/resolve_mentions_from_payload.cjs new file mode 100644 index 00000000000..5fcf2c8b7a7 --- /dev/null +++ b/pkg/workflow/js/resolve_mentions_from_payload.cjs @@ -0,0 +1,158 @@ +// @ts-check +/// + +/** + * Helper module for resolving allowed mentions from GitHub event payloads + */ + +const { resolveMentionsLazily, isPayloadUserBot } = require("./resolve_mentions.cjs"); + +/** + * Resolve allowed mentions from the current GitHub event context + * @param {any} context - GitHub Actions context + * @param {any} github - GitHub API client + * @param {any} core - GitHub Actions core + * @returns {Promise} Array of allowed mention usernames + */ +async function resolveAllowedMentionsFromPayload(context, github, core) { + // Return empty array if context is not available (e.g., in tests) + if (!context || !github || !core) { + return []; + } + + try { + const { owner, repo } = context.repo; + const knownAuthors = []; + + // Extract known authors from the event payload + switch (context.eventName) { + case "issues": + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + + case "pull_request": + case "pull_request_target": + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + + case "issue_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.issue?.user?.login && !isPayloadUserBot(context.payload.issue.user)) { + knownAuthors.push(context.payload.issue.user.login); + } + if (context.payload.issue?.assignees && Array.isArray(context.payload.issue.assignees)) { + for (const assignee of context.payload.issue.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + + case "pull_request_review_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + + case "pull_request_review": + if (context.payload.review?.user?.login && !isPayloadUserBot(context.payload.review.user)) { + knownAuthors.push(context.payload.review.user.login); + } + if (context.payload.pull_request?.user?.login && !isPayloadUserBot(context.payload.pull_request.user)) { + knownAuthors.push(context.payload.pull_request.user.login); + } + if (context.payload.pull_request?.assignees && Array.isArray(context.payload.pull_request.assignees)) { + for (const assignee of context.payload.pull_request.assignees) { + if (assignee?.login && !isPayloadUserBot(assignee)) { + knownAuthors.push(assignee.login); + } + } + } + break; + + case "discussion": + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + + case "discussion_comment": + if (context.payload.comment?.user?.login && !isPayloadUserBot(context.payload.comment.user)) { + knownAuthors.push(context.payload.comment.user.login); + } + if (context.payload.discussion?.user?.login && !isPayloadUserBot(context.payload.discussion.user)) { + knownAuthors.push(context.payload.discussion.user.login); + } + break; + + case "release": + if (context.payload.release?.author?.login && !isPayloadUserBot(context.payload.release.author)) { + knownAuthors.push(context.payload.release.author.login); + } + break; + + case "workflow_dispatch": + // Add the actor who triggered the workflow + knownAuthors.push(context.actor); + break; + + default: + // No known authors for other event types + break; + } + + // Build allowed mentions list from known authors and collaborators + // We pass the known authors as fake mentions in text so they get processed + const fakeText = knownAuthors.map(author => `@${author}`).join(" "); + const mentionResult = await resolveMentionsLazily(fakeText, knownAuthors, owner, repo, github, core); + const allowedMentions = mentionResult.allowedMentions; + + // Log allowed mentions for debugging + if (allowedMentions.length > 0) { + core.info(`[OUTPUT COLLECTOR] Allowed mentions: ${allowedMentions.join(", ")}`); + } else { + core.info("[OUTPUT COLLECTOR] No allowed mentions - all mentions will be escaped"); + } + + return allowedMentions; + } catch (error) { + core.warning(`Failed to resolve mentions for output collector: ${error instanceof Error ? error.message : String(error)}`); + // Return empty array on error + return []; + } +} + +module.exports = { + resolveAllowedMentionsFromPayload, +}; diff --git a/pkg/workflow/js/safe_output_type_validator.cjs b/pkg/workflow/js/safe_output_type_validator.cjs index 1f7587c0bae..71d8ac7b960 100644 --- a/pkg/workflow/js/safe_output_type_validator.cjs +++ b/pkg/workflow/js/safe_output_type_validator.cjs @@ -239,9 +239,11 @@ function validateIssueNumberOrTemporaryId(value, fieldName, lineNum) { * @param {FieldValidation} validation - The validation configuration * @param {string} itemType - The item type for error messages * @param {number} lineNum - Line number for error messages + * @param {Object} [options] - Optional sanitization options + * @param {string[]} [options.allowedAliases] - List of allowed @mentions * @returns {{isValid: boolean, normalizedValue?: any, error?: string}} */ -function validateField(value, fieldName, validation, itemType, lineNum) { +function validateField(value, fieldName, validation, itemType, lineNum, options) { // For positiveInteger fields, delegate required check to validatePositiveInteger if (validation.positiveInteger) { return validatePositiveInteger(value, `${itemType} '${fieldName}'`, lineNum); @@ -327,14 +329,20 @@ function validateField(value, fieldName, validation, itemType, lineNum) { let normalizedResult = validation.enum[matchIndex]; // Apply sanitization if configured if (validation.sanitize && validation.maxLength) { - normalizedResult = sanitizeContent(normalizedResult, validation.maxLength); + normalizedResult = sanitizeContent(normalizedResult, { + maxLength: validation.maxLength, + allowedAliases: options?.allowedAliases || [], + }); } return { isValid: true, normalizedValue: normalizedResult }; } // Handle sanitization if (validation.sanitize) { - const sanitized = sanitizeContent(value, validation.maxLength || MAX_BODY_LENGTH); + const sanitized = sanitizeContent(value, { + maxLength: validation.maxLength || MAX_BODY_LENGTH, + allowedAliases: options?.allowedAliases || [], + }); return { isValid: true, normalizedValue: sanitized }; } @@ -369,7 +377,12 @@ function validateField(value, fieldName, validation, itemType, lineNum) { // Sanitize items if configured if (validation.itemSanitize) { const sanitizedItems = value.map(item => - typeof item === "string" ? sanitizeContent(item, validation.itemMaxLength || 128) : item + typeof item === "string" + ? sanitizeContent(item, { + maxLength: validation.itemMaxLength || 128, + allowedAliases: options?.allowedAliases || [], + }) + : item ); return { isValid: true, normalizedValue: sanitizedItems }; } @@ -459,9 +472,11 @@ function executeCustomValidation(item, customValidation, lineNum, itemType) { * @param {Object} item - The item to validate * @param {string} itemType - The item type (e.g., "create_issue") * @param {number} lineNum - Line number for error messages + * @param {Object} [options] - Optional sanitization options + * @param {string[]} [options.allowedAliases] - List of allowed @mentions * @returns {{isValid: boolean, normalizedItem?: Object, error?: string}} */ -function validateItem(item, itemType, lineNum) { +function validateItem(item, itemType, lineNum, options) { const validationConfig = loadValidationConfig(); const typeConfig = validationConfig[itemType]; @@ -484,7 +499,7 @@ function validateItem(item, itemType, lineNum) { // Validate each configured field for (const [fieldName, validation] of Object.entries(typeConfig.fields)) { const fieldValue = item[fieldName]; - const result = validateField(fieldValue, fieldName, validation, itemType, lineNum); + const result = validateField(fieldValue, fieldName, validation, itemType, lineNum, options); if (!result.isValid) { errors.push(result.error); diff --git a/pkg/workflow/js/sanitize_content.cjs b/pkg/workflow/js/sanitize_content.cjs index b8cab1dd33e..920fc6a75ed 100644 --- a/pkg/workflow/js/sanitize_content.cjs +++ b/pkg/workflow/js/sanitize_content.cjs @@ -1,96 +1,18 @@ // @ts-check /** - * Shared sanitization utilities for GitHub Actions output - * This module provides functions for sanitizing content to prevent security issues - * and unintended side effects in GitHub Actions workflows. + * Full sanitization utilities with mention filtering support + * This module provides the complete sanitization with selective mention filtering. + * For incoming text that doesn't need mention filtering, use sanitize_incoming_text.cjs instead. */ -/** - * Module-level set to collect redacted URL domains across sanitization calls. - * @type {string[]} - */ -const redactedDomains = []; - -/** - * Gets the list of redacted URL domains collected during sanitization. - * @returns {string[]} Array of redacted domain strings - */ -function getRedactedDomains() { - return [...redactedDomains]; -} - -/** - * Clears the list of redacted URL domains. - * Useful for testing or resetting state between operations. - */ -function clearRedactedDomains() { - redactedDomains.length = 0; -} - -/** - * Writes the collected redacted URL domains to a log file. - * Only creates the file if there are redacted domains. - * @param {string} [filePath] - Path to write the log file. Defaults to /tmp/gh-aw/redacted-urls.log - * @returns {string|null} The file path if written, null if no domains to write - */ -function writeRedactedDomainsLog(filePath) { - if (redactedDomains.length === 0) { - return null; - } - - const fs = require("fs"); - const path = require("path"); - const targetPath = filePath || "/tmp/gh-aw/redacted-urls.log"; - - // Ensure directory exists - const dir = path.dirname(targetPath); - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - - // Write domains to file, one per line - fs.writeFileSync(targetPath, redactedDomains.join("\n") + "\n"); - - return targetPath; -} - -/** - * Extract domains from a URL and return an array of domain variations - * @param {string} url - The URL to extract domains from - * @returns {string[]} Array of domain variations - */ -function extractDomainsFromUrl(url) { - if (!url || typeof url !== "string") { - return []; - } - - try { - // Parse the URL - const urlObj = new URL(url); - const hostname = urlObj.hostname.toLowerCase(); - - // Return both the exact hostname and common variations - const domains = [hostname]; - - // For github.com, add api and raw content domain variations - if (hostname === "github.com") { - domains.push("api.github.com"); - domains.push("raw.githubusercontent.com"); - domains.push("*.githubusercontent.com"); - } - // For custom GitHub Enterprise domains, add api. prefix and raw content variations - else if (!hostname.startsWith("api.")) { - domains.push("api." + hostname); - // For GitHub Enterprise, raw content is typically served from raw.hostname - domains.push("raw." + hostname); - } - - return domains; - } catch (e) { - // Invalid URL, return empty array - return []; - } -} +const { + sanitizeContentCore, + getRedactedDomains, + clearRedactedDomains, + writeRedactedDomainsLog, + extractDomainsFromUrl, + addRedactedDomain, +} = require("./sanitize_content_core.cjs"); /** * @typedef {Object} SanitizeOptions @@ -99,7 +21,7 @@ function extractDomainsFromUrl(url) { */ /** - * Sanitizes content for safe output in GitHub Actions + * Sanitizes content for safe output in GitHub Actions with optional mention filtering * @param {string} content - The content to sanitize * @param {number | SanitizeOptions} [maxLengthOrOptions] - Maximum length of content (default: 524288) or options object * @returns {string} The sanitized content @@ -118,6 +40,15 @@ function sanitizeContent(content, maxLengthOrOptions) { // Pre-process allowed aliases to lowercase for efficient comparison allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); } + + // If no allowed aliases specified, use core sanitization (which neutralizes all mentions) + if (allowedAliasesLowercase.length === 0) { + return sanitizeContentCore(content, maxLength); + } + + // If allowed aliases are specified, we need custom mention filtering + // We'll do most of the sanitization with the core, then apply selective mention filtering + if (!content || typeof content !== "string") { return ""; } @@ -134,7 +65,6 @@ function sanitizeContent(content, maxLengthOrOptions) { : defaultAllowedDomains; // Extract and add GitHub domains from GitHub context URLs - // This handles GitHub Enterprise deployments with custom domains const githubServerUrl = process.env.GITHUB_SERVER_URL; const githubApiUrl = process.env.GITHUB_API_URL; @@ -153,41 +83,37 @@ function sanitizeContent(content, maxLengthOrOptions) { let sanitized = content; - // Neutralize commands at the start of text (e.g., /bot-name) + // Neutralize commands at the start of text sanitized = neutralizeCommands(sanitized); - // Neutralize @mentions to prevent unintended notifications - sanitized = neutralizeMentions(sanitized); + // Neutralize @mentions with selective filtering + sanitized = neutralizeMentions(sanitized, allowedAliasesLowercase); - // Remove XML comments first + // Remove XML comments sanitized = removeXmlComments(sanitized); - // Convert XML tags to parentheses format to prevent injection + // Convert XML tags sanitized = convertXmlTags(sanitized); // Remove ANSI escape sequences sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); - // Remove control characters (except newlines and tabs) + // Remove control characters sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); - // URI filtering - replace non-https protocols with "(redacted)" + // URI filtering sanitized = sanitizeUrlProtocols(sanitized); + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); - // Domain filtering for HTTPS URIs - sanitized = sanitizeUrlDomains(sanitized); - - // Check line count before length to provide more specific truncation message + // Truncation const lines = sanitized.split("\n"); const maxLines = 65000; maxLength = maxLength || 524288; - // If content has too many lines, truncate by lines (primary limit) if (lines.length > maxLines) { const truncationMsg = "\n[Content truncated due to line count]"; const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; - // If still too long after line truncation, shorten but keep the line count message if (truncatedLines.length > maxLength) { sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; } else { @@ -197,118 +123,100 @@ function sanitizeContent(content, maxLengthOrOptions) { sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; } - // Neutralize common bot trigger phrases + // Neutralize bot triggers sanitized = neutralizeBotTriggers(sanitized); - // Trim excessive whitespace return sanitized.trim(); /** - * Remove unknown domains + * Sanitize URL domains * @param {string} s - The string to process - * @returns {string} The string with unknown domains redacted + * @param {string[]} allowed - Allowed domains + * @returns {string} Sanitized string */ - function sanitizeUrlDomains(s) { - // First pass: match all HTTPS URLs and process them - // We need to handle URLs that might contain other URLs in query parameters - s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { - // Extract the hostname part (before first slash, colon, or other delimiter) - const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); - - // Check if this domain or any parent domain is in the allowlist - const isAllowed = allowedDomains.some(allowedDomain => { + function sanitizeUrlDomains(s, allowed) { + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + + const result = s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + + const isAllowed = allowed.some(allowedDomain => { const normalizedAllowed = allowedDomain.toLowerCase(); - return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + + if (hostname === normalizedAllowed) { + return true; + } + + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + + return hostname.endsWith("." + normalizedAllowed); }); if (isAllowed) { - return match; // Keep allowed URLs as-is - } - - // Log the redaction and collect the domain - const domain = hostname; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - - // For disallowed URLs, check if there are any allowed URLs in the query/fragment - // and preserve those while redacting the main URL - const urlParts = match.split(/([?&#])/); - let result = "(redacted)"; // Redact the main domain - - // Process query/fragment parts to preserve any allowed URLs within them - for (let i = 1; i < urlParts.length; i++) { - if (urlParts[i].match(/^[?&#]$/)) { - result += urlParts[i]; // Keep separators - } else { - // Recursively process this part to preserve any allowed URLs - result += sanitizeUrlDomains(urlParts[i]); + return match; + } else { + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; } - - return result; }); - return s; + return result; } /** - * Remove unknown protocols except https + * Sanitize URL protocols * @param {string} s - The string to process - * @returns {string} The string with non-https protocols redacted + * @returns {string} Sanitized string */ function sanitizeUrlProtocols(s) { - // Match protocol patterns but avoid command-line flags, file paths, and namespaces - // Protocol patterns typically have :// or are well-known schemes followed by : - // Use negative lookbehind to exclude patterns preceded by - (command flags) - // Match only patterns that look like actual protocols - return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { - // Allow https (case insensitive), redact everything else - // But only if it looks like a URL (has :// or is followed by non-colon content) - if (protocol.toLowerCase() === "https") { - return match; - } - - // Allow if it looks like a file path or namespace (::) - if (match.includes("::")) { - return match; - } - - // Redact if it has :// (definite protocol) - if (match.includes("://")) { - // Log the redaction and collect the domain - // Extract domain from URL - const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); - const domain = domainMatch ? domainMatch[1] : match; - const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(domain); - return "(redacted)"; - } - - // Redact well-known dangerous protocols like javascript:, data:, etc. - const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; - if (dangerousProtocols.includes(protocol.toLowerCase())) { - // Log the redaction and collect the domain - // For dangerous protocols without ://, show protocol and beginning of content - const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; - core.info(`Redacted URL: ${truncated}`); - core.debug(`Redacted URL (full): ${match}`); - redactedDomains.push(protocol + ":"); + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + // Extract domain for http/ftp/file/ssh/git protocols + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + // For other protocols (data:, javascript:, etc.), track the protocol itself + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } return "(redacted)"; } - - // Otherwise preserve (could be file:path, namespace:thing, etc.) - return match; - }); + ); } /** - * Neutralizes commands at the start of text by wrapping them in backticks + * Neutralize commands * @param {string} s - The string to process - * @returns {string} The string with neutralized commands + * @returns {string} Processed string */ function neutralizeCommands(s) { const commandName = process.env.GH_AW_COMMAND; @@ -316,53 +224,46 @@ function sanitizeContent(content, maxLengthOrOptions) { return s; } - // Escape special regex characters in command name const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - - // Neutralize /command at the start of text (with optional leading whitespace) - // Only match at the start of the string or after leading whitespace return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); } /** - * Neutralizes @mentions by wrapping them in backticks - * Skips mentions that are in the allowedAliases list + * Neutralize @mentions with selective filtering * @param {string} s - The string to process - * @returns {string} The string with neutralized mentions + * @param {string[]} allowedLowercase - List of allowed aliases (lowercase) + * @returns {string} Processed string */ - function neutralizeMentions(s) { - // Replace @name or @org/team outside code with `@name` - // Skip mentions that are in the allowed aliases list + function neutralizeMentions(s, allowedLowercase) { return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { // Check if this mention is in the allowed aliases list (case-insensitive) - // allowedAliasesLowercase is pre-processed to lowercase for efficient comparison - const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + const isAllowed = allowedLowercase.includes(p2.toLowerCase()); if (isAllowed) { return `${p1}@${p2}`; // Keep the original mention } - // Log when a mention is escaped to help debug issues - core.info(`Escaped mention: @${p2} (not in allowed list)`); + // Log when a mention is escaped + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } return `${p1}\`@${p2}\``; // Neutralize the mention }); } /** - * Removes XML comments from content + * Remove XML comments * @param {string} s - The string to process - * @returns {string} The string with XML comments removed + * @returns {string} Processed string */ function removeXmlComments(s) { - // Remove and malformed return s.replace(//g, "").replace(//g, ""); } /** - * Converts XML/HTML tags to parentheses format to prevent injection + * Convert XML tags * @param {string} s - The string to process - * @returns {string} The string with XML tags converted to parentheses + * @returns {string} Processed string */ function convertXmlTags(s) { - // Allow safe HTML tags: b, blockquote, br, code, details, em, h1–h6, hr, i, li, ol, p, pre, strong, sub, summary, sup, table, tbody, td, th, thead, tr, ul const allowedTags = [ "b", "blockquote", @@ -395,38 +296,29 @@ function sanitizeContent(content, maxLengthOrOptions) { "ul", ]; - // First, process CDATA sections specially - convert tags inside them and the CDATA markers s = s.replace(//g, (match, content) => { - // Convert tags inside CDATA content const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); - // Return with CDATA markers also converted to parentheses return `(![CDATA[${convertedContent}]])`; }); - // Convert opening tags: or to (tag) or (tag attr="value") - // Convert closing tags: to (/tag) - // Convert self-closing tags: or to (tag/) or (tag /) - // But preserve allowed safe tags return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { - // Extract tag name from the content (handle closing tags and attributes) const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); if (tagNameMatch) { const tagName = tagNameMatch[1].toLowerCase(); if (allowedTags.includes(tagName)) { - return match; // Preserve allowed tags + return match; } } - return `(${tagContent})`; // Convert other tags to parentheses + return `(${tagContent})`; }); } /** - * Neutralizes bot trigger phrases by wrapping them in backticks + * Neutralize bot triggers * @param {string} s - The string to process - * @returns {string} The string with neutralized bot triggers + * @returns {string} Processed string */ function neutralizeBotTriggers(s) { - // Neutralize common bot trigger phrases like "fixes #123", "closes #asdfs", etc. return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); } } diff --git a/pkg/workflow/js/sanitize_content_core.cjs b/pkg/workflow/js/sanitize_content_core.cjs new file mode 100644 index 00000000000..40f3f1b2b18 --- /dev/null +++ b/pkg/workflow/js/sanitize_content_core.cjs @@ -0,0 +1,423 @@ +// @ts-check +/** + * Core sanitization utilities without mention filtering + * This module provides the base sanitization functions that don't require + * mention resolution or filtering. It's designed to be imported by both + * sanitize_content.cjs (full version) and sanitize_incoming_text.cjs (minimal version). + */ + +/** + * Module-level set to collect redacted URL domains across sanitization calls. + * @type {string[]} + */ +const redactedDomains = []; + +/** + * Gets the list of redacted URL domains collected during sanitization. + * @returns {string[]} Array of redacted domain strings + */ +function getRedactedDomains() { + return [...redactedDomains]; +} + +/** + * Adds a domain to the redacted domains list + * @param {string} domain - Domain to add + */ +function addRedactedDomain(domain) { + redactedDomains.push(domain); +} + +/** + * Clears the list of redacted URL domains. + * Useful for testing or resetting state between operations. + */ +function clearRedactedDomains() { + redactedDomains.length = 0; +} + +/** + * Writes the collected redacted URL domains to a log file. + * Only creates the file if there are redacted domains. + * @param {string} [filePath] - Path to write the log file. Defaults to /tmp/gh-aw/redacted-urls.log + * @returns {string|null} The file path if written, null if no domains to write + */ +function writeRedactedDomainsLog(filePath) { + if (redactedDomains.length === 0) { + return null; + } + + const fs = require("fs"); + const path = require("path"); + const targetPath = filePath || "/tmp/gh-aw/redacted-urls.log"; + + // Ensure directory exists + const dir = path.dirname(targetPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + // Write domains to file, one per line + fs.writeFileSync(targetPath, redactedDomains.join("\n") + "\n"); + + return targetPath; +} + +/** + * Extract domains from a URL and return an array of domain variations + * @param {string} url - The URL to extract domains from + * @returns {string[]} Array of domain variations + */ +function extractDomainsFromUrl(url) { + if (!url || typeof url !== "string") { + return []; + } + + try { + // Parse the URL + const urlObj = new URL(url); + const hostname = urlObj.hostname.toLowerCase(); + + // Return both the exact hostname and common variations + const domains = [hostname]; + + // For github.com, add api and raw content domain variations + if (hostname === "github.com") { + domains.push("api.github.com"); + domains.push("raw.githubusercontent.com"); + domains.push("*.githubusercontent.com"); + } + // For custom GitHub Enterprise domains, add api. prefix and raw content variations + else if (!hostname.startsWith("api.")) { + domains.push("api." + hostname); + // For GitHub Enterprise, raw content is typically served from raw.hostname + domains.push("raw." + hostname); + } + + return domains; + } catch (e) { + // Invalid URL, return empty array + return []; + } +} + +/** + * Core sanitization function without mention filtering + * @param {string} content - The content to sanitize + * @param {number} [maxLength] - Maximum length of content (default: 524288) + * @returns {string} The sanitized content + */ +function sanitizeContentCore(content, maxLength) { + if (!content || typeof content !== "string") { + return ""; + } + + // Read allowed domains from environment variable + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + + // Extract and add GitHub domains from GitHub context URLs + // This handles GitHub Enterprise deployments with custom domains + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + + // Remove duplicates + allowedDomains = [...new Set(allowedDomains)]; + + let sanitized = content; + + // Neutralize commands at the start of text (e.g., /bot-name) + sanitized = neutralizeCommands(sanitized); + + // Neutralize ALL @mentions (no filtering in core version) + sanitized = neutralizeAllMentions(sanitized); + + // Remove XML comments first + sanitized = removeXmlComments(sanitized); + + // Convert XML tags to parentheses format to prevent injection + sanitized = convertXmlTags(sanitized); + + // Remove ANSI escape sequences + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + + // Remove control characters (except newlines and tabs) + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + + // URI filtering - replace non-https protocols with "(redacted)" + sanitized = sanitizeUrlProtocols(sanitized); + + // Domain filtering for HTTPS URIs + sanitized = sanitizeUrlDomains(sanitized, allowedDomains); + + // Check line count before length to provide more specific truncation message + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + + // If content has too many lines, truncate by lines (primary limit) + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + + // If still too long after line truncation, shorten but keep the line count message + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + + // Neutralize common bot trigger phrases + sanitized = neutralizeBotTriggers(sanitized); + + // Trim excessive whitespace + return sanitized.trim(); + + /** + * Remove unknown domains + * @param {string} s - The string to process + * @param {string[]} allowed - List of allowed domains + * @returns {string} The string with unknown domains redacted + */ + function sanitizeUrlDomains(s, allowed) { + // Match HTTPS URLs with optional port and path + // This regex is designed to: + // 1. Match https:// URIs with explicit protocol + // 2. Capture the hostname/domain + // 3. Allow optional port (:8080) + // 4. Allow optional path and query string + const httpsUrlRegex = /https:\/\/([\w.-]+(?::\d+)?)(\/[^\s]*)?/gi; + + return s.replace(httpsUrlRegex, (match, hostnameWithPort, pathPart) => { + // Extract just the hostname (remove port if present) + const hostname = hostnameWithPort.split(":")[0].toLowerCase(); + pathPart = pathPart || ""; + + // Check if domain is in the allowed list or is a subdomain of an allowed domain + const isAllowed = allowed.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + + // Exact match + if (hostname === normalizedAllowed) { + return true; + } + + // Wildcard match (*.example.com matches subdomain.example.com) + if (normalizedAllowed.startsWith("*.")) { + const baseDomain = normalizedAllowed.substring(2); // Remove *. + return hostname.endsWith("." + baseDomain) || hostname === baseDomain; + } + + // Subdomain match (example.com matches subdomain.example.com) + return hostname.endsWith("." + normalizedAllowed); + }); + + if (isAllowed) { + return match; // Keep the full URL as-is + } else { + // Redact the domain but preserve the protocol and structure for debugging + const truncated = hostname.length > 12 ? hostname.substring(0, 12) + "..." : hostname; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(hostname); + return "(redacted)"; + } + }); + } + + /** + * Sanitize URL protocols - replace non-https with (redacted) + * @param {string} s - The string to process + * @returns {string} The string with non-https protocols redacted + */ + function sanitizeUrlProtocols(s) { + // Match common non-https protocols + // This regex matches: protocol://domain or protocol:path + // Examples: http://, ftp://, file://, data:, javascript:, mailto:, tel:, ssh://, git:// + return s.replace( + /\b((?:http|ftp|file|ssh|git):\/\/([\w.-]+)(?:[^\s]*)|(?:data|javascript|vbscript|about|mailto|tel):[^\s]+)/gi, + (match, _fullMatch, domain) => { + // Extract domain for http/ftp/file/ssh/git protocols + if (domain) { + const domainLower = domain.toLowerCase(); + const truncated = domainLower.length > 12 ? domainLower.substring(0, 12) + "..." : domainLower; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${truncated}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(domainLower); + } else { + // For other protocols (data:, javascript:, etc.), track the protocol itself + const protocolMatch = match.match(/^([^:]+):/); + if (protocolMatch) { + const protocol = protocolMatch[1] + ":"; + if (typeof core !== "undefined" && core.info) { + core.info(`Redacted URL: ${protocol}`); + } + if (typeof core !== "undefined" && core.debug) { + core.debug(`Redacted URL (full): ${match}`); + } + addRedactedDomain(protocol); + } + } + return "(redacted)"; + } + ); + } + + /** + * Neutralizes commands at the start of text by wrapping them in backticks + * @param {string} s - The string to process + * @returns {string} The string with neutralized commands + */ + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + + // Escape special regex characters in command name + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + + // Neutralize /command at the start of text (with optional leading whitespace) + // Only match at the start of the string or after leading whitespace + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + + /** + * Neutralizes ALL @mentions by wrapping them in backticks + * This is the core version without any filtering + * @param {string} s - The string to process + * @returns {string} The string with neutralized mentions + */ + function neutralizeAllMentions(s) { + // Replace @name or @org/team outside code with `@name` + // No filtering - all mentions are neutralized + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (m, p1, p2) => { + // Log when a mention is escaped to help debug issues + if (typeof core !== "undefined" && core.info) { + core.info(`Escaped mention: @${p2} (not in allowed list)`); + } + return `${p1}\`@${p2}\``; + }); + } + + /** + * Removes XML comments from content + * @param {string} s - The string to process + * @returns {string} The string with XML comments removed + */ + function removeXmlComments(s) { + // Remove and malformed + return s.replace(//g, "").replace(//g, ""); + } + + /** + * Converts XML/HTML tags to parentheses format to prevent injection + * @param {string} s - The string to process + * @returns {string} The string with XML tags converted to parentheses + */ + function convertXmlTags(s) { + // Allow safe HTML tags: b, blockquote, br, code, details, em, h1–h6, hr, i, li, ol, p, pre, strong, sub, summary, sup, table, tbody, td, th, thead, tr, ul + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + + // First, process CDATA sections specially - convert tags inside them and the CDATA markers + s = s.replace(//g, (match, content) => { + // Convert tags inside CDATA content + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + // Return with CDATA markers also converted to parentheses + return `(![CDATA[${convertedContent}]])`; + }); + + // Convert opening tags: or to (tag) or (tag attr="value") + // Convert closing tags: to (/tag) + // Convert self-closing tags: or to (tag/) or (tag /) + // But preserve allowed safe tags + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + // Extract tag name from the content (handle closing tags and attributes) + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; // Preserve allowed tags + } + } + return `(${tagContent})`; // Convert other tags to parentheses + }); + } + + /** + * Neutralizes bot trigger phrases by wrapping them in backticks + * @param {string} s - The string to process + * @returns {string} The string with neutralized bot triggers + */ + function neutralizeBotTriggers(s) { + // Neutralize common bot trigger phrases like "fixes #123", "closes #asdfs", etc. + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } +} + +module.exports = { + sanitizeContentCore, + getRedactedDomains, + addRedactedDomain, + clearRedactedDomains, + writeRedactedDomainsLog, + extractDomainsFromUrl, +}; diff --git a/pkg/workflow/js/sanitize_content_old.cjs b/pkg/workflow/js/sanitize_content_old.cjs new file mode 100644 index 00000000000..b8cab1dd33e --- /dev/null +++ b/pkg/workflow/js/sanitize_content_old.cjs @@ -0,0 +1,434 @@ +// @ts-check +/** + * Shared sanitization utilities for GitHub Actions output + * This module provides functions for sanitizing content to prevent security issues + * and unintended side effects in GitHub Actions workflows. + */ + +/** + * Module-level set to collect redacted URL domains across sanitization calls. + * @type {string[]} + */ +const redactedDomains = []; + +/** + * Gets the list of redacted URL domains collected during sanitization. + * @returns {string[]} Array of redacted domain strings + */ +function getRedactedDomains() { + return [...redactedDomains]; +} + +/** + * Clears the list of redacted URL domains. + * Useful for testing or resetting state between operations. + */ +function clearRedactedDomains() { + redactedDomains.length = 0; +} + +/** + * Writes the collected redacted URL domains to a log file. + * Only creates the file if there are redacted domains. + * @param {string} [filePath] - Path to write the log file. Defaults to /tmp/gh-aw/redacted-urls.log + * @returns {string|null} The file path if written, null if no domains to write + */ +function writeRedactedDomainsLog(filePath) { + if (redactedDomains.length === 0) { + return null; + } + + const fs = require("fs"); + const path = require("path"); + const targetPath = filePath || "/tmp/gh-aw/redacted-urls.log"; + + // Ensure directory exists + const dir = path.dirname(targetPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + // Write domains to file, one per line + fs.writeFileSync(targetPath, redactedDomains.join("\n") + "\n"); + + return targetPath; +} + +/** + * Extract domains from a URL and return an array of domain variations + * @param {string} url - The URL to extract domains from + * @returns {string[]} Array of domain variations + */ +function extractDomainsFromUrl(url) { + if (!url || typeof url !== "string") { + return []; + } + + try { + // Parse the URL + const urlObj = new URL(url); + const hostname = urlObj.hostname.toLowerCase(); + + // Return both the exact hostname and common variations + const domains = [hostname]; + + // For github.com, add api and raw content domain variations + if (hostname === "github.com") { + domains.push("api.github.com"); + domains.push("raw.githubusercontent.com"); + domains.push("*.githubusercontent.com"); + } + // For custom GitHub Enterprise domains, add api. prefix and raw content variations + else if (!hostname.startsWith("api.")) { + domains.push("api." + hostname); + // For GitHub Enterprise, raw content is typically served from raw.hostname + domains.push("raw." + hostname); + } + + return domains; + } catch (e) { + // Invalid URL, return empty array + return []; + } +} + +/** + * @typedef {Object} SanitizeOptions + * @property {number} [maxLength] - Maximum length of content (default: 524288) + * @property {string[]} [allowedAliases] - List of aliases (@mentions) that should not be neutralized + */ + +/** + * Sanitizes content for safe output in GitHub Actions + * @param {string} content - The content to sanitize + * @param {number | SanitizeOptions} [maxLengthOrOptions] - Maximum length of content (default: 524288) or options object + * @returns {string} The sanitized content + */ +function sanitizeContent(content, maxLengthOrOptions) { + // Handle both old signature (maxLength) and new signature (options object) + /** @type {number | undefined} */ + let maxLength; + /** @type {string[]} */ + let allowedAliasesLowercase = []; + + if (typeof maxLengthOrOptions === "number") { + maxLength = maxLengthOrOptions; + } else if (maxLengthOrOptions && typeof maxLengthOrOptions === "object") { + maxLength = maxLengthOrOptions.maxLength; + // Pre-process allowed aliases to lowercase for efficient comparison + allowedAliasesLowercase = (maxLengthOrOptions.allowedAliases || []).map(alias => alias.toLowerCase()); + } + if (!content || typeof content !== "string") { + return ""; + } + + // Read allowed domains from environment variable + const allowedDomainsEnv = process.env.GH_AW_ALLOWED_DOMAINS; + const defaultAllowedDomains = ["github.com", "github.io", "githubusercontent.com", "githubassets.com", "github.dev", "codespaces.new"]; + + let allowedDomains = allowedDomainsEnv + ? allowedDomainsEnv + .split(",") + .map(d => d.trim()) + .filter(d => d) + : defaultAllowedDomains; + + // Extract and add GitHub domains from GitHub context URLs + // This handles GitHub Enterprise deployments with custom domains + const githubServerUrl = process.env.GITHUB_SERVER_URL; + const githubApiUrl = process.env.GITHUB_API_URL; + + if (githubServerUrl) { + const serverDomains = extractDomainsFromUrl(githubServerUrl); + allowedDomains = allowedDomains.concat(serverDomains); + } + + if (githubApiUrl) { + const apiDomains = extractDomainsFromUrl(githubApiUrl); + allowedDomains = allowedDomains.concat(apiDomains); + } + + // Remove duplicates + allowedDomains = [...new Set(allowedDomains)]; + + let sanitized = content; + + // Neutralize commands at the start of text (e.g., /bot-name) + sanitized = neutralizeCommands(sanitized); + + // Neutralize @mentions to prevent unintended notifications + sanitized = neutralizeMentions(sanitized); + + // Remove XML comments first + sanitized = removeXmlComments(sanitized); + + // Convert XML tags to parentheses format to prevent injection + sanitized = convertXmlTags(sanitized); + + // Remove ANSI escape sequences + sanitized = sanitized.replace(/\x1b\[[0-9;]*[mGKH]/g, ""); + + // Remove control characters (except newlines and tabs) + sanitized = sanitized.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + + // URI filtering - replace non-https protocols with "(redacted)" + sanitized = sanitizeUrlProtocols(sanitized); + + // Domain filtering for HTTPS URIs + sanitized = sanitizeUrlDomains(sanitized); + + // Check line count before length to provide more specific truncation message + const lines = sanitized.split("\n"); + const maxLines = 65000; + maxLength = maxLength || 524288; + + // If content has too many lines, truncate by lines (primary limit) + if (lines.length > maxLines) { + const truncationMsg = "\n[Content truncated due to line count]"; + const truncatedLines = lines.slice(0, maxLines).join("\n") + truncationMsg; + + // If still too long after line truncation, shorten but keep the line count message + if (truncatedLines.length > maxLength) { + sanitized = truncatedLines.substring(0, maxLength - truncationMsg.length) + truncationMsg; + } else { + sanitized = truncatedLines; + } + } else if (sanitized.length > maxLength) { + sanitized = sanitized.substring(0, maxLength) + "\n[Content truncated due to length]"; + } + + // Neutralize common bot trigger phrases + sanitized = neutralizeBotTriggers(sanitized); + + // Trim excessive whitespace + return sanitized.trim(); + + /** + * Remove unknown domains + * @param {string} s - The string to process + * @returns {string} The string with unknown domains redacted + */ + function sanitizeUrlDomains(s) { + // First pass: match all HTTPS URLs and process them + // We need to handle URLs that might contain other URLs in query parameters + s = s.replace(/\bhttps:\/\/([^\s\])}'"<>&\x00-\x1f,;]+)/gi, (match, rest) => { + // Extract the hostname part (before first slash, colon, or other delimiter) + const hostname = rest.split(/[\/:\?#]/)[0].toLowerCase(); + + // Check if this domain or any parent domain is in the allowlist + const isAllowed = allowedDomains.some(allowedDomain => { + const normalizedAllowed = allowedDomain.toLowerCase(); + return hostname === normalizedAllowed || hostname.endsWith("." + normalizedAllowed); + }); + + if (isAllowed) { + return match; // Keep allowed URLs as-is + } + + // Log the redaction and collect the domain + const domain = hostname; + const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; + core.info(`Redacted URL: ${truncated}`); + core.debug(`Redacted URL (full): ${match}`); + redactedDomains.push(domain); + + // For disallowed URLs, check if there are any allowed URLs in the query/fragment + // and preserve those while redacting the main URL + const urlParts = match.split(/([?&#])/); + let result = "(redacted)"; // Redact the main domain + + // Process query/fragment parts to preserve any allowed URLs within them + for (let i = 1; i < urlParts.length; i++) { + if (urlParts[i].match(/^[?&#]$/)) { + result += urlParts[i]; // Keep separators + } else { + // Recursively process this part to preserve any allowed URLs + result += sanitizeUrlDomains(urlParts[i]); + } + } + + return result; + }); + + return s; + } + + /** + * Remove unknown protocols except https + * @param {string} s - The string to process + * @returns {string} The string with non-https protocols redacted + */ + function sanitizeUrlProtocols(s) { + // Match protocol patterns but avoid command-line flags, file paths, and namespaces + // Protocol patterns typically have :// or are well-known schemes followed by : + // Use negative lookbehind to exclude patterns preceded by - (command flags) + // Match only patterns that look like actual protocols + return s.replace(/(?&\x00-\x1f]+/g, (match, protocol) => { + // Allow https (case insensitive), redact everything else + // But only if it looks like a URL (has :// or is followed by non-colon content) + if (protocol.toLowerCase() === "https") { + return match; + } + + // Allow if it looks like a file path or namespace (::) + if (match.includes("::")) { + return match; + } + + // Redact if it has :// (definite protocol) + if (match.includes("://")) { + // Log the redaction and collect the domain + // Extract domain from URL + const domainMatch = match.match(/^[^:]+:\/\/([^\/\s?#]+)/); + const domain = domainMatch ? domainMatch[1] : match; + const truncated = domain.length > 12 ? domain.substring(0, 12) + "..." : domain; + core.info(`Redacted URL: ${truncated}`); + core.debug(`Redacted URL (full): ${match}`); + redactedDomains.push(domain); + return "(redacted)"; + } + + // Redact well-known dangerous protocols like javascript:, data:, etc. + const dangerousProtocols = ["javascript", "data", "vbscript", "file", "about", "mailto", "tel", "ssh", "ftp"]; + if (dangerousProtocols.includes(protocol.toLowerCase())) { + // Log the redaction and collect the domain + // For dangerous protocols without ://, show protocol and beginning of content + const truncated = match.length > 12 ? match.substring(0, 12) + "..." : match; + core.info(`Redacted URL: ${truncated}`); + core.debug(`Redacted URL (full): ${match}`); + redactedDomains.push(protocol + ":"); + return "(redacted)"; + } + + // Otherwise preserve (could be file:path, namespace:thing, etc.) + return match; + }); + } + + /** + * Neutralizes commands at the start of text by wrapping them in backticks + * @param {string} s - The string to process + * @returns {string} The string with neutralized commands + */ + function neutralizeCommands(s) { + const commandName = process.env.GH_AW_COMMAND; + if (!commandName) { + return s; + } + + // Escape special regex characters in command name + const escapedCommand = commandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + + // Neutralize /command at the start of text (with optional leading whitespace) + // Only match at the start of the string or after leading whitespace + return s.replace(new RegExp(`^(\\s*)/(${escapedCommand})\\b`, "i"), "$1`/$2`"); + } + + /** + * Neutralizes @mentions by wrapping them in backticks + * Skips mentions that are in the allowedAliases list + * @param {string} s - The string to process + * @returns {string} The string with neutralized mentions + */ + function neutralizeMentions(s) { + // Replace @name or @org/team outside code with `@name` + // Skip mentions that are in the allowed aliases list + return s.replace(/(^|[^\w`])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\/[A-Za-z0-9._-]+)?)/g, (_m, p1, p2) => { + // Check if this mention is in the allowed aliases list (case-insensitive) + // allowedAliasesLowercase is pre-processed to lowercase for efficient comparison + const isAllowed = allowedAliasesLowercase.includes(p2.toLowerCase()); + if (isAllowed) { + return `${p1}@${p2}`; // Keep the original mention + } + // Log when a mention is escaped to help debug issues + core.info(`Escaped mention: @${p2} (not in allowed list)`); + return `${p1}\`@${p2}\``; // Neutralize the mention + }); + } + + /** + * Removes XML comments from content + * @param {string} s - The string to process + * @returns {string} The string with XML comments removed + */ + function removeXmlComments(s) { + // Remove and malformed + return s.replace(//g, "").replace(//g, ""); + } + + /** + * Converts XML/HTML tags to parentheses format to prevent injection + * @param {string} s - The string to process + * @returns {string} The string with XML tags converted to parentheses + */ + function convertXmlTags(s) { + // Allow safe HTML tags: b, blockquote, br, code, details, em, h1–h6, hr, i, li, ol, p, pre, strong, sub, summary, sup, table, tbody, td, th, thead, tr, ul + const allowedTags = [ + "b", + "blockquote", + "br", + "code", + "details", + "em", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "li", + "ol", + "p", + "pre", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", + ]; + + // First, process CDATA sections specially - convert tags inside them and the CDATA markers + s = s.replace(//g, (match, content) => { + // Convert tags inside CDATA content + const convertedContent = content.replace(/<(\/?[A-Za-z][A-Za-z0-9]*(?:[^>]*?))>/g, "($1)"); + // Return with CDATA markers also converted to parentheses + return `(![CDATA[${convertedContent}]])`; + }); + + // Convert opening tags: or to (tag) or (tag attr="value") + // Convert closing tags: to (/tag) + // Convert self-closing tags: or to (tag/) or (tag /) + // But preserve allowed safe tags + return s.replace(/<(\/?[A-Za-z!][^>]*?)>/g, (match, tagContent) => { + // Extract tag name from the content (handle closing tags and attributes) + const tagNameMatch = tagContent.match(/^\/?\s*([A-Za-z][A-Za-z0-9]*)/); + if (tagNameMatch) { + const tagName = tagNameMatch[1].toLowerCase(); + if (allowedTags.includes(tagName)) { + return match; // Preserve allowed tags + } + } + return `(${tagContent})`; // Convert other tags to parentheses + }); + } + + /** + * Neutralizes bot trigger phrases by wrapping them in backticks + * @param {string} s - The string to process + * @returns {string} The string with neutralized bot triggers + */ + function neutralizeBotTriggers(s) { + // Neutralize common bot trigger phrases like "fixes #123", "closes #asdfs", etc. + return s.replace(/\b(fixes?|closes?|resolves?|fix|close|resolve)\s+#(\w+)/gi, (match, action, ref) => `\`${action} #${ref}\``); + } +} + +module.exports = { sanitizeContent, getRedactedDomains, clearRedactedDomains, writeRedactedDomainsLog }; diff --git a/pkg/workflow/js/sanitize_incoming_text.cjs b/pkg/workflow/js/sanitize_incoming_text.cjs new file mode 100644 index 00000000000..0874ed4a20c --- /dev/null +++ b/pkg/workflow/js/sanitize_incoming_text.cjs @@ -0,0 +1,27 @@ +// @ts-check +/** + * Slimmed-down sanitization for incoming text (compute_text) + * This version does NOT include mention filtering - all @mentions are escaped + */ + +const { sanitizeContentCore, writeRedactedDomainsLog } = require("./sanitize_content_core.cjs"); + +/** + * Sanitizes incoming text content without selective mention filtering + * All @mentions are escaped to prevent unintended notifications + * + * Uses the core sanitization functions directly to minimize bundle size. + * + * @param {string} content - The content to sanitize + * @param {number} [maxLength] - Maximum length of content (default: 524288) + * @returns {string} The sanitized content with all mentions escaped + */ +function sanitizeIncomingText(content, maxLength) { + // Call core sanitization which neutralizes all mentions + return sanitizeContentCore(content, maxLength); +} + +module.exports = { + sanitizeIncomingText, + writeRedactedDomainsLog, +}; diff --git a/pkg/workflow/scripts.go b/pkg/workflow/scripts.go index 4d38fcca3c8..55f4f471f29 100644 --- a/pkg/workflow/scripts.go +++ b/pkg/workflow/scripts.go @@ -137,6 +137,17 @@ var mcpHTTPTransportScriptSource string //go:embed js/substitute_placeholders.cjs var substitutePlaceholdersScriptSource string +// Helper modules for bundling +// +//go:embed js/resolve_mentions_from_payload.cjs +var resolveMentionsFromPayloadScriptSource string + +//go:embed js/sanitize_incoming_text.cjs +var sanitizeIncomingTextScriptSource string + +//go:embed js/sanitize_content_core.cjs +var sanitizeContentCoreScriptSource string + // init registers all scripts with the DefaultScriptRegistry. // Scripts are bundled lazily on first access via the getter functions. func init() { @@ -187,6 +198,11 @@ func init() { // Template substitution scripts DefaultScriptRegistry.Register("substitute_placeholders", substitutePlaceholdersScriptSource) + // Helper modules (for inlining via bundler) + DefaultScriptRegistry.Register("resolve_mentions_from_payload", resolveMentionsFromPayloadScriptSource) + DefaultScriptRegistry.Register("sanitize_incoming_text", sanitizeIncomingTextScriptSource) + DefaultScriptRegistry.Register("sanitize_content_core", sanitizeContentCoreScriptSource) + scriptsLog.Print("Completed script registration") }