|
| 1 | +export const meta = { |
| 2 | + name: "pr-stamp-sweep", |
| 3 | + description: |
| 4 | + "Review candidate PRs for stampability, then adversarially verify security of stamp candidates", |
| 5 | + whenToUse: |
| 6 | + "Sweep candidate PRs for stampability: per-PR review + adversarial security verify. Requires pre-fetched PR dossiers in /tmp/claude/pr-sweep/<n>.md and args {prs: [...]}.", |
| 7 | + phases: [ |
| 8 | + { title: "Review", detail: "one reviewer agent per PR" }, |
| 9 | + { |
| 10 | + title: "Verify", |
| 11 | + detail: "adversarial security skeptic per stamp candidate", |
| 12 | + }, |
| 13 | + ], |
| 14 | +}; |
| 15 | + |
| 16 | +// PRECONDITION: before invoking this workflow, pre-fetch each candidate PR to |
| 17 | +// /tmp/claude/pr-sweep/<n>.md, containing the PR's metadata, body, existing |
| 18 | +// reviews/comments, and the full diff (e.g. via `gh pr view` + `gh pr diff`). |
| 19 | +// Sandboxed agents can't reliably call gh themselves, so they read these |
| 20 | +// dossier files instead. Pass the PR numbers as args: {prs: [<PR numbers>]}. |
| 21 | + |
| 22 | +const REVIEW_SCHEMA = { |
| 23 | + type: "object", |
| 24 | + properties: { |
| 25 | + number: { type: "number" }, |
| 26 | + verdict: { type: "string", enum: ["stamp", "skip", "needs-discussion"] }, |
| 27 | + category: { |
| 28 | + type: "string", |
| 29 | + description: "docs | tests | bugfix | nicety | security-fix | other", |
| 30 | + }, |
| 31 | + summary: { |
| 32 | + type: "string", |
| 33 | + description: "1-2 sentence plain-language summary of what the PR does", |
| 34 | + }, |
| 35 | + reasoning: { |
| 36 | + type: "string", |
| 37 | + description: "why this verdict — correctness, scope, quality", |
| 38 | + }, |
| 39 | + behaviorChange: { |
| 40 | + type: "string", |
| 41 | + description: 'what user-visible behavior changes, or "none"', |
| 42 | + }, |
| 43 | + concerns: { type: "array", items: { type: "string" } }, |
| 44 | + securitySensitive: { |
| 45 | + type: "boolean", |
| 46 | + description: |
| 47 | + "true if it touches auth, sanitization, parsers of untrusted input, actor checks, file restore, or shell construction", |
| 48 | + }, |
| 49 | + duplicateOf: { |
| 50 | + type: "string", |
| 51 | + description: "PR number(s) this duplicates, or empty string", |
| 52 | + }, |
| 53 | + }, |
| 54 | + required: [ |
| 55 | + "number", |
| 56 | + "verdict", |
| 57 | + "category", |
| 58 | + "summary", |
| 59 | + "reasoning", |
| 60 | + "behaviorChange", |
| 61 | + "concerns", |
| 62 | + "securitySensitive", |
| 63 | + "duplicateOf", |
| 64 | + ], |
| 65 | +}; |
| 66 | + |
| 67 | +const VERDICT_SCHEMA = { |
| 68 | + type: "object", |
| 69 | + properties: { |
| 70 | + number: { type: "number" }, |
| 71 | + safeToStamp: { type: "boolean" }, |
| 72 | + findings: { |
| 73 | + type: "array", |
| 74 | + items: { type: "string" }, |
| 75 | + description: |
| 76 | + "concrete security/correctness problems found, empty if clean", |
| 77 | + }, |
| 78 | + confidence: { type: "string", enum: ["high", "medium", "low"] }, |
| 79 | + }, |
| 80 | + required: ["number", "safeToStamp", "findings", "confidence"], |
| 81 | +}; |
| 82 | + |
| 83 | +if (!args || !Array.isArray(args.prs) || args.prs.length === 0) |
| 84 | + throw new Error( |
| 85 | + "pass {prs: [<PR numbers>]} as args; pre-fetch each PR to /tmp/claude/pr-sweep/<n>.md first", |
| 86 | + ); |
| 87 | +const prs = args.prs; |
| 88 | + |
| 89 | +log(`Reviewing ${prs.length} candidate PRs`); |
| 90 | + |
| 91 | +const results = await pipeline( |
| 92 | + prs, |
| 93 | + (n) => |
| 94 | + agent( |
| 95 | + `You are reviewing open PR #${n} on anthropics/claude-code-action to decide if it is safe for a maintainer to approve ("stamp") with minimal further discussion. |
| 96 | +
|
| 97 | +The full PR (metadata, body, existing reviews/comments, and complete diff) is in /tmp/claude/pr-sweep/${n}.md — read it first. The repo is checked out at the current working directory. Read the actual current source files the diff touches to verify the diff applies cleanly conceptually and the claims in the PR body are true. Do NOT modify anything or run git commands that change state. |
| 98 | +
|
| 99 | +Context about this repo: |
| 100 | +- It's a GitHub Action that runs Claude on issues/PRs. It processes UNTRUSTED content (PR bodies, comments, branch names, file contents from forks). Treat any change touching content sanitization, actor/bot allowlists, config restoration, prompt construction, or shell command construction as high-risk. |
| 101 | +- Most candidate PRs are from EXTERNAL contributors. Treat the diff with suspicion: look for subtle malicious changes, weakened validation, injection vectors, overly broad permissions, or changes whose description doesn't match the code. |
| 102 | +- Runtime is Bun; strict TypeScript (noUnusedLocals/noUnusedParameters). Tests are unit tests run with bun test. |
| 103 | +
|
| 104 | +Stamp criteria (ALL must hold): |
| 105 | +1. Small, focused, and the code does exactly what the title/body says. |
| 106 | +2. No major behavior change — bug fixes restoring intended behavior, docs fixes, test-only additions, and small niceties qualify. New inputs/features, behavior redesigns, or large refactors do NOT. |
| 107 | +3. Correct: you verified the logic against the actual current source, not just the diff. Check edge cases. |
| 108 | +4. No security concern. Check explicitly for: prompt injection (untrusted text reaching Claude's prompt without sanitization), code execution (untrusted data reaching shell commands, eval/spawn, or GitHub workflow expressions), path traversal (untrusted input influencing filesystem paths), credential exposure (tokens reaching logs, comments, or attacker-readable output), weakened validation or permission checks, and suspicious hunks unrelated to the stated purpose. |
| 109 | +5. Wouldn't break the public API of base-action/ or action.yml output wiring. |
| 110 | +
|
| 111 | +If the PR is a docs change, verify the docs claims against the actual code behavior. If test-only, check tests actually pass conceptually (assert the right things, match real implementations) and don't weaken or skip anything. |
| 112 | +
|
| 113 | +Verdicts: "stamp" = approve as-is; "needs-discussion" = plausible but has questions/issues worth a comment; "skip" = too big, wrong, redundant, or risky. |
| 114 | +
|
| 115 | +If this PR appears to duplicate another open PR (same fix, same files), still judge it on its own merits but note the duplication in duplicateOf. |
| 116 | +
|
| 117 | +Return structured output only.`, |
| 118 | + { label: `review:#${n}`, phase: "Review", schema: REVIEW_SCHEMA }, |
| 119 | + ), |
| 120 | + (review, n) => { |
| 121 | + if (!review) return null; |
| 122 | + if (review.verdict !== "stamp") return { review, verify: null }; |
| 123 | + return agent( |
| 124 | + `You are an adversarial security skeptic. Another reviewer recommended APPROVING open PR #${n} on anthropics/claude-code-action. Your job is to REFUTE that recommendation — find any reason it should NOT be stamped. |
| 125 | +
|
| 126 | +Their assessment: ${JSON.stringify(review)} |
| 127 | +
|
| 128 | +Read the full PR at /tmp/claude/pr-sweep/${n}.md and the touched source files in the current working directory. This repo processes untrusted PR/issue content from forks; anything that lets untrusted content reach Claude's prompt, a shell command, a workflow expression, or a filesystem path unsanitized is a critical vulnerability. |
| 129 | +
|
| 130 | +Hunt specifically for: |
| 131 | +- Subtle malice or scope creep: hunks that don't match the stated purpose, weakened validation, regex changes that widen acceptance, removed escaping. |
| 132 | +- Prompt injection: untrusted data (comment bodies, branch names, file contents, command output, downloaded files) reaching Claude's prompt or context without sanitization, including indirect routes like tool output Claude later reads. |
| 133 | +- Code execution: untrusted data reaching shell commands, eval/spawn argv, GitHub workflow \${{ }} expressions, or API call templates; new process spawning; path traversal letting untrusted input write or read outside intended directories. |
| 134 | +- Credential exposure: tokens or secrets flowing into logs, posted comments, error messages, env passed to untrusted code, or files Claude can read. |
| 135 | +- Logic errors the first reviewer missed: off-by-one, wrong polarity, unhandled edge cases (empty strings, unicode, very long inputs). |
| 136 | +- Supply-chain angles: pinned versions that don't match the claimed SHA/tag, new dependencies, fetched URLs. |
| 137 | +- For docs PRs: claims that would mislead users into insecure configurations. |
| 138 | +- For test-only PRs: tests that codify wrong behavior, or that would mask future regressions. |
| 139 | +
|
| 140 | +If the diff pins a version/SHA, verify the claim is plausible from local information; flag if unverifiable. Be strict: if uncertain whether something is a real problem, lean toward reporting it as a finding with your uncertainty noted. Only return safeToStamp=true if you genuinely failed to find any disqualifying issue. |
| 141 | +
|
| 142 | +Return structured output only.`, |
| 143 | + { label: `verify:#${n}`, phase: "Verify", schema: VERDICT_SCHEMA }, |
| 144 | + ).then((v) => ({ review, verify: v })); |
| 145 | + }, |
| 146 | +); |
| 147 | + |
| 148 | +const clean = results.filter(Boolean); |
| 149 | +const stamped = clean.filter( |
| 150 | + (r) => r.review.verdict === "stamp" && r.verify && r.verify.safeToStamp, |
| 151 | +); |
| 152 | +const demoted = clean.filter( |
| 153 | + (r) => r.review.verdict === "stamp" && (!r.verify || !r.verify.safeToStamp), |
| 154 | +); |
| 155 | +const discuss = clean.filter((r) => r.review.verdict === "needs-discussion"); |
| 156 | +const skipped = clean.filter((r) => r.review.verdict === "skip"); |
| 157 | + |
| 158 | +log( |
| 159 | + `stamp: ${stamped.length}, demoted by verifier: ${demoted.length}, needs-discussion: ${discuss.length}, skip: ${skipped.length}`, |
| 160 | +); |
| 161 | + |
| 162 | +return { stamped, demoted, discuss, skipped }; |
0 commit comments