anoman
Sign InGet API Key
Documentation home

GitHub PR review bot.

Claude Sonnet 4.6 + tool calling + a GitHub webhook. Posts inline comments with severity buckets. ~250 lines of Python.

Architecture — What we're building

  1. GitHub fires a pull_request webhook on PR open / sync.
  2. Our FastAPI handler verifies the GitHub signature + kicks off a background review.
  3. The review loop calls Claude Sonnet with the PR diff + two tools: read_file and post_review_comment.
  4. The model loops: read files, think, post comments, repeat until it's satisfied (or hits the 15-step cap).
  5. Anoman's guardrails inspect every prompt + response. Tool policy enforces that the bot can only call our two declared tools.

1. Tool definitions — What the bot can do

Two tools. The schemas are sent to the model on every call. Set tool_choice: "auto" and let it decide.

Tool definitions

# tools.py — what the model can do
TOOLS = [
    {
        "type": "function",
        "function": {
            "name": "read_file",
            "description": "Read a file from the PR being reviewed.",
            "parameters": {
                "type": "object",
                "properties": {
                    "path": {"type": "string", "description": "Path relative to repo root."},
                    "start_line": {"type": "integer"},
                    "end_line":   {"type": "integer"},
                },
                "required": ["path"],
            },
        },
    },
    {
        "type": "function",
        "function": {
            "name": "post_review_comment",
            "description": "Post an inline comment on a specific line of a file in the PR.",
            "parameters": {
                "type": "object",
                "properties": {
                    "path":     {"type": "string"},
                    "line":     {"type": "integer"},
                    "severity": {"type": "string", "enum": ["nit", "concern", "blocker"]},
                    "body":     {"type": "string"},
                },
                "required": ["path", "line", "severity", "body"],
            },
        },
    },
]

2. Review loop — Agent loop with a step cap

Standard tool-using agent loop: keep calling until the model returns text without tool calls (or we hit the step cap to bound cost).

Review loop

# review_bot.py — main agentic loop
import os
from openai import OpenAI
from tools import TOOLS
from impl import read_file_impl, post_review_comment_impl
 
client = OpenAI(
    base_url="https://api.anoman.io/v1",
    api_key=os.environ["ANOMAN_API_KEY"],
)
 
SYSTEM = """
You are a careful code reviewer. Given a PR diff, you can:
  - Read full file contents via the read_file tool
  - Post inline review comments via the post_review_comment tool
 
Rules:
  - Read the surrounding code before commenting on a change
  - Use severity "nit" for style, "concern" for likely bugs,
    "blocker" for security or correctness issues
  - Don't comment on whitespace or auto-formatter output
  - At most 10 comments per PR
"""
 
def review_pr(pr_diff: str, pr_metadata: dict) -> dict:
    """Run the review loop. Returns counts of comments posted by severity."""
    messages = [
        {"role": "system", "content": SYSTEM},
        {"role": "user",   "content": f"PR diff:\n\n{pr_diff}"},
    ]
    posted = {"nit": 0, "concern": 0, "blocker": 0}
 
    for step in range(15):  # cap loop length
        response = client.chat.completions.create(
            model="claude-sonnet-4-6",
            messages=messages,
            tools=TOOLS,
            tool_choice="auto",
            max_tokens=2000,
            metadata={"pr_id": pr_metadata["pr_id"]},
        )
 
        msg = response.choices[0].message
 
        # Bot replied with text but no tool — wrap up
        if not msg.tool_calls:
            print(msg.content)
            break
 
        # Execute every tool the model called this turn
        messages.append({"role": "assistant", "content": msg.content, "tool_calls": msg.tool_calls})
        for call in msg.tool_calls:
            name = call.function.name
            args = json.loads(call.function.arguments)
            if name == "read_file":
                result = read_file_impl(pr_metadata["repo"], args)
            elif name == "post_review_comment":
                result = post_review_comment_impl(pr_metadata, args)
                posted[args["severity"]] += 1
            else:
                result = {"error": f"unknown tool {name}"}
 
            messages.append({
                "role": "tool",
                "tool_call_id": call.id,
                "content": json.dumps(result),
            })
 
    return posted

3. GitHub webhook entrypoint — Fire the bot from a webhook

GitHub webhook entrypoint (FastAPI)

# webhook.py — fire the bot when a PR is opened
from fastapi import FastAPI, Request, HTTPException, BackgroundTasks
import hmac, hashlib, os, json
 
from review_bot import review_pr
from impl import fetch_pr_diff
 
app = FastAPI()
GITHUB_SECRET = os.environ["GITHUB_WEBHOOK_SECRET"].encode()
 
@app.post("/webhook/github")
async def github_webhook(request: Request, bg: BackgroundTasks):
    raw = await request.body()
    sig = request.headers.get("x-hub-signature-256", "")
    expected = "sha256=" + hmac.new(GITHUB_SECRET, raw, hashlib.sha256).hexdigest()
    if not hmac.compare_digest(sig, expected):
        raise HTTPException(401, "invalid signature")
 
    event = request.headers.get("x-github-event")
    payload = json.loads(raw)
 
    if event == "pull_request" and payload["action"] in {"opened", "synchronize"}:
        pr = payload["pull_request"]
        bg.add_task(
            run_review,
            pr_diff=fetch_pr_diff(pr),
            pr_metadata={
                "pr_id":   pr["id"],
                "repo":    pr["base"]["repo"]["full_name"],
                "number":  pr["number"],
                "head_sha": pr["head"]["sha"],
            },
        )
 
    return {"ok": True}
 
def run_review(pr_diff, pr_metadata):
    try:
        counts = review_pr(pr_diff, pr_metadata)
        print(f"[review] PR #{pr_metadata['number']}: {counts}")
    except Exception as e:
        # Anoman returned 403 / 429 / 500. Don't crash the worker.
        print(f"[review] failed: {e}")

Same signature pattern as Anoman's own webhooks — see /docs/webhooks for the verification recipe.

4. Production hardening — Handle guardrail blocks + rate limits

Don't retry on guardrail blocks (they'll fail again the same way). Do retry on 503/504 with exponential backoff. Stop fast on budget exceeded.

Robust call with retry + guardrail awareness

# Wrap each completion in retry-with-backoff that respects
# Anoman-specific error codes.
from openai import APIStatusError, RateLimitError
 
def safe_completion(client, **kwargs):
    for attempt in range(5):
        try:
            return client.chat.completions.create(**kwargs)
        except RateLimitError as e:
            wait = float(e.response.headers.get("retry-after", 2 ** attempt))
            time.sleep(wait)
        except APIStatusError as e:
            err = e.body.get("error", {}) if isinstance(e.body, dict) else {}
            code = err.get("code", "")
 
            if code in {"prompt_injection", "tool_denied", "content_violation"}:
                # Guardrail block — don't retry. Log the trace + skip this PR.
                print(f"[guardrails] {code}: {err.get('message')}")
                raise
 
            if code == "budget_exceeded":
                # Out of money. Don't keep firing.
                print("[budget] monthly budget exhausted")
                raise
 
            # Provider error — retryable
            if e.status_code in {503, 504}:
                time.sleep(2 ** attempt)
                continue
 
            raise
 
    raise RuntimeError("max retries")

Production deploy — Real-world tips

  • Per-key budget — set a monthly USD cap on your bot's Anoman key. Runaway PRs hit 402 instead of draining you.
  • Policy group for tools — create a dedicated policy group with read_file + post_review_comment in the allowlist. Now if Anthropic ever returns an unexpected tool, it's blocked.
  • Agent ID tag — pass anoman-agent-id: code-reviewer header so the dashboard groups all calls under one agent.
  • Vision for screenshots — extend to call gpt-4o on PRs containing image attachments — useful for design-system PRs.

More recipes for agentic patterns.

Streaming chat, batch document analysis, RAG with caching.