anoman
Recipe · Streaming Chat UI

Streaming chat UI with Next.js.

Token-by-token streaming over SSE, cancellation, mid-stream error handling, and the API key stays server-side. ~150 lines of TypeScript.

Architecture

Three pieces

  1. Next.js route handler on the server — proxies the browser request to Anoman so your API key never reaches the client.
  2. useChat hook — manages message state, parses SSE, exposes send() + cancel().
  3. ChatBox component — renders messages + input. Disables input while streaming. Shows a Stop button.

1. Server-side proxy

Pipe Anoman’s stream through to the browser

The route handler calls Anoman, then re-emits each chunk as SSE for the browser. We don’t modify chunks — pass them through verbatim so the _anoman metadata frame reaches the client unchanged.

// Proxies the browser to Anoman so the API key stays server-side.
// Pipes the SSE response straight through.

import OpenAI from "openai";

export const runtime = "edge";  // optional — faster cold start

const client = new OpenAI({
  baseURL: "https://api.anoman.io/v1",
  apiKey: process.env.ANOMAN_API_KEY!,
});

export async function POST(req: Request) {
  const { messages, model = "claude-sonnet-4-6" } = await req.json();

  const stream = await client.chat.completions.create({
    model,
    messages,
    stream: true,
    // Surface guardrail metadata to the client via the _anoman block
    // already present on the final chunk. No special header needed.
  });

  // The OpenAI SDK returns an async iterator over typed chunks.
  // Re-encode as plain SSE for the browser EventSource client.
  const encoder = new TextEncoder();
  const body = new ReadableStream({
    async start(controller) {
      try {
        for await (const chunk of stream) {
          const line = `data: ${JSON.stringify(chunk)}\n\n`;
          controller.enqueue(encoder.encode(line));
        }
        controller.enqueue(encoder.encode("data: [DONE]\n\n"));
      } catch (err: any) {
        const errFrame = {
          error: {
            type: "provider_error",
            message: err.message ?? "stream interrupted",
          },
        };
        controller.enqueue(
          encoder.encode(`data: ${JSON.stringify(errFrame)}\n\n`),
        );
      } finally {
        controller.close();
      }
    },
  });

  return new Response(body, {
    headers: {
      "Content-Type": "text/event-stream",
      "Cache-Control": "no-cache",
      Connection: "keep-alive",
    },
  });
}

2. SSE parser hook

Append deltas to the last message

Persistent buffer for chunk-boundary tolerance. Mid-stream error frame is thrown so the catch block surfaces it. AbortController for cancellation.

import { useCallback, useState, useRef } from "react";

type Message = { role: "user" | "assistant"; content: string };

export function useChat() {
  const [messages, setMessages] = useState<Message[]>([]);
  const [streaming, setStreaming] = useState(false);
  const [error, setError] = useState<string | null>(null);
  const abortRef = useRef<AbortController | null>(null);

  const send = useCallback(async (content: string) => {
    setError(null);
    const userMsg: Message = { role: "user", content };
    const assistantMsg: Message = { role: "assistant", content: "" };
    setMessages((prev) => [...prev, userMsg, assistantMsg]);
    setStreaming(true);

    const controller = new AbortController();
    abortRef.current = controller;

    try {
      const response = await fetch("/api/chat", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ messages: [...messages, userMsg] }),
        signal: controller.signal,
      });

      if (!response.ok || !response.body) {
        throw new Error(`HTTP ${response.status}`);
      }

      const reader = response.body.getReader();
      const decoder = new TextDecoder();
      let buffer = "";

      while (true) {
        const { done, value } = await reader.read();
        if (done) break;
        buffer += decoder.decode(value, { stream: true });

        const lines = buffer.split("\n");
        buffer = lines.pop() ?? "";

        for (const line of lines) {
          if (!line.startsWith("data: ")) continue;
          const payload = line.slice(6).trim();
          if (payload === "[DONE]") continue;
          let parsed: any;
          try {
            parsed = JSON.parse(payload);
          } catch {
            continue;
          }
          // Mid-stream error frame
          if (parsed.error) {
            throw new Error(parsed.error.message);
          }
          const delta = parsed.choices?.[0]?.delta?.content;
          if (delta) {
            setMessages((prev) => {
              const next = [...prev];
              next[next.length - 1] = {
                ...next[next.length - 1],
                content: next[next.length - 1].content + delta,
              };
              return next;
            });
          }
        }
      }
    } catch (e) {
      if ((e as Error).name !== "AbortError") {
        setError((e as Error).message);
      }
    } finally {
      setStreaming(false);
      abortRef.current = null;
    }
  }, [messages]);

  const cancel = useCallback(() => {
    abortRef.current?.abort();
  }, []);

  return { messages, streaming, error, send, cancel };
}

3. UI component

Stop button while streaming, Send when idle

"use client";
import { useState } from "react";
import { useChat } from "@/hooks/useChat";

export function ChatBox() {
  const { messages, streaming, error, send, cancel } = useChat();
  const [input, setInput] = useState("");

  return (
    <div className="flex flex-col h-screen max-w-3xl mx-auto">
      <div className="flex-1 overflow-y-auto p-4 space-y-3">
        {messages.map((m, i) => (
          <div
            key={i}
            className={`p-3 rounded-lg ${
              m.role === "user" ? "bg-blue-50 ml-12" : "bg-gray-50 mr-12"
            }`}
          >
            <div className="text-xs font-semibold text-gray-500 mb-1">
              {m.role === "user" ? "You" : "Assistant"}
            </div>
            <div className="whitespace-pre-wrap">{m.content}</div>
          </div>
        ))}
        {error && (
          <div className="p-3 rounded-lg bg-red-50 text-red-700 text-sm">
            Error: {error}
          </div>
        )}
      </div>

      <form
        onSubmit={(e) => {
          e.preventDefault();
          if (input.trim()) {
            send(input);
            setInput("");
          }
        }}
        className="flex gap-2 p-4 border-t"
      >
        <input
          className="flex-1 border rounded-lg px-3 py-2"
          value={input}
          onChange={(e) => setInput(e.target.value)}
          disabled={streaming}
          placeholder="Ask anything..."
        />
        {streaming ? (
          <button
            type="button"
            onClick={cancel}
            className="px-4 py-2 bg-red-600 text-white rounded-lg"
          >
            Stop
          </button>
        ) : (
          <button
            type="submit"
            className="px-4 py-2 bg-blue-600 text-white rounded-lg"
          >
            Send
          </button>
        )}
      </form>
    </div>
  );
}

Production tips

  • Cloudflare proxy compatibility — Anoman’s edge forwards SSE frames as they arrive. Each chunk resets the proxy timeout, so streaming responses survive arbitrarily long completions.
  • Show guardrail status inline — the final _anoman frame has the full guardrail breakdown. Render a small “Verified clean ✓” badge under each assistant message.
  • Rate limit handling — when your route handler hits 429 from Anoman, surface a friendly “Slow down” message to the user. The header retry-after tells you the wait.
  • Multi-turn context — pass the full messages array on every request. Anoman + the upstream handle the rest.
  • Persistent sessions — add anoman-session-id header to group all turns under one trace in the dashboard Sessions view.

See the playground for a full reference UI.

3-pane streaming compare with model picker + parameter controls.

Open playgroundMore recipes