Last updated 2026-05-17

Error Handling

llmrouter surfaces errors from five distinct places. Knowing which is which is the difference between a retry loop that rescues a flaky network and a retry loop that hammers a permanently misconfigured endpoint. This page walks the taxonomy, the sentinel and wrapper types, and the patterns for handling each.

Where errors come from

Source Returned from Retryable?
Configuration provider.New(...) No — fix the config
Request build CompletionStream No — fix the request
Network CompletionStream Yes (with care)
Upstream HTTP CompletionStream Depends on status code
Stream read Stream.Err() Depends — usually no

The two functions that produce errors are CompletionStream (synchronous error on open) and Stream.Err() (terminal error after the producer goroutine finishes). Treating those as the only error checkpoints is the simplest mental model: 

stream, err := p.CompletionStream(ctx, req)
if err != nil {
    // configuration, request-build, network, or upstream HTTP error
}
for chunk := range stream.Chunks() {
    // ...
}
if err := stream.Err(); err != nil {
    // network read failure during streaming, or context cancellation
}

ErrInvalidConfig — configuration errors

var ErrInvalidConfig = errors.New("invalid provider config")

A sentinel returned (wrapped) from provider.New when an option fails validation or the provider rejects the final configuration. Match it with errors.Is: 

p, err := openai.New(llmrouter.WithAPIKey(""))
if errors.Is(err, llmrouter.ErrInvalidConfig) {
    log.Fatalf("invalid configuration: %v", err)
}

Causes include empty API key, malformed base URL, negative timeout, and provider-specific requirements (the Anthropic provider requires an API key; future Azure/Bedrock providers will require keys from WithExtra).

These errors are eager: you see them at construction time, not at request time. That's deliberate — config errors should fail fast at startup, not at 3 AM under load.

*ErrUpstream — upstream HTTP errors

type ErrUpstream struct {
    Provider   string
    StatusCode int
    Body       string
}


func (e *ErrUpstream) Error() string {
    return fmt.Sprintf("%s upstream %d: %s", e.Provider, e.StatusCode, e.Body)
}

Returned synchronously from CompletionStream when the upstream responds with status >= 400, before any SSE frames have been read. Match it with errors.As: 

var ue *llmrouter.ErrUpstream
if errors.As(err, &ue) {
    log.Printf("provider=%s status=%d body=%s",
        ue.Provider, ue.StatusCode, ue.Body)
}

The body is truncated for log safety:

  • OpenAI provider — up to 1KB.
  • Anthropic provider — up to 8KB.

Truncation happens because some upstreams return HTML error pages, multi-megabyte stack traces, or other noise on failure. The truncated snippet is enough to debug the typical {"error":{"message":"..."}} case without blowing up your logs.

Distinguishing upstream errors

The StatusCode field is your routing key:

StatusMeaningAction
400 Malformed request, unknown model, content policy Fail — bad input
401 Invalid API key Fail loudly — config issue
403 Key valid but not authorized for this resource Fail — permission issue
404 Model not found, wrong base URL Fail — config issue
408 Request timeout (rare for chat completions) Retry with backoff
429 Rate-limited or quota exhausted Retry with backoff; honor Retry-After if present
500 / 502 / 503 / 504 Upstream server error Retry with backoff (bounded)

A simple classifier:

func isRetryable(err error) bool {
    var ue *llmrouter.ErrUpstream
    if !errors.As(err, &ue) {
        // Not an upstream error — could be network, config, etc.
        return false
    }
    switch ue.StatusCode {
    case 408, 429, 500, 502, 503, 504:
        return true
    default:
        return false
    }
}

Retrying transient errors

The most reliable place to retry is inside the http.RoundTripper — it sits below the streaming layer, so it can retry on the initial response status (when no bytes have been delivered to the caller yet) without re-running an in-progress completion. 

package main


import (
    "errors"
    "fmt"
    "io"
    "math/rand"
    "net/http"
    "strconv"
    "time"


    "github.com/elloloop/llmrouter"
    "github.com/elloloop/llmrouter/providers/openai"
)


// retryTransport retries on 429 and 5xx with exponential backoff
// and jitter. Stops retrying as soon as the body starts streaming —
// at that point the upstream has committed tokens to the response.
type retryTransport struct {
    base        http.RoundTripper
    maxAttempts int
}


func (t *retryTransport) RoundTrip(req *http.Request) (*http.Response, error) {
    for attempt := 1; ; attempt++ {
        resp, err := t.base.RoundTrip(req)
        // Network error: try again unless we're out of attempts.
        if err != nil {
            if attempt >= t.maxAttempts {
                return nil, err
            }
            if !sleep(req, backoff(attempt)) {
                return nil, req.Context().Err()
            }
            continue
        }
        // Don't retry success or non-retryable failures.
        if resp.StatusCode < 400 ||
            (resp.StatusCode != 429 && resp.StatusCode < 500) {
            return resp, nil
        }
        // Retryable error. Drain and close to free the connection.
        _, _ = io.Copy(io.Discard, resp.Body)
        resp.Body.Close()
        if attempt >= t.maxAttempts {
            return nil, fmt.Errorf("after %d attempts: %s",
                attempt, resp.Status)
        }
        wait := retryAfter(resp.Header)
        if wait == 0 {
            wait = backoff(attempt)
        }
        if !sleep(req, wait) {
            return nil, req.Context().Err()
        }
    }
}


func backoff(attempt int) time.Duration {
    base := time.Duration(1<<attempt) * 500 * time.Millisecond
    jitter := time.Duration(rand.Int63n(int64(base) / 2))
    return base + jitter
}


func retryAfter(h http.Header) time.Duration {
    v := h.Get("Retry-After")
    if v == "" {
        return 0
    }
    if secs, err := strconv.Atoi(v); err == nil {
        return time.Duration(secs) * time.Second
    }
    return 0
}


func sleep(req *http.Request, d time.Duration) bool {
    select {
    case <-time.After(d):
        return true
    case <-req.Context().Done():
        return false
    }
}


// Compose into a provider.
func newProvider(apiKey string) (llmrouter.Provider, error) {
    client := &http.Client{
        Timeout: 5 * time.Minute,
        Transport: &retryTransport{
            base:        http.DefaultTransport,
            maxAttempts: 4,
        },
    }
    return openai.New(
        llmrouter.WithAPIKey(apiKey),
        llmrouter.WithHTTPClient(client),
    )
}


// Suppress unused: errors import for the larger example below.
var _ = errors.New

Stream errors

A stream that opened successfully may still fail later. The producer goroutine reports any terminal failure through Stream.Err(): 

for chunk := range stream.Chunks() {
    // ...
}
if err := stream.Err(); err != nil {
    // possibilities:
    //   - context.Canceled         (caller called cancel or stream.Cancel)
    //   - context.DeadlineExceeded (request context deadline fired)
    //   - network read error       (wrapped, e.g. "openai: read stream: ...")
    //   - upstream protocol error  (SSE buffer overflow, malformed frame)
}

The OpenAI provider wraps read errors like this:

if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) {
    hooks.Finish(fmt.Errorf("openai: read stream: %w", err))
    return
}

And the Anthropic provider:

if err := scanner.Err(); err != nil && err != io.EOF {
    hooks.Finish(fmt.Errorf("anthropic: read stream: %w", err))
    return
}

Stream errors are not safely retryable from outside the stream — you've already shipped some chunks to the caller, and you can't un-ship them. The right response is usually to surface the error and let the caller (a UI, an upstream gateway) decide whether to start a new request.

Context-cancelled errors

Two well-known sentinels surface from Stream.Err():

  • context.Canceled — caller cancelled the parent context, or called stream.Cancel(). Treat as expected; not an error to log noisily.
  • context.DeadlineExceeded — context.WithTimeout or context.WithDeadline fired. The request ran out of budget. Treat as a soft failure; possibly retry with a longer budget. 
switch {
case errors.Is(err, context.Canceled):
    // intentional — silent
case errors.Is(err, context.DeadlineExceeded):
    log.Printf("request exceeded deadline")
case err != nil:
    log.Printf("stream failed: %v", err)
}

Structured logging

For production, log upstream errors with structured fields so you can build alerts and dashboards on them: 

import "log/slog"


func logErr(ctx context.Context, err error) {
    var ue *llmrouter.ErrUpstream
    switch {
    case errors.As(err, &ue):
        slog.ErrorContext(ctx, "llm upstream error",
            "provider", ue.Provider,
            "status", ue.StatusCode,
            "body", ue.Body,
        )
    case errors.Is(err, context.DeadlineExceeded):
        slog.WarnContext(ctx, "llm request exceeded deadline")
    case errors.Is(err, context.Canceled):
        // expected; suppress
    case errors.Is(err, llmrouter.ErrInvalidConfig):
        slog.ErrorContext(ctx, "llm config error", "err", err)
    default:
        slog.ErrorContext(ctx, "llm unknown error", "err", err)
    }
}

Full example: tiered error handler

Putting it together — a function that retries 429 and 5xx, fails fast on other 4xx, surfaces stream errors to the caller, and distinguishes cancellation from deadline-exceeded: 

package main


import (
    "context"
    "errors"
    "fmt"
    "io"
    "log/slog"
    "strings"
    "time"


    "github.com/elloloop/llmrouter"
)


// completeWithRetry calls p.CompletionStream with retry on transient
// failures, then streams the response into a string. The retry only
// applies to the open phase; stream-read errors are surfaced as-is.
func completeWithRetry(
    ctx context.Context,
    p llmrouter.Provider,
    req llmrouter.ChatRequest,
    maxAttempts int,
) (string, error) {
    var lastErr error
    for attempt := 1; attempt <= maxAttempts; attempt++ {
        text, err := completeOnce(ctx, p, req)
        if err == nil {
            return text, nil
        }
        if !isTransient(err) {
            return "", err
        }
        lastErr = err
        slog.WarnContext(ctx, "llm transient failure",
            "provider", p.Name(),
            "attempt", attempt,
            "err", err.Error(),
        )
        select {
        case <-time.After(backoff(attempt)):
        case <-ctx.Done():
            return "", ctx.Err()
        }
    }
    return "", fmt.Errorf("after %d attempts: %w", maxAttempts, lastErr)
}


func completeOnce(ctx context.Context, p llmrouter.Provider, req llmrouter.ChatRequest) (string, error) {
    stream, err := p.CompletionStream(ctx, req)
    if err != nil {
        return "", err
    }
    var out strings.Builder
    for chunk := range stream.Chunks() {
        for _, c := range chunk.Choices {
            out.WriteString(c.Delta.Content)
        }
    }
    if err := stream.Err(); err != nil {
        // Stream errors are NOT retried — partial content was already
        // observed by the producer goroutine. Surface and let the
        // caller decide.
        return out.String(), err
    }
    return out.String(), nil
}


// isTransient returns true for errors that may resolve on retry.
func isTransient(err error) bool {
    // context cancellation / deadline are not retryable here — the
    // caller's deadline applies to every attempt.
    if errors.Is(err, context.Canceled) ||
        errors.Is(err, context.DeadlineExceeded) {
        return false
    }
    var ue *llmrouter.ErrUpstream
    if errors.As(err, &ue) {
        switch ue.StatusCode {
        case 408, 429, 500, 502, 503, 504:
            return true
        default:
            return false
        }
    }
    // Bare network errors at the open phase: retry.
    return errors.Is(err, io.ErrUnexpectedEOF) ||
        strings.Contains(err.Error(), "connection refused") ||
        strings.Contains(err.Error(), "EOF")
}


func backoff(attempt int) time.Duration {
    return time.Duration(1<<attempt) * 500 * time.Millisecond
}

Next steps