feat(resilience): per-provider circuit breaker trip on quota errors with reset

.env.template

@@ -348,6 +348,16 @@
 # CIRCUIT_BREAKER_SUCCESS_THRESHOLD=2
 # Circuit breaker open-state timeout duration (default: 30s)
 # CIRCUIT_BREAKER_TIMEOUT=30s
+# Open the breaker instantly when an upstream error message matches a rule.
+# Rules are named groups; declare as many as you like with one MATCH
+# (+ optional TTL) var per group. A zero TTL uses CIRCUIT_BREAKER_TIMEOUT
+# above. Unset = feature inert.
+# CIRCUIT_BREAKER_TRIP_ON_WEEKLY_QUOTA_MATCH="weekly \(7-day\) usage limit"
+# CIRCUIT_BREAKER_TRIP_ON_WEEKLY_QUOTA_TTL=4h
+# CIRCUIT_BREAKER_TRIP_ON_QUOTA_EXCEEDED_MATCH="quota exceeded"  # no TTL
+# Per-provider override: replaces only the global rule of the same name;
+# other global rules stay in effect.
+# KIMICODE_CIRCUIT_BREAKER_TRIP_ON_WEEKLY_QUOTA_MATCH="..."
 
 # =============================================================================
 # Admin API & Dashboard Configuration

config/config.example.yaml

@@ -335,6 +335,20 @@ resilience:
     failure_threshold: 5
     success_threshold: 2
     timeout: 30s
+    # trip_on opens the breaker instantly when an upstream error message
+    # matches `match` (a Go regexp tested against the error message and code):
+    # failures are not counted, the breaker simply stays open for `ttl`
+    # (default: the `timeout` above), then probes recovery as usual. Meant for
+    # hard quota/billing stops that retries cannot fix. Rules are named groups
+    # (docker-compose style); they are inert until configured, `trip_on: {}`
+    # disables them, and an empty match, an invalid regexp, or a negative ttl
+    # fails config load. Env overrides by group name:
+    # CIRCUIT_BREAKER_TRIP_ON_<GROUP>_MATCH / _TTL (global),
+    # <PROVIDER>_CIRCUIT_BREAKER_TRIP_ON_<GROUP>_MATCH / _TTL (per provider).
+    # trip_on:
+    #   insufficient_quota:
+    #     match: "insufficient_quota|exceeded your current quota"
+    #     ttl: 30m
 
 guardrails:
   enabled: false
@@ -479,6 +493,10 @@ providers:
     #     max_retries: 5
     #   circuit_breaker:
     #     enabled: false
+    #     # A trip_on list here replaces the global one entirely; omit to inherit.
+    #     trip_on:
+    #       - match: "billing hard limit reached"
+    #         ttl: 1h
 
   anthropic:
     type: anthropic

config/config_helpers_test.go

@@ -361,6 +361,20 @@ func TestApplyEnvOverrides(t *testing.T) {
 				assert.Equal(t, 10*time.Second, cfg.Resilience.CircuitBreaker.Timeout)
 			},
 		},
+		{
+			name: "circuit breaker trip_on override",
+			envVars: map[string]string{
+				"CIRCUIT_BREAKER_TRIP_ON_QUOTA_MATCH": "quota exceeded",
+				"CIRCUIT_BREAKER_TRIP_ON_QUOTA_TTL":   "15m",
+				"CIRCUIT_BREAKER_TRIP_ON_USAGE_MATCH": "usage limit",
+			},
+			check: func(t *testing.T, cfg *Config) {
+				assert.Equal(t, TripRuleMap{
+					"QUOTA": {Name: "QUOTA", Match: "quota exceeded", TTL: 15 * time.Minute},
+					"USAGE": {Name: "USAGE", Match: "usage limit"},
+				}, cfg.Resilience.CircuitBreaker.TripOn)
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -375,3 +389,15 @@ func TestApplyEnvOverrides(t *testing.T) {
 		})
 	}
 }
+
+// A malformed trip_on env group fails configuration loading instead of
+// silently dropping quota protection.
+func TestApplyEnvOverrides_TripOnInvalid(t *testing.T) {
+	t.Setenv("CIRCUIT_BREAKER_TRIP_ON_BAD_TTL_MATCH", "quota exceeded")
+	t.Setenv("CIRCUIT_BREAKER_TRIP_ON_BAD_TTL_TTL", "banana")
+
+	cfg := buildDefaultConfig()
+	err := applyEnvOverrides(cfg)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "CIRCUIT_BREAKER_TRIP_ON_BAD_TTL_TTL")
+}

config/env.go

@@ -15,6 +15,18 @@ func applyEnvOverrides(cfg *Config) error {
 	if err := applyEnvOverridesValue(reflect.ValueOf(cfg).Elem()); err != nil {
 		return err
 	}
+	// Trip rules use named env groups (CIRCUIT_BREAKER_TRIP_ON_<GROUP>_MATCH /
+	// _TTL) instead of a single list variable, mirroring the per-provider
+	// <PROVIDER>_CIRCUIT_BREAKER_TRIP_ON_<GROUP>_* convention. A group
+	// replaces the config rule of the same name; other rules survive.
+	groups, err := CollectTripRuleGroups(os.Environ(), "CIRCUIT_BREAKER_TRIP_ON")
+	if err != nil {
+		return err
+	}
+	if len(groups) > 0 {
+		cfg.Resilience.CircuitBreaker.TripOn = TripRuleMapFromList(
+			MergeTripRuleGroups(cfg.Resilience.CircuitBreaker.TripOn.List(), groups))
+	}
 	normalizeModelListURL(cfg)
 	applyOfflineMode(cfg)
 	return nil

config/resilience.go

@@ -1,6 +1,11 @@
 package config
 
-import "time"
+import (
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+)
 
 // RetryConfig holds resolved retry settings for an LLM client.
 // This is the canonical type shared between config and llmclient.
@@ -26,6 +31,147 @@ func DefaultRetryConfig() RetryConfig {
 	}
 }
 
+// TripRuleConfig opens the circuit breaker instantly when an upstream error
+// message matches Match. A zero TTL uses the breaker's open-state timeout.
+// TTL encodes as JSON nanoseconds wherever the rule crosses the admin/store
+// wire as {match, ttl}.
+//
+// Name identifies the group the rule belongs to. It comes from the YAML map
+// key in `trip_on` blocks and from env group names; it only matters while
+// merging config with env overrides and is empty elsewhere.
+type TripRuleConfig struct {
+	Name  string        `yaml:"-" json:"name,omitempty"`
+	Match string        `yaml:"match" json:"match"`
+	TTL   time.Duration `yaml:"ttl" json:"ttl"`
+}
+
+// TripRuleMap holds named trip rules the way operators configure them:
+// docker-compose style groups keyed by name. The yaml shape is
+//
+//	trip_on:
+//	  weekly_quota:
+//	    match: "weekly usage limit"
+//	    ttl: 4h
+//
+// Unmarshal injects the key into each rule's Name; resolution turns the map
+// into TripRuleMap.List() so evaluation order is deterministic.
+type TripRuleMap map[string]TripRuleConfig
+
+// UnmarshalYAML accepts only the named map form. Each rule inherits the key
+// as its Name; an empty key is a config error.
+func (m *TripRuleMap) UnmarshalYAML(unmarshal func(any) error) error {
+	var raw map[string]TripRuleConfig
+	if err := unmarshal(&raw); err != nil {
+		return fmt.Errorf("trip_on must be a map of name -> {match, ttl}: %w", err)
+	}
+	for name, rule := range raw {
+		if strings.TrimSpace(name) == "" {
+			return fmt.Errorf("trip_on group names must not be empty")
+		}
+		rule.Name = name
+		raw[name] = rule
+	}
+	*m = TripRuleMap(raw)
+	return nil
+}
+
+// TripRuleMapFromList converts a name-carrying rule list back into a named
+// map. A rule with an empty name gets a generated key (`rule_N`) so unnamed
+// entries never collide with named ones.
+func TripRuleMapFromList(rules []TripRuleConfig) TripRuleMap {
+	m := make(TripRuleMap, len(rules))
+	unnamed := 0
+	for _, rule := range rules {
+		name := rule.Name
+		if name == "" {
+			unnamed++
+			name = fmt.Sprintf("rule_%d", unnamed)
+		}
+		m[name] = rule
+	}
+	return m
+}
+
+// List returns the rules with their names injected, ordered by group name so
+// "first matching rule wins" stays deterministic no matter how the map was
+// assembled (yaml, env, or merged).
+func (m TripRuleMap) List() []TripRuleConfig {
+	rules := make([]TripRuleConfig, 0, len(m))
+	for name, rule := range m {
+		rule.Name = name
+		rules = append(rules, rule)
+	}
+	slices.SortFunc(rules, func(a, b TripRuleConfig) int { return strings.Compare(a.Name, b.Name) })
+	return rules
+}
+
+// MergeTripRuleGroups returns the merged rule set: an incoming group replaces
+// the existing rule with the same name (case-insensitive); other existing
+// rules survive; new groups append. The result is ordered by name.
+func MergeTripRuleGroups(existing []TripRuleConfig, groups []TripRuleConfig) []TripRuleConfig {
+	merged := make(map[string]TripRuleConfig, len(existing)+len(groups))
+	for _, rule := range existing {
+		merged[strings.ToLower(rule.Name)] = rule
+	}
+	for _, group := range groups {
+		merged[strings.ToLower(group.Name)] = group
+	}
+	rules := make([]TripRuleConfig, 0, len(merged))
+	for _, rule := range merged {
+		rules = append(rules, rule)
+	}
+	slices.SortFunc(rules, func(a, b TripRuleConfig) int { return strings.Compare(a.Name, b.Name) })
+	return rules
+}
+
+// CollectTripRuleGroups reads `env` entries for `<PREFIX>_<GROUP>_MATCH` and
+// `<PREFIX>_<GROUP>_TTL` pairs. A group must carry MATCH; TTL is an optional
+// Go duration (zero uses the breaker timeout). Entries with a different
+// prefix or no suffix are ignored. Groups are returned ordered by name.
+func CollectTripRuleGroups(environ []string, prefix string) ([]TripRuleConfig, error) {
+	matchKey := prefix + "_"
+	groups := make(map[string]TripRuleConfig)
+	for _, entry := range environ {
+		key, value, ok := strings.Cut(entry, "=")
+		if !ok || value == "" || !strings.HasPrefix(key, matchKey) {
+			continue
+		}
+		// The attribute is the LAST underscore segment; group names may
+		// contain underscores themselves.
+		rest := key[len(matchKey):]
+		i := strings.LastIndex(rest, "_")
+		if i < 1 {
+			continue
+		}
+		name, attr := rest[:i], rest[i+1:]
+		if name == "" {
+			continue
+		}
+		rule := groups[name]
+		rule.Name = name
+		switch attr {
+		case "MATCH":
+			rule.Match = value
+		case "TTL":
+			ttl, err := time.ParseDuration(strings.TrimSpace(value))
+			if err != nil {
+				return nil, fmt.Errorf("%s: invalid ttl %q: %w", key, value, err)
+			}
+			rule.TTL = ttl
+		}
+		groups[name] = rule
+	}
+	rules := make([]TripRuleConfig, 0, len(groups))
+	for _, rule := range groups {
+		if rule.Match == "" {
+			return nil, fmt.Errorf("%s_%s_MATCH: group declares a TTL but no match pattern", prefix, rule.Name)
+		}
+		rules = append(rules, rule)
+	}
+	slices.SortFunc(rules, func(a, b TripRuleConfig) int { return strings.Compare(a.Name, b.Name) })
+	return rules, nil
+}
+
 // CircuitBreakerConfig holds resolved circuit breaker settings.
 // This is the canonical type shared between config and llmclient.
 type CircuitBreakerConfig struct {
@@ -35,10 +181,11 @@ type CircuitBreakerConfig struct {
 	// Enabled switches the circuit breaker on or off. When false, requests are
 	// never short-circuited regardless of the thresholds below.
 	// Default: true
-	Enabled          bool          `yaml:"enabled"           env:"CIRCUIT_BREAKER_ENABLED"`
-	FailureThreshold int           `yaml:"failure_threshold" env:"CIRCUIT_BREAKER_FAILURE_THRESHOLD"`
-	SuccessThreshold int           `yaml:"success_threshold" env:"CIRCUIT_BREAKER_SUCCESS_THRESHOLD"`
-	Timeout          time.Duration `yaml:"timeout"           env:"CIRCUIT_BREAKER_TIMEOUT"`
+	Enabled          bool             `yaml:"enabled"           env:"CIRCUIT_BREAKER_ENABLED"`
+	FailureThreshold int              `yaml:"failure_threshold" env:"CIRCUIT_BREAKER_FAILURE_THRESHOLD"`
+	SuccessThreshold int              `yaml:"success_threshold" env:"CIRCUIT_BREAKER_SUCCESS_THRESHOLD"`
+	Timeout          time.Duration    `yaml:"timeout"           env:"CIRCUIT_BREAKER_TIMEOUT"`
+	TripOn           TripRuleMap `yaml:"trip_on"`
 }
 
 // DefaultCircuitBreakerConfig returns the default circuit breaker settings.
@@ -69,12 +216,13 @@ type RawResilienceConfig struct {
 // RawCircuitBreakerConfig holds optional per-provider circuit breaker overrides from YAML.
 // Nil fields inherit from the global CircuitBreakerConfig.
 type RawCircuitBreakerConfig struct {
-	FailureOnStatuses []string       `yaml:"failure_on_statuses"`
-	Scope             *string        `yaml:"scope"`
-	Enabled           *bool          `yaml:"enabled"`
-	FailureThreshold  *int           `yaml:"failure_threshold"`
-	SuccessThreshold  *int           `yaml:"success_threshold"`
-	Timeout           *time.Duration `yaml:"timeout"`
+	FailureOnStatuses []string         `yaml:"failure_on_statuses"`
+	Scope             *string          `yaml:"scope"`
+	Enabled           *bool            `yaml:"enabled"`
+	FailureThreshold  *int             `yaml:"failure_threshold"`
+	SuccessThreshold  *int             `yaml:"success_threshold"`
+	Timeout           *time.Duration   `yaml:"timeout"`
+	TripOn            TripRuleMap `yaml:"trip_on"`
 }
 
 // RawRetryConfig holds optional per-provider retry overrides from YAML.

config/resilience_policy.go

@@ -1,6 +1,9 @@
 package config
 
-import "fmt"
+import (
+	"fmt"
+	"regexp"
+)
 
 // ParseResilienceStatuses expands exact HTTP codes and classes. Nil uses the
 // supplied defaults; an explicit empty list disables status-based matches.
@@ -40,6 +43,9 @@ func validateResilienceConfig(global ResilienceConfig, providers map[string]RawP
 			if cb.Scope != nil {
 				r.CircuitBreaker.Scope = *cb.Scope
 			}
+			if cb.TripOn != nil {
+				r.CircuitBreaker.TripOn = cb.TripOn
+			}
 		}
 		if err := ValidateResilience(r); err != nil {
 			return fmt.Errorf("providers.%s.resilience: %w", name, err)
@@ -61,6 +67,18 @@ func ValidateResilience(r ResilienceConfig) error {
 	default:
 		return fmt.Errorf("circuit_breaker.scope must be provider or model")
 	}
+	for _, rule := range r.CircuitBreaker.TripOn.List() {
+		name := "trip_on[" + rule.Name + "]"
+		if rule.Match == "" {
+			return fmt.Errorf("circuit_breaker.%s: match must not be empty", name)
+		}
+		if _, err := regexp.Compile(rule.Match); err != nil {
+			return fmt.Errorf("circuit_breaker.%s: %w", name, err)
+		}
+		if rule.TTL < 0 {
+			return fmt.Errorf("circuit_breaker.%s: ttl must not be negative", name)
+		}
+	}
 	return nil
 }