diff --git a/management/internals/modules/agentnetwork/policyselect_model_test.go b/management/internals/modules/agentnetwork/policyselect_model_test.go new file mode 100644 index 000000000..a9b73b0b2 --- /dev/null +++ b/management/internals/modules/agentnetwork/policyselect_model_test.go @@ -0,0 +1,336 @@ +package agentnetwork + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types" + "github.com/netbirdio/netbird/management/server/store" +) + +// guardedPolicy builds an enabled, uncapped policy that authorises sourceGroups +// to reach providerID under the given guardrails. Uncapped keeps the selector's +// headroom scoring trivial so these tests isolate the model-allowlist gate. +func guardedPolicy(id, account string, sourceGroups []string, providerID string, guardrailIDs ...string) *types.Policy { + return &types.Policy{ + ID: id, + AccountID: account, + Enabled: true, + SourceGroups: sourceGroups, + DestinationProviderIDs: []string{providerID}, + GuardrailIDs: guardrailIDs, + CreatedAt: time.Now().UTC(), + } +} + +// allowlistGuardrail builds a guardrail whose model allowlist is enabled and +// carries the given models. +func allowlistGuardrail(id, account string, models ...string) *types.Guardrail { + return &types.Guardrail{ + ID: id, + AccountID: account, + Checks: types.GuardrailChecks{ + ModelAllowlist: types.GuardrailModelAllowlist{Enabled: true, Models: models}, + }, + } +} + +func expectPolicies(mockStore *store.MockStore, account string, policies ...*types.Policy) { + mockStore.EXPECT(). + GetAccountAgentNetworkPolicies(gomock.Any(), gomock.Any(), account). + Return(policies, nil) +} + +func expectGuardrails(mockStore *store.MockStore, account string, guardrails ...*types.Guardrail) { + mockStore.EXPECT(). + GetAccountAgentNetworkGuardrails(gomock.Any(), gomock.Any(), account). + Return(guardrails, nil) +} + +// TestSelectPolicy_ModelBlockedByAllowlist proves the authoritative allowlist +// decision: a policy authorises the (provider, group) but restricts the model, +// and the requested model isn't on the list, so the request is denied. +func TestSelectPolicy_ModelBlockedByAllowlist(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o")) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + UserID: "user-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "claude-opus-4", + }) + require.NoError(t, err) + assert.False(t, res.Allow, "a model outside the only applicable policy's allowlist must be denied") + assert.Equal(t, denyCodeModelBlocked, res.DenyCode, "deny code must be model_blocked") + assert.NotEmpty(t, res.DenyReason, "deny reason must be populated") +} + +// TestSelectPolicy_ModelAllowedByAllowlist is the allow counterpart: the model +// is on the applicable policy's allowlist, so selection proceeds normally. +func TestSelectPolicy_ModelAllowedByAllowlist(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o", "claude-opus-4")) + expectConsumptionBatch(mockStore, nil) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + UserID: "user-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "claude-opus-4", + }) + require.NoError(t, err) + assert.True(t, res.Allow, "a model on the applicable policy's allowlist must be allowed") + assert.Equal(t, "pol-A", res.SelectedPolicyID) +} + +// TestSelectPolicy_CaseInsensitiveModelMatch proves the compare tolerates case +// and surrounding whitespace, matching the proxy guardrail's normalisation. +func TestSelectPolicy_CaseInsensitiveModelMatch(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", " GPT-4o ")) + expectConsumptionBatch(mockStore, nil) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "gpt-4o", + }) + require.NoError(t, err) + assert.True(t, res.Allow, "case/whitespace variants must match the allowlist entry") +} + +// TestSelectPolicy_UnguardedPolicyIsUnrestricted is the false-deny fix: when two +// policies authorise the same (provider, group) and one has no guardrail, that +// policy makes the request unrestricted — not caught by the other's allowlist. +func TestSelectPolicy_UnguardedPolicyIsUnrestricted(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + restricted := guardedPolicy("pol-restricted", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + open := guardedPolicy("pol-open", "acc-1", []string{"grp-eng"}, "prov-1") // no guardrail + expectPolicies(mockStore, "acc-1", restricted, open) + expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o")) + expectConsumptionBatch(mockStore, nil) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "claude-opus-4", + }) + require.NoError(t, err) + assert.True(t, res.Allow, "an un-guardrailed policy for the same (provider, group) must leave the request unrestricted") + assert.Equal(t, "pol-open", res.SelectedPolicyID, "the unrestricted policy must be the one that pays") +} + +// TestSelectPolicy_AllowlistDoesNotLeakAcrossGroups is the false-allow fix: a +// model allowlisted only for grp-b must not be usable by a grp-a caller. The +// selector considers only policies applicable to the caller's groups. +func TestSelectPolicy_AllowlistDoesNotLeakAcrossGroups(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + polA := guardedPolicy("pol-a", "acc-1", []string{"grp-a"}, "prov-1", "g-a") + polB := guardedPolicy("pol-b", "acc-1", []string{"grp-b"}, "prov-1", "g-b") + expectPolicies(mockStore, "acc-1", polA, polB) + expectGuardrails(mockStore, "acc-1", + allowlistGuardrail("g-a", "acc-1", "gpt-4o"), + allowlistGuardrail("g-b", "acc-1", "claude-opus-4"), + ) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + GroupIDs: []string{"grp-a"}, + ProviderID: "prov-1", + Model: "claude-opus-4", // only allowed for grp-b + }) + require.NoError(t, err) + assert.False(t, res.Allow, "grp-b's allowlisted model must not leak to a grp-a caller") + assert.Equal(t, denyCodeModelBlocked, res.DenyCode) +} + +// TestSelectPolicy_UndeterminedModelFailsClosed proves the fail-closed contract +// mirrors the proxy: with a restricted applicable policy and an empty model +// (e.g. a path-routed shape the parser couldn't map), the request is denied. +func TestSelectPolicy_UndeterminedModelFailsClosed(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o")) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "", // undetermined + }) + require.NoError(t, err) + assert.False(t, res.Allow, "an undetermined model must fail closed against a restricted policy") + assert.Equal(t, denyCodeModelBlocked, res.DenyCode) +} + +// TestSelectPolicy_DisabledAllowlistDoesNotRestrict proves a guardrail whose +// model allowlist is disabled imposes no model restriction, even though the +// policy references it. +func TestSelectPolicy_DisabledAllowlistDoesNotRestrict(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + disabled := &types.Guardrail{ + ID: "g-1", + AccountID: "acc-1", + Checks: types.GuardrailChecks{ + ModelAllowlist: types.GuardrailModelAllowlist{Enabled: false, Models: []string{"gpt-4o"}}, + }, + } + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", disabled) + expectConsumptionBatch(mockStore, nil) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "anything-goes", + }) + require.NoError(t, err) + assert.True(t, res.Allow, "a disabled allowlist must not restrict the model") + assert.Equal(t, "pol-A", res.SelectedPolicyID) +} + +// TestSelectPolicy_UnionAcrossPolicyGuardrails proves a policy with multiple +// allowlist guardrails permits the union of their models (not just the first). +func TestSelectPolicy_UnionAcrossPolicyGuardrails(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1", "g-2") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", + allowlistGuardrail("g-1", "acc-1", "gpt-4o"), + allowlistGuardrail("g-2", "acc-1", "claude-opus-4"), + ) + expectConsumptionBatch(mockStore, nil) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "claude-opus-4", // only in the second guardrail's list + }) + require.NoError(t, err) + assert.True(t, res.Allow, "a model in any of the policy's allowlist guardrails must be permitted") + assert.Equal(t, "pol-A", res.SelectedPolicyID) +} + +// TestSelectPolicy_GuardrailLookupErrorPropagates proves a store failure while +// resolving the candidate policies' guardrails surfaces as an error rather than +// silently allowing or denying the request. +func TestSelectPolicy_GuardrailLookupErrorPropagates(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + expectPolicies(mockStore, "acc-1", policy) + mockStore.EXPECT(). + GetAccountAgentNetworkGuardrails(gomock.Any(), gomock.Any(), "acc-1"). + Return(nil, errors.New("store unavailable")) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "gpt-4o", + }) + require.Error(t, err, "a guardrail-lookup failure must surface as an error, not a silent allow/deny") + assert.Nil(t, res) +} + +// TestSelectPolicy_MissingGuardrailReferenceTreatedAsUnrestricted proves a +// policy referencing a guardrail ID absent from the account's guardrails (a +// stale/orphaned reference) imposes no model restriction on its own — it is +// treated the same as no guardrail at all rather than failing closed. +func TestSelectPolicy_MissingGuardrailReferenceTreatedAsUnrestricted(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + // pol-A references "g-missing", which does not exist in the returned + // guardrails set. + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-missing") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1") + expectConsumptionBatch(mockStore, nil) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "anything-goes", + }) + require.NoError(t, err) + assert.True(t, res.Allow, "an orphaned guardrail reference must not restrict the model") + assert.Equal(t, "pol-A", res.SelectedPolicyID) +} + +// TestSelectPolicy_PartialCandidatesPermittedAfterModelFilter proves the +// model-allowlist gate narrows candidates before cap-based scoring runs: when +// two policies are otherwise applicable but only one permits the requested +// model, the permitting policy must be selected even though the other has a +// larger, more attractive cap. +func TestSelectPolicy_PartialCandidatesPermittedAfterModelFilter(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + // pol-big has a much larger group token cap and would normally win the + // "bigger pool first" tiebreak, but its guardrail blocks the requested model. + polBig := guardedPolicy("pol-big", "acc-1", []string{"grp-eng"}, "prov-1", "g-restrict") + polBig.Limits = types.PolicyLimits{ + TokenLimit: types.PolicyTokenLimit{Enabled: true, GroupCap: 1_000_000, WindowSeconds: 3600}, + } + polSmall := guardedPolicy("pol-small", "acc-1", []string{"grp-eng"}, "prov-1", "g-permit") + polSmall.Limits = types.PolicyLimits{ + TokenLimit: types.PolicyTokenLimit{Enabled: true, GroupCap: 100, WindowSeconds: 3600}, + } + expectPolicies(mockStore, "acc-1", polBig, polSmall) + expectGuardrails(mockStore, "acc-1", + allowlistGuardrail("g-restrict", "acc-1", "gpt-4o"), + allowlistGuardrail("g-permit", "acc-1", "claude-opus-4"), + ) + expectConsumptionBatch(mockStore, nil) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "claude-opus-4", // only pol-small's guardrail permits this + }) + require.NoError(t, err) + assert.True(t, res.Allow) + assert.Equal(t, "pol-small", res.SelectedPolicyID, + "the model filter must exclude pol-big before cap scoring, even though it would otherwise outrank pol-small") +} diff --git a/management/internals/shared/grpc/proxy_llm_policy_limits_test.go b/management/internals/shared/grpc/proxy_llm_policy_limits_test.go new file mode 100644 index 000000000..6191d59e7 --- /dev/null +++ b/management/internals/shared/grpc/proxy_llm_policy_limits_test.go @@ -0,0 +1,145 @@ +package grpc + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + grpcstatus "google.golang.org/grpc/status" + + "github.com/netbirdio/netbird/management/internals/modules/agentnetwork" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// fakeAgentNetworkLimits is a minimal AgentNetworkLimitsService double that +// records the PolicySelectionInput it was invoked with and returns a +// pre-programmed result, so tests can assert exactly what CheckLLMPolicyLimits +// forwards to the selector. +type fakeAgentNetworkLimits struct { + gotInput agentnetwork.PolicySelectionInput + result *agentnetwork.PolicySelectionResult + err error +} + +func (f *fakeAgentNetworkLimits) SelectPolicyForRequest(_ context.Context, in agentnetwork.PolicySelectionInput) (*agentnetwork.PolicySelectionResult, error) { + f.gotInput = in + if f.err != nil { + return nil, f.err + } + return f.result, nil +} + +func (f *fakeAgentNetworkLimits) RecordUsage(_ context.Context, _ agentnetwork.RecordUsageInput) error { + return nil +} + +// TestCheckLLMPolicyLimits_ForwardsModelToSelector proves the wiring added in +// this PR: the model the proxy extracted from the request body must reach the +// selector's PolicySelectionInput.Model unchanged, alongside the pre-existing +// account/user/group/provider fields. +func TestCheckLLMPolicyLimits_ForwardsModelToSelector(t *testing.T) { + fake := &fakeAgentNetworkLimits{result: &agentnetwork.PolicySelectionResult{Allow: true, SelectedPolicyID: "pol-1"}} + s := &ProxyServiceServer{} + s.SetAgentNetworkLimitsService(fake) + + req := &proto.CheckLLMPolicyLimitsRequest{ + AccountId: "acc-1", + UserId: "user-1", + GroupIds: []string{"grp-a", "grp-b"}, + ProviderId: "prov-1", + Model: "claude-opus-4", + } + + resp, err := s.CheckLLMPolicyLimits(context.Background(), req) + require.NoError(t, err) + require.NotNil(t, resp) + + assert.Equal(t, "acc-1", fake.gotInput.AccountID) + assert.Equal(t, "user-1", fake.gotInput.UserID) + assert.Equal(t, []string{"grp-a", "grp-b"}, fake.gotInput.GroupIDs) + assert.Equal(t, "prov-1", fake.gotInput.ProviderID) + assert.Equal(t, "claude-opus-4", fake.gotInput.Model, + "the request's model must be forwarded to the selector's PolicySelectionInput") +} + +// TestCheckLLMPolicyLimits_EmptyModelForwardedAsEmpty proves an +// undetermined model (empty string) is forwarded as-is rather than defaulted +// to some sentinel — the selector is the one that decides how to treat it +// (fail closed when a restriction applies). +func TestCheckLLMPolicyLimits_EmptyModelForwardedAsEmpty(t *testing.T) { + fake := &fakeAgentNetworkLimits{result: &agentnetwork.PolicySelectionResult{Allow: true}} + s := &ProxyServiceServer{} + s.SetAgentNetworkLimitsService(fake) + + req := &proto.CheckLLMPolicyLimitsRequest{ + AccountId: "acc-1", + ProviderId: "prov-1", + } + + _, err := s.CheckLLMPolicyLimits(context.Background(), req) + require.NoError(t, err) + assert.Equal(t, "", fake.gotInput.Model, "an absent model must be forwarded as empty, not fabricated") +} + +// TestCheckLLMPolicyLimits_DenyResponseCarriesModelBlockedCode proves the deny +// envelope surfaces the model-allowlist deny code + reason end to end through +// the gRPC response, matching the proxy guardrail's code so both layers agree. +func TestCheckLLMPolicyLimits_DenyResponseCarriesModelBlockedCode(t *testing.T) { + fake := &fakeAgentNetworkLimits{result: &agentnetwork.PolicySelectionResult{ + Allow: false, + DenyCode: "llm_policy.model_blocked", + DenyReason: `model "claude-opus-4" is not permitted by any applicable policy allowlist`, + }} + s := &ProxyServiceServer{} + s.SetAgentNetworkLimitsService(fake) + + resp, err := s.CheckLLMPolicyLimits(context.Background(), &proto.CheckLLMPolicyLimitsRequest{ + AccountId: "acc-1", + ProviderId: "prov-1", + Model: "claude-opus-4", + }) + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, "deny", resp.Decision) + assert.Equal(t, "llm_policy.model_blocked", resp.DenyCode) + assert.NotEmpty(t, resp.DenyReason) + assert.Empty(t, resp.SelectedPolicyId, "a denied request must carry no selected policy") +} + +// TestCheckLLMPolicyLimits_SelectorErrorSurfacesAsInternal proves a selector +// failure (e.g. a store error propagated from SelectPolicyForRequest) is +// surfaced as an Internal gRPC error rather than silently allowed. +func TestCheckLLMPolicyLimits_SelectorErrorSurfacesAsInternal(t *testing.T) { + fake := &fakeAgentNetworkLimits{err: errors.New("boom")} + s := &ProxyServiceServer{} + s.SetAgentNetworkLimitsService(fake) + + _, err := s.CheckLLMPolicyLimits(context.Background(), &proto.CheckLLMPolicyLimitsRequest{ + AccountId: "acc-1", + ProviderId: "prov-1", + Model: "gpt-4o", + }) + require.Error(t, err) + st, ok := grpcstatus.FromError(err) + require.True(t, ok) + assert.Equal(t, codes.Internal, st.Code(), "selector errors must never fail open on the hot path") +} + +// TestCheckLLMPolicyLimits_UnconfiguredServiceReturnsUnimplemented locks the +// documented fallback: with no AgentNetworkLimitsService wired, the RPC must +// return Unimplemented rather than panic or silently allow. +func TestCheckLLMPolicyLimits_UnconfiguredServiceReturnsUnimplemented(t *testing.T) { + s := &ProxyServiceServer{} + + _, err := s.CheckLLMPolicyLimits(context.Background(), &proto.CheckLLMPolicyLimitsRequest{ + AccountId: "acc-1", + ProviderId: "prov-1", + }) + require.Error(t, err) + st, ok := grpcstatus.FromError(err) + require.True(t, ok) + assert.Equal(t, codes.Unimplemented, st.Code()) +} \ No newline at end of file diff --git a/proxy/internal/middleware/builtin/llm_guardrail/middleware_test.go b/proxy/internal/middleware/builtin/llm_guardrail/middleware_test.go index cd7e256dd..c244c2a0b 100644 --- a/proxy/internal/middleware/builtin/llm_guardrail/middleware_test.go +++ b/proxy/internal/middleware/builtin/llm_guardrail/middleware_test.go @@ -26,6 +26,25 @@ func newInput(meta ...middleware.KV) *middleware.Input { return &middleware.Input{Slot: middleware.SlotOnRequest, Metadata: meta} } +const ( + testProvider = "prov-1" + otherProvider = "prov-2" +) + +// providerCfg builds a Config restricting testProvider to the given models. +func providerCfg(models ...string) Config { + return Config{ProviderAllowlists: map[string][]string{testProvider: models}} +} + +// newInputProvider builds an input that carries a resolved provider id (as +// llm_router would stamp) plus any extra metadata. +func newInputProvider(provider string, meta ...middleware.KV) *middleware.Input { + all := make([]middleware.KV, 0, len(meta)+1) + all = append(all, middleware.KV{Key: middleware.KeyLLMResolvedProviderID, Value: provider}) + all = append(all, meta...) + return &middleware.Input{Slot: middleware.SlotOnRequest, Metadata: all} +} + func TestMiddlewareIdentity(t *testing.T) { mw := New(Config{}) assert.Equal(t, ID, mw.ID(), "middleware ID must be llm_guardrail") @@ -47,12 +66,12 @@ func TestMiddlewareIdentity(t *testing.T) { func TestAllowlistEmptyAllowsAnyModel(t *testing.T) { mw := New(Config{}) - out, err := mw.Invoke(context.Background(), newInput( + out, err := mw.Invoke(context.Background(), newInputProvider(testProvider, middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"}, )) require.NoError(t, err) require.NotNil(t, out) - assert.Equal(t, middleware.DecisionAllow, out.Decision, "empty allowlist must allow any model") + assert.Equal(t, middleware.DecisionAllow, out.Decision, "no provider allowlists must allow any model") v, ok := metaValue(t, out.Metadata, middleware.KeyLLMPolicyDecision) require.True(t, ok, "decision metadata must be emitted") assert.Equal(t, "allow", v, "decision must be allow") @@ -62,8 +81,8 @@ func TestAllowlistEmptyAllowsAnyModel(t *testing.T) { } func TestAllowlistMatchAllows(t *testing.T) { - mw := New(Config{ModelAllowlist: []string{"gpt-4o", "claude-opus-4"}}) - out, err := mw.Invoke(context.Background(), newInput( + mw := New(providerCfg("gpt-4o", "claude-opus-4")) + out, err := mw.Invoke(context.Background(), newInputProvider(testProvider, middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"}, )) require.NoError(t, err) @@ -71,8 +90,8 @@ func TestAllowlistMatchAllows(t *testing.T) { } func TestAllowlistMissDenies(t *testing.T) { - mw := New(Config{ModelAllowlist: []string{"gpt-4o"}}) - out, err := mw.Invoke(context.Background(), newInput( + mw := New(providerCfg("gpt-4o")) + out, err := mw.Invoke(context.Background(), newInputProvider(testProvider, middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-opus-4"}, )) require.NoError(t, err) @@ -91,10 +110,10 @@ func TestAllowlistMissDenies(t *testing.T) { } func TestAllowlistCaseInsensitive(t *testing.T) { - mw := New(Config{ModelAllowlist: []string{" GPT-4o ", "Claude-OPUS-4"}}) + mw := New(providerCfg(" GPT-4o ", "Claude-OPUS-4")) cases := []string{"gpt-4o", "GPT-4O", " claude-opus-4 "} for _, model := range cases { - out, err := mw.Invoke(context.Background(), newInput( + out, err := mw.Invoke(context.Background(), newInputProvider(testProvider, middleware.KV{Key: middleware.KeyLLMModel, Value: model}, )) require.NoError(t, err) @@ -103,14 +122,15 @@ func TestAllowlistCaseInsensitive(t *testing.T) { } func TestAllowlistMissingModelKeyDenies(t *testing.T) { - // Fail closed: with an allowlist configured, a request whose model the - // parser could not extract (URL/path-routed providers such as Bedrock or - // Vertex whose shape wasn't recognised) must be denied, not allowed. - mw := New(Config{ModelAllowlist: []string{"gpt-4o"}}) - out, err := mw.Invoke(context.Background(), newInput()) + // Fail closed: with an allowlist in effect for the resolved provider, a + // request whose model the parser could not extract (URL/path-routed + // providers such as Bedrock or Vertex whose shape wasn't recognised) must be + // denied, not allowed. + mw := New(providerCfg("gpt-4o")) + out, err := mw.Invoke(context.Background(), newInputProvider(testProvider)) require.NoError(t, err) require.NotNil(t, out) - assert.Equal(t, middleware.DecisionDeny, out.Decision, "absent model must be denied when an allowlist is set") + assert.Equal(t, middleware.DecisionDeny, out.Decision, "absent model must be denied when the provider is restricted") assert.Equal(t, 403, out.DenyStatus, "deny status must be 403") require.NotNil(t, out.DenyReason, "deny reason must be populated") assert.Equal(t, "llm_policy.model_unknown", out.DenyReason.Code, "deny code must be model_unknown") @@ -122,26 +142,86 @@ func TestAllowlistMissingModelKeyDenies(t *testing.T) { func TestAllowlistEmptyModelValueDenies(t *testing.T) { // A present-but-empty model is as undeterminable as an absent one. - mw := New(Config{ModelAllowlist: []string{"gpt-4o"}}) - out, err := mw.Invoke(context.Background(), newInput( + mw := New(providerCfg("gpt-4o")) + out, err := mw.Invoke(context.Background(), newInputProvider(testProvider, middleware.KV{Key: middleware.KeyLLMModel, Value: " "}, )) require.NoError(t, err) require.NotNil(t, out) - assert.Equal(t, middleware.DecisionDeny, out.Decision, "empty model must be denied when an allowlist is set") + assert.Equal(t, middleware.DecisionDeny, out.Decision, "empty model must be denied when the provider is restricted") require.NotNil(t, out.DenyReason, "deny reason must be populated") assert.Equal(t, "llm_policy.model_unknown", out.DenyReason.Code, "deny code must be model_unknown") } func TestAllowlistEmptyListAllowsMissingModel(t *testing.T) { - // Without an allowlist there is nothing to enforce, so a missing model is - // still allowed — the fail-closed rule only applies when a list is set. + // Without any provider allowlists there is nothing to enforce, so a missing + // model is still allowed — the fail-closed rule only applies when a + // restriction is in effect. mw := New(Config{}) out, err := mw.Invoke(context.Background(), newInput()) require.NoError(t, err) assert.Equal(t, middleware.DecisionAllow, out.Decision, "no allowlist must allow even without a model") } +func TestUnrestrictedProviderAllowsAnyModel(t *testing.T) { + // The request resolved to otherProvider, which has no allowlist, so its + // traffic must not be caught by testProvider's restriction — the + // cross-provider-leak / false-deny guard. + mw := New(providerCfg("gpt-4o")) + out, err := mw.Invoke(context.Background(), newInputProvider(otherProvider, + middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-opus-4"}, + )) + require.NoError(t, err) + assert.Equal(t, middleware.DecisionAllow, out.Decision, "an unrestricted provider must not inherit another provider's allowlist") +} + +func TestPerProviderAllowlistsAreIsolated(t *testing.T) { + // gpt-4o is allowed only on testProvider; claude-opus-4 only on + // otherProvider. A model allowlisted for one provider must not be usable on + // the other — the fail-closed layer never unions allowlists across providers. + mw := New(Config{ProviderAllowlists: map[string][]string{ + testProvider: {"gpt-4o"}, + otherProvider: {"claude-opus-4"}, + }}) + out, err := mw.Invoke(context.Background(), newInputProvider(testProvider, + middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-opus-4"}, + )) + require.NoError(t, err) + assert.Equal(t, middleware.DecisionDeny, out.Decision, "claude-opus-4 is allowed only on otherProvider, not testProvider") + require.NotNil(t, out.DenyReason) + assert.Equal(t, "llm_policy.model_blocked", out.DenyReason.Code, "cross-provider model must be blocked, not model_unknown") +} + +func TestRestrictionsButNoResolvedProviderFailsClosed(t *testing.T) { + // Restrictions exist for the account but the resolved provider id is absent, + // so the request cannot be scoped to a provider. Fail closed rather than + // wave it through. + mw := New(providerCfg("gpt-4o")) + out, err := mw.Invoke(context.Background(), newInput( + middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"}, + )) + require.NoError(t, err) + require.NotNil(t, out) + assert.Equal(t, middleware.DecisionDeny, out.Decision, "missing resolved provider must fail closed when restrictions exist") + require.NotNil(t, out.DenyReason) + assert.Equal(t, "llm_policy.model_unknown", out.DenyReason.Code, "deny code must be model_unknown") +} + +func TestEnabledButEmptyAllowlistDeniesEveryModel(t *testing.T) { + // An allowlist-enabled provider with zero models is distinct from an + // unrestricted (absent) provider: it must deny every model, not allow + // everything through. + mw := New(providerCfg()) + out, err := mw.Invoke(context.Background(), newInputProvider(testProvider, + middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"}, + )) + require.NoError(t, err) + require.NotNil(t, out) + assert.Equal(t, middleware.DecisionDeny, out.Decision, "an enabled-but-empty allowlist must deny every model") + require.NotNil(t, out.DenyReason) + assert.Equal(t, "llm_policy.model_blocked", out.DenyReason.Code, "deny code must be model_blocked, not model_unknown") +} + func TestPromptCaptureDisabledEmitsNoPrompt(t *testing.T) { mw := New(Config{}) out, err := mw.Invoke(context.Background(), newInput( @@ -217,8 +297,8 @@ func TestFactoryAcceptsZeroConfigs(t *testing.T) { func TestFactoryDecodesValidConfig(t *testing.T) { cfg := Config{ - ModelAllowlist: []string{"gpt-4o"}, - PromptCapture: PromptCapture{Enabled: true, RedactPii: true}, + ProviderAllowlists: map[string][]string{testProvider: {"gpt-4o"}}, + PromptCapture: PromptCapture{Enabled: true, RedactPii: true}, } raw, err := json.Marshal(cfg) require.NoError(t, err, "marshalling test config must succeed") @@ -233,16 +313,33 @@ func TestFactoryRejectsMalformedJSON(t *testing.T) { assert.Nil(t, mw, "no middleware must be returned on malformed config") } -func TestFactoryNormalisesAllowlist(t *testing.T) { - raw := []byte(`{"model_allowlist":[" GPT-4o ","",""," Claude-3 "]}`) +func TestFactoryAllEmptyEntriesDenyEveryModel(t *testing.T) { + // All the provider's allowlist entries are blank/whitespace-only. They must + // collapse to a non-nil empty list (deny everything for that provider), + // not to "no restriction" for the provider. + raw := []byte(`{"provider_allowlists":{"prov-1":[""," "]}}`) mw, err := Factory{}.New(raw) require.NoError(t, err) - out, err := mw.Invoke(context.Background(), newInput( + out, err := mw.Invoke(context.Background(), newInputProvider(testProvider, + middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"}, + )) + require.NoError(t, err) + require.NotNil(t, out) + assert.Equal(t, middleware.DecisionDeny, out.Decision, "all-blank allowlist entries must still restrict the provider") + require.NotNil(t, out.DenyReason) + assert.Equal(t, "llm_policy.model_blocked", out.DenyReason.Code, "deny code must be model_blocked, confirming the provider is treated as restricted-to-none") +} + +func TestFactoryNormalisesAllowlist(t *testing.T) { + raw := []byte(`{"provider_allowlists":{"prov-1":[" GPT-4o ","",""," Claude-3 "]}}`) + mw, err := Factory{}.New(raw) + require.NoError(t, err) + out, err := mw.Invoke(context.Background(), newInputProvider(testProvider, middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"}, )) require.NoError(t, err) assert.Equal(t, middleware.DecisionAllow, out.Decision, "factory must lowercase + trim allowlist entries") - out2, err := mw.Invoke(context.Background(), newInput( + out2, err := mw.Invoke(context.Background(), newInputProvider(testProvider, middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-3"}, )) require.NoError(t, err)