diff --git a/e2e/agentnetwork/chat_test.go b/e2e/agentnetwork/chat_test.go index 4ecafe7ea..a0cb25213 100644 --- a/e2e/agentnetwork/chat_test.go +++ b/e2e/agentnetwork/chat_test.go @@ -429,6 +429,13 @@ func TestProvidersMatrix(t *testing.T) { // session id and confirm the marker propagated end-to-end. sessionID := "e2e-session-" + pc.name + // A long-form prompt so every provider produces a realistic + // (~1k-word) completion — cost validation then runs on token + // counts of the same order as real traffic instead of a + // one-word pong. max_tokens in the harness bodies is sized + // (2048) to let the full answer through. + const matrixPrompt = "explain GitHub workflow in 1000 words" + // Retry briefly to absorb tunnel/DNS jitter on the first call. var code int var body string @@ -439,11 +446,11 @@ func TestProvidersMatrix(t *testing.T) { var cerr error switch pc.kind { case harness.WireVertex: - c, b, cerr = cl.Vertex(ctx, settings.Endpoint, proxyIP, pc.project, pc.region, pc.model, "Reply with exactly: pong", sessionID) + c, b, cerr = cl.Vertex(ctx, settings.Endpoint, proxyIP, pc.project, pc.region, pc.model, matrixPrompt, sessionID) case harness.WireBedrock: - c, b, cerr = cl.Bedrock(ctx, settings.Endpoint, proxyIP, pc.model, "Reply with exactly: pong", sessionID) + c, b, cerr = cl.Bedrock(ctx, settings.Endpoint, proxyIP, pc.model, matrixPrompt, sessionID) default: - c, b, cerr = cl.ChatPrefixed(ctx, settings.Endpoint, proxyIP, pc.pathPrefix, pc.kind, pc.model, "Reply with exactly: pong", sessionID) + c, b, cerr = cl.ChatPrefixed(ctx, settings.Endpoint, proxyIP, pc.pathPrefix, pc.kind, pc.model, matrixPrompt, sessionID) } if cerr == nil { code, body = c, b diff --git a/e2e/harness/client.go b/e2e/harness/client.go index 2ffcf653d..29389e16c 100644 --- a/e2e/harness/client.go +++ b/e2e/harness/client.go @@ -256,7 +256,7 @@ func (cl *Client) ChatPrefixed(ctx context.Context, endpoint, proxyIP, pathPrefi case WireMessages: path = "/v1/messages" headers = []string{"anthropic-version: 2023-06-01"} - body = fmt.Sprintf(`{"model":%q,"max_tokens":64,"messages":[{"role":"user","content":%q}]}`, model, prompt) + body = fmt.Sprintf(`{"model":%q,"max_tokens":2048,"messages":[{"role":"user","content":%q}]}`, model, prompt) default: path = "/v1/chat/completions" body = fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":%q}]}`, model, prompt) @@ -271,7 +271,7 @@ func (cl *Client) ChatPrefixed(ctx context.Context, endpoint, proxyIP, pathPrefi // is sent as the universal x-session-id header the proxy records. func (cl *Client) Vertex(ctx context.Context, endpoint, proxyIP, project, region, model, prompt, sessionID string) (int, string, error) { path := fmt.Sprintf("/v1/projects/%s/locations/%s/publishers/anthropic/models/%s:rawPredict", project, region, model) - body := fmt.Sprintf(`{"anthropic_version":"vertex-2023-10-16","max_tokens":64,"messages":[{"role":"user","content":%q}]}`, prompt) + body := fmt.Sprintf(`{"anthropic_version":"vertex-2023-10-16","max_tokens":2048,"messages":[{"role":"user","content":%q}]}`, prompt) return cl.post(ctx, endpoint, proxyIP, path, body, withSessionID(nil, sessionID)) } @@ -282,7 +282,7 @@ func (cl *Client) Vertex(ctx context.Context, endpoint, proxyIP, project, region // header the proxy records. func (cl *Client) Bedrock(ctx context.Context, endpoint, proxyIP, model, prompt, sessionID string) (int, string, error) { path := "/model/" + model + "/invoke" - body := fmt.Sprintf(`{"anthropic_version":"bedrock-2023-05-31","max_tokens":64,"messages":[{"role":"user","content":%q}]}`, prompt) + body := fmt.Sprintf(`{"anthropic_version":"bedrock-2023-05-31","max_tokens":2048,"messages":[{"role":"user","content":%q}]}`, prompt) return cl.post(ctx, endpoint, proxyIP, path, body, withSessionID(nil, sessionID)) }