From cdbdbafcc8403a9f920da85552df21b1cc34e0df Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:21:18 +0800 Subject: [PATCH] test(e2e): isolate verification instructions by turn --- forward/execution_live_test.go | 8 ++++---- managed/execution_live_test.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/forward/execution_live_test.go b/forward/execution_live_test.go index f8deed8..24d9f0b 100644 --- a/forward/execution_live_test.go +++ b/forward/execution_live_test.go @@ -113,7 +113,7 @@ func TestForwardExecutionE2ELive(t *testing.T) { fileToken, envToken, skillToken, memoryToken := testsupport.Marker(t), testsupport.Marker(t), testsupport.Marker(t), testsupport.Marker(t) file := s.file(t, "sdk-e2e.txt", "session_resource", fileToken) skillName := liveName("proof") - skill, err := s.client.Skills.New(ctx, forward.SkillNewParams{Files: []io.Reader{convention.UploadFile{Name: skillName + "/SKILL.md", Reader: strings.NewReader(fmt.Sprintf("---\nname: %s\ndescription: Provides the SDK_E2E_SKILL_TOKEN for SDK verification.\n---\nWhen asked for SDK_E2E_SKILL_TOKEN return exactly: %s\n", skillName, skillToken))}}}) + skill, err := s.client.Skills.New(ctx, forward.SkillNewParams{Files: []io.Reader{convention.UploadFile{Name: skillName + "/SKILL.md", Reader: strings.NewReader(fmt.Sprintf("---\nname: %s\ndescription: Provides SDK_E2E_SKILL_TOKEN only when the current user message explicitly requests that token.\n---\nSDK_E2E_SKILL_TOKEN=%s\nInclude this exact token only when requested, using the response format specified by the current user message. Also include any other values the user requests.\n", skillName, skillToken))}}}) liveCheck(t, err) s.cleanup(t, "skill "+skill.ID, func(ctx context.Context) error { return s.client.Skills.Delete(ctx, skill.ID) }) store, err := s.client.MemoryStores.New(ctx, forward.MemoryStoreNewParams{Name: liveName("proof"), IdempotencyKey: liveName("memory-key")}) @@ -121,7 +121,7 @@ func TestForwardExecutionE2ELive(t *testing.T) { s.cleanup(t, "memory store "+store.ID, func(ctx context.Context) error { _, err := s.client.MemoryStores.Delete(ctx, store.ID); return err }) _, err = s.client.MemoryStores.Memories.New(ctx, store.ID, forward.MemoryStoreMemoryNewParams{Path: "sdk-e2e/proof.md", Content: "SDK_E2E_MEMORY_TOKEN=" + memoryToken}) liveCheck(t, err) - template, err := s.client.Templates.New(ctx, forward.TemplateNewParams{Name: liveName("proof"), EnvironmentID: env.ID, Model: forward.ModelConfigUnionParam{OfString: forward.String(os.Getenv("QODER_FORWARD_MODEL"))}, System: forward.String("Complete the requested SDK verification. Use the available tools to read files, environment variables, skills and memory. Do not guess missing values."), Tools: []forward.ToolParam{{Type: "agent_toolset_20260401"}}, Skills: []forward.SkillBindingParam{{Type: "custom", SkillID: skill.ID, Version: forward.String(skill.LatestVersion)}}, EnvironmentVariables: forward.EnvironmentVariablesUnionParam{OfMap: map[string]any{"SDK_E2E_VALUE": "template-default"}}}) + template, err := s.client.Templates.New(ctx, forward.TemplateNewParams{Name: liveName("proof"), EnvironmentID: env.ID, Model: forward.ModelConfigUnionParam{OfString: forward.String(os.Getenv("QODER_FORWARD_MODEL"))}, System: forward.String("Each user message is a separate SDK verification step. Follow the current user message and its response format; do not carry over instructions or answers from previous steps. Use tools only to access resources explicitly requested in the current message. Read skills only when that message explicitly requests a skill token. When asked to echo text already supplied in the message, reply directly without using tools. Return every requested value exactly. Do not guess missing values."), Tools: []forward.ToolParam{{Type: "agent_toolset_20260401"}}, Skills: []forward.SkillBindingParam{{Type: "custom", SkillID: skill.ID, Version: forward.String(skill.LatestVersion)}}, EnvironmentVariables: forward.EnvironmentVariablesUnionParam{OfMap: map[string]any{"SDK_E2E_VALUE": "template-default"}}}) liveCheck(t, err) s.cleanup(t, "template "+template.ID, func(ctx context.Context) error { _, err := s.client.Templates.Archive(ctx, template.ID, forward.TemplateArchiveParams{}) @@ -146,8 +146,8 @@ func TestForwardExecutionE2ELive(t *testing.T) { tool, stream bool }{ {"completion_and_sse", "Reply with exactly " + echo, []string{echo}, false, true}, - {"file_and_identity_config", "Use tools to read /data/workspace/sdk-e2e.txt and the SDK_E2E_VALUE environment variable. Reply with both exact values.", []string{fileToken, envToken}, true, false}, - {"skill_and_memory", "Use skill " + skillName + " to obtain SDK_E2E_SKILL_TOKEN. Read sdk-e2e/proof.md from the mounted memory store to obtain SDK_E2E_MEMORY_TOKEN. Reply with both exact tokens.", []string{skillToken, memoryToken}, true, false}, + {"file_and_identity_config", "Use tools to read /data/workspace/sdk-e2e.txt and the SDK_E2E_VALUE environment variable. Reply with both exact values on two lines: FILE= and ENV=.", []string{fileToken, envToken}, true, false}, + {"skill_and_memory", "Use skill " + skillName + " to obtain SDK_E2E_SKILL_TOKEN. Read sdk-e2e/proof.md from the mounted memory store to obtain SDK_E2E_MEMORY_TOKEN. Reply with both exact tokens on two lines: SKILL= and MEMORY=.", []string{skillToken, memoryToken}, true, false}, } { if !t.Run(scenario.name, func(t *testing.T) { after := s.sendTurn(t, session.ID, scenario.prompt) diff --git a/managed/execution_live_test.go b/managed/execution_live_test.go index d761096..e1ca09a 100644 --- a/managed/execution_live_test.go +++ b/managed/execution_live_test.go @@ -127,14 +127,14 @@ func TestManagedExecutionE2ELive(t *testing.T) { return err }) skillName := managedUnique("proof") - skill := liveResult(s.client.Skills.New(ctx, managed.SkillNewParams{Files: []io.Reader{convention.UploadFile{Name: skillName + "/SKILL.md", Reader: strings.NewReader(fmt.Sprintf("---\nname: %s\ndescription: Provides SDK_E2E_SKILL_TOKEN for SDK verification.\n---\nWhen asked for SDK_E2E_SKILL_TOKEN return exactly: %s\n", skillName, skillToken))}}})).require(t) + skill := liveResult(s.client.Skills.New(ctx, managed.SkillNewParams{Files: []io.Reader{convention.UploadFile{Name: skillName + "/SKILL.md", Reader: strings.NewReader(fmt.Sprintf("---\nname: %s\ndescription: Provides SDK_E2E_SKILL_TOKEN only when the current user message explicitly requests that token.\n---\nSDK_E2E_SKILL_TOKEN=%s\nInclude this exact token only when requested, using the response format specified by the current user message. Also include any other values the user requests.\n", skillName, skillToken))}}})).require(t) s.cleanup(t, "skill "+skill.ID, func(ctx context.Context) error { _, err := s.client.Skills.Delete(ctx, skill.ID, managed.SkillDeleteParams{}) return err }) store := s.createMemoryStore(t) liveResult(s.client.MemoryStores.Memories.New(ctx, store.ID, managed.MemoryStoreMemoryNewParams{Path: "sdk-e2e/proof.md", Content: managed.String("SDK_E2E_MEMORY_TOKEN=" + memoryToken)})).require(t) - agentParams := liveJSON[managed.AgentNewParams](t, map[string]any{"name": managedUnique("proof"), "model": map[string]any{"id": os.Getenv("QODER_MANAGED_MODEL")}, "system": "Complete the requested SDK verification. Use the available tools to read files, environment variables, skills and memory. Do not guess missing values.", "tools": []any{map[string]any{"type": "agent_toolset_20260401"}}, "skills": []any{map[string]any{"type": "custom", "skill_id": skill.ID, "version": skill.LatestVersionID}}}) + agentParams := liveJSON[managed.AgentNewParams](t, map[string]any{"name": managedUnique("proof"), "model": map[string]any{"id": os.Getenv("QODER_MANAGED_MODEL")}, "system": "Each user message is a separate SDK verification step. Follow the current user message and its response format; do not carry over instructions or answers from previous steps. Use tools only to access resources explicitly requested in the current message. Read skills only when that message explicitly requests a skill token. When asked to echo text already supplied in the message, reply directly without using tools. Return every requested value exactly. Do not guess missing values.", "tools": []any{map[string]any{"type": "agent_toolset_20260401"}}, "skills": []any{map[string]any{"type": "custom", "skill_id": skill.ID, "version": skill.LatestVersionID}}}) agent := liveResult(s.client.Agents.New(ctx, agentParams)).require(t) s.cleanup(t, "agent "+agent.ID, func(ctx context.Context) error { _, err := s.client.Agents.Archive(ctx, agent.ID, managed.AgentArchiveParams{}) @@ -151,8 +151,8 @@ func TestManagedExecutionE2ELive(t *testing.T) { tool, stream bool }{ {"completion_and_sse", "Reply with exactly " + echo, []string{echo}, false, true}, - {"file_and_environment_config", "Use tools to read /data/workspace/sdk-e2e.txt and the SDK_E2E_VALUE environment variable. Reply with both exact values.", []string{fileToken, envToken}, true, false}, - {"skill_and_memory", "Use skill " + skillName + " to obtain SDK_E2E_SKILL_TOKEN. Read sdk-e2e/proof.md from the mounted memory store to obtain SDK_E2E_MEMORY_TOKEN. Reply with both exact tokens.", []string{skillToken, memoryToken}, true, false}, + {"file_and_environment_config", "Use tools to read /data/workspace/sdk-e2e.txt and the SDK_E2E_VALUE environment variable. Reply with both exact values on two lines: FILE= and ENV=.", []string{fileToken, envToken}, true, false}, + {"skill_and_memory", "Use skill " + skillName + " to obtain SDK_E2E_SKILL_TOKEN. Read sdk-e2e/proof.md from the mounted memory store to obtain SDK_E2E_MEMORY_TOKEN. Reply with both exact tokens on two lines: SKILL= and MEMORY=.", []string{skillToken, memoryToken}, true, false}, } { if !t.Run(scenario.name, func(t *testing.T) { after := s.sendTurn(t, session.ID, scenario.prompt)