From ba423e08dfd7a8da2186ace0e1e8d0ea7cfb64d4 Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Thu, 19 Nov 2026 14:23:31 -0800 Subject: [PATCH 1/4] feat(agents): deep agent harness PoC deep_agent() builds an Agent with a persisted plan (write_todos), a per-run file workspace shared with sub-agents, and optional approval on chosen tools. Verified end to end against a live server, including resuming a run after a worker crash. --- .gitignore | 1 + examples/agents/deep_agent_durability.py | 123 ++++++ examples/agents/deep_agent_research.py | 150 +++++++ src/conductor/ai/agents/deep.py | 506 +++++++++++++++++++++++ tests/unit/ai/test_deep_agent.py | 290 +++++++++++++ 5 files changed, 1070 insertions(+) create mode 100644 examples/agents/deep_agent_durability.py create mode 100644 examples/agents/deep_agent_research.py create mode 100644 src/conductor/ai/agents/deep.py create mode 100644 tests/unit/ai/test_deep_agent.py diff --git a/.gitignore b/.gitignore index 34da6705..a3f91b44 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ __pycache__/ # Example-run artifacts (generated by run_all_examples.py / coding-agent examples) examples/agents/run_report.html examples/agents/codingexamples/ +examples/agents/workspace/ # C extensions *.so diff --git a/examples/agents/deep_agent_durability.py b/examples/agents/deep_agent_durability.py new file mode 100644 index 00000000..574f8378 --- /dev/null +++ b/examples/agents/deep_agent_durability.py @@ -0,0 +1,123 @@ +"""Deep agent — survive a worker crash in the middle of a tool call. + +Phase 1 starts a run whose only tool, send_notification, records its side +effect and then kills its own worker process before reporting back, the way a +crash would. The script then kills every worker it started and hands the +execution id to phase 2. + +Phase 2 runs in a fresh process. It resumes the execution by id, which +registers new workers; the server retries the task after its response timeout; +the run completes. The script then reports how many times the tool was +attempted and how many times the side effect actually happened. + +What this shows: + - a run survives losing every worker mid call; nothing is persisted by the + caller except the execution id + - the server delivers a tool call at least once, so a tool with a side + effect must be idempotent; here it dedupes on (execution, message) + +Requirements: + - Conductor server with LLM support + - CONDUCTOR_SERVER_URL=http://localhost:8080/api as environment variable + - CONDUCTOR_AGENT_LLM_MODEL=openai/gpt-4o-mini as environment variable +""" + +import multiprocessing +import os +import subprocess +import sys +import time + +from conductor.ai.agents import AgentRuntime, tool +from conductor.ai.agents.deep import deep_agent, workspace_dir +from conductor.ai.agents.tool import ToolContext +from settings import settings + +WORKSPACE = "./workspace" + + +def _logs(execution_id: str): + ws = workspace_dir(execution_id, WORKSPACE) + ws.mkdir(parents=True, exist_ok=True) + return ws / "attempts.log", ws / "effects.log", ws / "crashed.flag" + + +@tool +def send_notification(message: str, context: ToolContext = None) -> dict: + """Send a notification message to the user. Call it once per message. + + Args: + message: The text to send. + """ + attempts, effects, crashed = _logs(context.execution_id) + with attempts.open("a") as f: + f.write(message + "\n") + + # Idempotency: the server may deliver this call again after a crash, so + # skip the side effect if this run already sent this message. + already = effects.exists() and message in effects.read_text().splitlines() + if not already: + with effects.open("a") as f: + f.write(message + "\n") + + # First attempt only: die after the side effect, before reporting back. + if not crashed.exists(): + crashed.write_text("1") + os._exit(1) + + return {"sent": message, "duplicate": already} + + +agent = deep_agent( + name="notifier", + model=settings.llm_model, + instructions=( + "Send exactly one notification with the text 'deploy finished' using " + "send_notification, then reply with the word done. This is a single-step " + "task, so do not write a plan." + ), + tools=[send_notification], + workspace=WORKSPACE, +) + + +def phase1() -> None: + with AgentRuntime() as runtime: + handle = runtime.start(agent, "Notify me that the deploy finished.") + print(f"Phase 1: started {handle.execution_id}") + attempts, _, crashed = _logs(handle.execution_id) + for _ in range(120): + if crashed.exists(): + break + time.sleep(1) + else: + print("Phase 1: the tool never ran; giving up") + return + print( + f"Phase 1: tool crashed after {len(attempts.read_text().splitlines())} attempt(s); " + "killing every worker" + ) + for p in multiprocessing.active_children(): + p.kill() + execution_id = handle.execution_id + # The runtime is gone. A new process picks the run up from the id alone. + subprocess.run([sys.executable, __file__, "--resume", execution_id], check=False) + + +def phase2(execution_id: str) -> None: + with AgentRuntime() as runtime: + handle = runtime.resume(execution_id, agent) + print(f"Phase 2: resumed {execution_id}; waiting for the server to retry the task") + result = handle.join(timeout=240) + attempts, effects, _ = _logs(execution_id) + n_attempts = len(attempts.read_text().splitlines()) if attempts.exists() else 0 + n_effects = len(effects.read_text().splitlines()) if effects.exists() else 0 + print(f"Phase 2: status={result.status} attempts={n_attempts} side_effects={n_effects}") + print(f"Phase 2: output={result.output!r}") + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--resume": + phase2(sys.argv[2]) + else: + phase1() diff --git a/examples/agents/deep_agent_research.py b/examples/agents/deep_agent_research.py new file mode 100644 index 00000000..4ad482b2 --- /dev/null +++ b/examples/agents/deep_agent_research.py @@ -0,0 +1,150 @@ +"""Deep agent — plan, delegate to a sub-agent, pause for approval, write a report. + +A travel assistant that plans with write_todos, hands weather lookups to a +weather-analyst sub-agent, asks for approval before writing a file, and writes +its recommendation to report.md in the run's workspace. + +Demonstrates: + - deep_agent(): one call gives an Agent with planning, file tools and sub-agents + - The plan lives in _agent_state, outside the conversation + - A sub-agent with its own context, sharing the run's workspace + - Human approval before write_file + +Requirements: + - Conductor server with LLM support + - CONDUCTOR_SERVER_URL=http://localhost:8080/api as environment variable + - CONDUCTOR_AGENT_LLM_MODEL=openai/gpt-4o-mini as environment variable + - Network access to open-meteo.com for the weather tool +""" + +import json +import urllib.parse +import urllib.request + +from conductor.ai.agents import Agent, AgentRuntime, EventType, tool +from conductor.ai.agents.deep import deep_agent, workspace_dir +from settings import settings + +_WMO = { + 0: "clear", + 1: "mainly clear", + 2: "partly cloudy", + 3: "overcast", + 45: "fog", + 51: "light drizzle", + 53: "drizzle", + 55: "heavy drizzle", + 61: "light rain", + 63: "rain", + 65: "heavy rain", + 71: "light snow", + 73: "snow", + 75: "heavy snow", + 80: "light showers", + 81: "showers", + 82: "violent showers", + 95: "thunderstorm", +} + + +def _get_json(url: str, params: dict) -> dict: + with urllib.request.urlopen(f"{url}?{urllib.parse.urlencode(params)}", timeout=15) as resp: + return json.load(resp) + + +@tool +def get_weather(city: str) -> str: + """Get the current weather and a 7-day daily forecast for a city. + + Args: + city: City name in any language, for example "Beijing" or "上海". + """ + geo = _get_json( + "https://geocoding-api.open-meteo.com/v1/search", + {"name": city, "count": 1, "language": "en"}, + ) + if not geo.get("results"): + return f"City not found: {city}" + place = geo["results"][0] + fc = _get_json( + "https://api.open-meteo.com/v1/forecast", + { + "latitude": place["latitude"], + "longitude": place["longitude"], + "current": "temperature_2m,weather_code,wind_speed_10m", + "daily": "weather_code,temperature_2m_max,temperature_2m_min,precipitation_probability_max", + "timezone": "auto", + "forecast_days": 7, + }, + ) + cur, daily = fc["current"], fc["daily"] + lines = [ + f"{place['name']} ({place.get('admin1', '')}, {place.get('country', '')})", + f"Now ({cur['time']}): {_WMO.get(cur['weather_code'], cur['weather_code'])}, " + f"{cur['temperature_2m']}°C, wind {cur['wind_speed_10m']} km/h", + "Next 7 days:", + ] + for i, day in enumerate(daily["time"]): + lines.append( + f" {day}: {_WMO.get(daily['weather_code'][i], daily['weather_code'][i])}, " + f"{daily['temperature_2m_min'][i]}~{daily['temperature_2m_max'][i]}°C, " + f"precipitation {daily['precipitation_probability_max'][i]}%" + ) + return "\n".join(lines) + + +# The sub-agent has its own context. It gets the file tools from deep_agent so +# it can write its full comparison to the shared workspace. +weather_analyst = Agent( + name="weather_analyst", + model=settings.llm_model, + instructions=( + "You are a weather analyst. Call get_weather for each city you are given. " + "Write the full forecast for each city to weather-.md in the workspace " + "with write_file, then reply with a two-sentence summary and the file paths." + ), + tools=[get_weather], + metadata={ + "description": "Looks up and compares the weather in several cities. Give it the full list of cities." + }, +) + +agent = deep_agent( + name="travel_planner", + model=settings.llm_model, + instructions=( + "You are a travel assistant. Delegate all weather lookups to weather_analyst " + "in one call with every city. Then write your recommendation to report.md " + "and tell the user what you chose and why." + ), + subagents=[weather_analyst], + workspace="./workspace", + approval=["write_file"], +) + + +if __name__ == "__main__": + with AgentRuntime() as runtime: + handle = runtime.start( + agent, + "I want to visit Beijing or Shanghai this weekend. Compare the weather and recommend one.", + ) + print(f"Started: {handle.execution_id}\n") + + for event in handle.stream(): + if event.type == EventType.TOOL_CALL: + print(f" [tool_call] {event.tool_name}({str(event.args)[:120]})") + elif event.type == EventType.TOOL_RESULT: + print(f" [tool_result] {event.tool_name} -> {str(event.result)[:120]}") + elif event.type == EventType.WAITING: + print(" [waiting] approving write_file") + handle.approve() + elif event.type == EventType.DONE: + print(f"\nDone: {event.output}") + + ws = workspace_dir(handle.execution_id, "./workspace") + report = ws / "report.md" + print(f"\nWorkspace: {ws}") + if report.exists(): + print("--- report.md ---") + print(report.read_text()) diff --git a/src/conductor/ai/agents/deep.py b/src/conductor/ai/agents/deep.py new file mode 100644 index 00000000..62261879 --- /dev/null +++ b/src/conductor/ai/agents/deep.py @@ -0,0 +1,506 @@ +"""Deep agent harness: planning, files and sub-agents on top of a plain ``Agent``. + +A deep agent runs for a long time and does many steps. The harness keeps three +things out of the conversation so the model's context stays small: + +- the plan, in ``_agent_state`` through :func:`write_todos`, private to the + agent that wrote it and shown to it on every turn; +- large data, in files under a per-run workspace directory, shared with the + sub-agents of the same run; +- delegated work, in sub-agents that return a short result and never see the + parent's plan or history. + +This is a proof of concept on the Python SDK alone: the todo and file tools are +ordinary ``@tool`` workers. See ``examples/agents/deep_agent_research.py``. + +Example:: + + from conductor.ai.agents.deep import deep_agent + + agent = deep_agent( + name="researcher", + model="openai/gpt-4o-mini", + instructions="You are a travel assistant.", + tools=[get_weather], + subagents=[weather_analyst], + workspace="./workspace", + approval=["write_file"], + ) +""" + +from __future__ import annotations + +import dataclasses +import datetime as dt +import os +import threading +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Union + +from conductor.ai.agents.agent import Agent +from conductor.ai.agents.tool import ToolContext, agent_tool, get_tool_def, tool +from conductor.client.configuration.configuration import Configuration +from conductor.client.orkes_clients import OrkesClients + +WORKSPACE_ENV = "CONDUCTOR_DEEP_AGENT_WORKSPACE" +"""Environment variable naming the workspace root directory. + +Tool workers run in separate processes, so the root is passed through the +environment rather than captured in a closure. +""" + +DEFAULT_WORKSPACE = "./workspace" + +TODOS_KEY = "todos" +"""Key under which the plan lives in ``ToolContext.state`` (``_agent_state``). + +The value is ``{"items": [...]}`` rather than a bare list: the server merges a +list-valued update by appending its new items to the stored list, which would +keep every superseded step; an object-valued update replaces the stored one. +""" + +TODO_STATUSES = ("pending", "in_progress", "completed") + +FILE_TOOL_NAMES = ("ls", "read_file", "write_file", "edit_file") + +DEFAULT_INSTRUCTIONS = """\ +You are a capable agent that works through long tasks step by step. You have a +plan, a file workspace, and possibly sub-agents. Use them like this. + +## Plan first with write_todos +If the task needs more than one step, your first action is write_todos with the +full list of steps, each with status "pending". Do this before any other tool +call. Then, as you work: set a step to "in_progress" when you start it, and +call write_todos again to mark it "completed" the moment it is done. Send the +whole list every time; it replaces the previous one. Call write_todos only when +a status changes, never twice in a row, and never skip the plan on a multi-step +task. Your current plan is shown to you under "Context" on every turn. + +## Files +The workspace is private to this run. Paths are relative to it; "/report.md" +and "report.md" are the same file. Write anything long or anything worth +keeping to a file instead of repeating it in the conversation: research notes, +drafts, the final report. Read a file only when you need its content. Use +edit_file for small changes; use write_file to create or replace a whole file. +Use the file names your instructions ask for. + +## Sub-agents +Each sub-agent is a tool. Delegate a self-contained piece of work with a clear +request and everything it needs to know; it does not see your plan or this +conversation. Ask it to write long results to a file in the workspace and to +return a short summary plus the file path. Delegate when a step would otherwise +fill your context with material you only need a summary of. + +## Finishing +When every step is completed, give the user the result they asked for in your +final message, in the same turn or the next one. Marking the last todo +completed is not an answer, and the plan does not need further edits once it +is complete. If something cannot be done, say so and finish; do not rewrite +the plan to work around it. +""" + + +# ── Plan ──────────────────────────────────────────────────────────────── + + +def _validate_todos(todos: Any) -> List[Dict[str, str]]: + if not isinstance(todos, list): + raise ValueError("todos must be a list of {content, status} items") + cleaned: List[Dict[str, str]] = [] + for i, item in enumerate(todos): + if not isinstance(item, dict): + raise ValueError(f"todos[{i}] must be an object with content and status") + content = str(item.get("content", "")).strip() + status = str(item.get("status", "pending")).strip().lower() + if not content: + raise ValueError(f"todos[{i}].content is empty") + if status not in TODO_STATUSES: + raise ValueError( + f"todos[{i}].status is {status!r}; expected one of {', '.join(TODO_STATUSES)}" + ) + cleaned.append({"content": content, "status": status}) + return cleaned + + +@tool +def write_todos(todos: List[Dict[str, Any]], context: ToolContext = None) -> dict: + """Replace the plan for this run with a new list of steps. + + Each item is {"content": "", "status": "pending" | "in_progress" | "completed"}. + Send the complete list every time; it replaces the previous plan. Keep one + item in_progress at a time and mark a step completed as soon as it is done. + + Args: + todos: The full list of steps with their current status. + + Returns: + The stored plan and a one-line progress summary. + """ + cleaned = _validate_todos(todos) + previous = stored_todos(context.state) if context is not None else None + if context is not None: + context.state[TODOS_KEY] = {"items": cleaned} + done = sum(1 for t in cleaned if t["status"] == "completed") + out: Dict[str, Any] = {"todos": cleaned, "summary": f"{done}/{len(cleaned)} completed"} + # A clear stop signal: models otherwise keep polishing a finished plan + # instead of answering, until the turn limit ends the run. + if done == len(cleaned): + out["next"] = ( + "All steps are completed. Reply to the user with the result now; " + "do not call write_todos again." + ) + elif previous == cleaned: + out["next"] = "The plan is unchanged. Continue with the in-progress step." + return out + + +def stored_todos(state: Optional[Dict[str, Any]]) -> Optional[List[Dict[str, str]]]: + """The plan held in an agent state dict, or None when none was written.""" + if not state: + return None + value = state.get(TODOS_KEY) + if isinstance(value, dict): + items = value.get("items") + return list(items) if isinstance(items, list) else None + return None + + +# The reflected schema for List[Dict[str, Any]] has no item properties, which +# leaves weaker models guessing the field names. Spell the item out. +write_todos._tool_def.input_schema = { # type: ignore[attr-defined] + "type": "object", + "properties": { + "todos": { + "type": "array", + "description": "The full list of steps with their current status.", + "items": { + "type": "object", + "properties": { + "content": {"type": "string", "description": "What this step does."}, + "status": {"type": "string", "enum": list(TODO_STATUSES)}, + }, + "required": ["content", "status"], + }, + } + }, + "required": ["todos"], +} + + +# ── Workspace ─────────────────────────────────────────────────────────── + +_root_cache: Dict[str, str] = {} +_root_lock = threading.Lock() + + +def resolve_root_execution_id(execution_id: str, workflow_client: Any) -> str: + """Walk ``parent_workflow_id`` up from *execution_id* to the root run. + + A sub-agent runs as a sub-workflow with its own execution id; its files + must land in the parent's workspace, so the directory is keyed by the root. + Any lookup failure stops the walk and returns the furthest id reached. + """ + root = execution_id + seen = set() + current = execution_id + while current and current not in seen: + seen.add(current) + parent = _parent_of(workflow_client, current) + if not parent: + break + root = parent + current = parent + return root + + +def _parent_of(workflow_client: Any, execution_id: str) -> Optional[str]: + try: + wf = workflow_client.get_workflow(execution_id, include_tasks=False) + except Exception: + return None + return getattr(wf, "parent_workflow_id", None) + + +def _root_execution_id(execution_id: str) -> str: + with _root_lock: + cached = _root_cache.get(execution_id) + if cached: + return cached + try: + client = OrkesClients(configuration=Configuration()).get_workflow_client() + root = resolve_root_execution_id(execution_id, client) + except Exception: + root = execution_id + with _root_lock: + _root_cache[execution_id] = root + return root + + +def _run_id(context: Optional[ToolContext]) -> str: + if context is None or not context.execution_id: + return "local" + return _root_execution_id(context.execution_id) + + +def _workspace_root(context: Optional[ToolContext]) -> Path: + base = Path(os.environ.get(WORKSPACE_ENV) or DEFAULT_WORKSPACE) + root = (base / _run_id(context)).resolve() + root.mkdir(parents=True, exist_ok=True) + return root + + +def _resolve(root: Path, path: str) -> Path: + """Map a model-supplied path onto the workspace, refusing escapes.""" + rel = (path or "").strip().lstrip("/") + target = (root / rel).resolve() if rel else root + if target != root and root not in target.parents: + raise ValueError(f"path {path!r} is outside the workspace") + return target + + +# ── File tools ────────────────────────────────────────────────────────── + + +@tool +def ls(path: str = "/", context: ToolContext = None) -> dict: + """List the files and directories at a path in the workspace. + + Args: + path: Directory to list, relative to the workspace root. Default is the root. + + Returns: + The entries with their type and size in bytes. + """ + root = _workspace_root(context) + target = _resolve(root, path) + if not target.exists(): + return {"path": path, "entries": [], "error": "no such directory"} + if not target.is_dir(): + return {"path": path, "entries": [], "error": "not a directory"} + entries = [ + { + "name": p.name, + "type": "dir" if p.is_dir() else "file", + "size": p.stat().st_size if p.is_file() else 0, + } + for p in sorted(target.iterdir()) + ] + return {"path": path, "entries": entries} + + +@tool +def read_file(path: str, offset: int = 0, limit: int = 2000, context: ToolContext = None) -> dict: + """Read a text file from the workspace. + + Args: + path: File path relative to the workspace root. + offset: First line to return, counting from 0. + limit: Maximum number of lines to return. + + Returns: + The requested lines and the file's total line count. + """ + root = _workspace_root(context) + target = _resolve(root, path) + if not target.is_file(): + return {"path": path, "error": "no such file"} + lines = target.read_text(encoding="utf-8").splitlines() + chunk = lines[offset : offset + limit] + return { + "path": path, + "content": "\n".join(chunk), + "total_lines": len(lines), + "offset": offset, + "returned_lines": len(chunk), + } + + +@tool +def write_file(path: str, content: str, context: ToolContext = None) -> dict: + """Create or replace a text file in the workspace. + + Args: + path: File path relative to the workspace root. Parent directories are created. + content: The full file content. + + Returns: + The path written and its size in bytes. + """ + root = _workspace_root(context) + target = _resolve(root, path) + if target == root: + raise ValueError("path must name a file") + target.parent.mkdir(parents=True, exist_ok=True) + data = content.encode("utf-8") + target.write_bytes(data) + return {"path": path, "bytes": len(data)} + + +@tool +def edit_file( + path: str, + old_string: str, + new_string: str, + replace_all: bool = False, + context: ToolContext = None, +) -> dict: + """Replace text in a workspace file. + + Args: + path: File path relative to the workspace root. + old_string: The exact text to find. It must occur once unless replace_all is set. + new_string: The replacement text. + replace_all: Replace every occurrence instead of requiring a unique match. + + Returns: + The path edited and how many replacements were made. + """ + root = _workspace_root(context) + target = _resolve(root, path) + if not target.is_file(): + return {"path": path, "error": "no such file"} + if not old_string: + raise ValueError("old_string is empty") + text = target.read_text(encoding="utf-8") + count = text.count(old_string) + if count == 0: + return {"path": path, "error": "old_string not found", "replacements": 0} + if count > 1 and not replace_all: + return { + "path": path, + "error": f"old_string occurs {count} times; include more context or set replace_all", + "replacements": 0, + } + updated = ( + text.replace(old_string, new_string) + if replace_all + else text.replace(old_string, new_string, 1) + ) + target.write_text(updated, encoding="utf-8") + return {"path": path, "replacements": count if replace_all else 1} + + +FILE_TOOLS = (ls, read_file, write_file, edit_file) + + +# ── Harness entry point ───────────────────────────────────────────────── + + +def _with_approval(obj: Any, approval: Iterable[str]) -> Any: + """Return *obj* with approval_required set when its name is in *approval*.""" + names = set(approval) + if not names: + return obj + td = get_tool_def(obj) + if td.name not in names or td.approval_required: + return obj + return dataclasses.replace(td, approval_required=True) + + +def _name_of(obj: Any) -> Optional[str]: + try: + return get_tool_def(obj).name + except TypeError: + return None + + +def _tool_names(tools: Sequence[Any]) -> set: + return {n for n in (_name_of(t) for t in tools) if n} + + +def _describe(sub: Agent) -> str: + meta = getattr(sub, "metadata", None) or {} + desc = meta.get("description") + if desc: + return str(desc) + return f"Delegate a self-contained task to the {sub.name} agent. Give it everything it needs." + + +def deep_agent( + name: str, + *, + model: Union[str, Any] = "", + instructions: str = "", + tools: Optional[Sequence[Any]] = None, + subagents: Optional[Sequence[Agent]] = None, + workspace: Optional[Union[str, Path]] = None, + approval: Iterable[str] = (), + max_turns: int = 50, + **agent_kwargs: Any, +) -> Agent: + """Build an :class:`Agent` with planning, file tools and sub-agents. + + Args: + name: Agent name. + model: Model for the agent, as ``provider/model``. + instructions: Your instructions. They follow the harness's default + prompt, which explains planning, files and delegation. + tools: Your own tools, in addition to the harness tools. + subagents: Agents to expose as tools. Each gets the file tools so it + can write to the shared workspace, and does not get ``write_todos``. + workspace: Root directory for run workspaces. Each run gets a + sub-directory named by its root execution id. Defaults to + ``./workspace``. + approval: Names of tools that pause for human approval before they + run, for example ``["write_file"]``. + max_turns: Turn limit for the agent loop; deep agents need more than + the ``Agent`` default. + **agent_kwargs: Passed through to :class:`Agent`. + """ + if workspace is not None: + os.environ[WORKSPACE_ENV] = str(Path(workspace).resolve()) + + approval = tuple(approval) + agent_tools: List[Any] = [write_todos] + agent_tools.extend(_with_approval(t, approval) for t in FILE_TOOLS) + agent_tools.extend(_with_approval(t, approval) for t in tools or ()) + + for sub in subagents or (): + existing = _tool_names(sub.tools or []) + missing = [ft for ft in FILE_TOOLS if get_tool_def(ft).name not in existing] + if missing: + sub.tools = [*(sub.tools or []), *missing] + agent_tools.append(agent_tool(sub, name=sub.name, description=_describe(sub))) + + # The model's idea of today comes from its training data, which makes it + # argue with dated tool results; tell it the date. + today = dt.date.today().strftime("%A, %B %d, %Y") + full_instructions = f"{DEFAULT_INSTRUCTIONS}\nToday is {today}.\n" + if instructions: + full_instructions += f"\n## Your task\n{instructions.strip()}\n" + + return Agent( + name=name, + model=model, + instructions=full_instructions, + tools=agent_tools, + max_turns=max_turns, + **agent_kwargs, + ) + + +def workspace_dir(execution_id: str, workspace: Optional[Union[str, Path]] = None) -> Path: + """The directory a run's files live in, for callers outside the agent.""" + base = ( + Path(workspace) + if workspace is not None + else Path(os.environ.get(WORKSPACE_ENV) or DEFAULT_WORKSPACE) + ) + return (base / _root_execution_id(execution_id)).resolve() + + +__all__ = [ + "DEFAULT_INSTRUCTIONS", + "FILE_TOOLS", + "TODOS_KEY", + "WORKSPACE_ENV", + "deep_agent", + "edit_file", + "ls", + "read_file", + "resolve_root_execution_id", + "stored_todos", + "workspace_dir", + "write_file", + "write_todos", +] diff --git a/tests/unit/ai/test_deep_agent.py b/tests/unit/ai/test_deep_agent.py new file mode 100644 index 00000000..1dea3c56 --- /dev/null +++ b/tests/unit/ai/test_deep_agent.py @@ -0,0 +1,290 @@ +"""Tests for the deep agent harness: plan tool, file tools, and the entry point.""" + +import pytest + +from conductor.ai.agents import Agent +from conductor.ai.agents import deep +from conductor.ai.agents.deep import ( + DEFAULT_INSTRUCTIONS, + TODOS_KEY, + WORKSPACE_ENV, + deep_agent, + edit_file, + ls, + read_file, + resolve_root_execution_id, + stored_todos, + write_file, + write_todos, +) +from conductor.ai.agents.tool import ToolContext, get_tool_def + + +@pytest.fixture +def workspace(tmp_path, monkeypatch): + """A workspace root in tmp, with root-id resolution short-circuited.""" + monkeypatch.setenv(WORKSPACE_ENV, str(tmp_path)) + monkeypatch.setattr(deep, "_root_execution_id", lambda eid: eid) + return tmp_path + + +def ctx(execution_id="exec-1", state=None): + return ToolContext( + execution_id=execution_id, agent_name="t", state=state if state is not None else {} + ) + + +# ── write_todos ───────────────────────────────────────────────────────── + + +def test_write_todos_stores_the_plan_in_state(): + c = ctx() + out = write_todos.__wrapped__( + todos=[ + {"content": "Look up Beijing", "status": "completed"}, + {"content": "Look up Shanghai", "status": "in_progress"}, + {"content": "Write the report", "status": "pending"}, + ], + context=c, + ) + # Stored as an object so a later write replaces it; the server appends + # list-valued updates instead of replacing them. + assert c.state[TODOS_KEY] == {"items": out["todos"]} + assert stored_todos(c.state) == out["todos"] + assert out["summary"] == "1/3 completed" + assert [t["status"] for t in out["todos"]] == ["completed", "in_progress", "pending"] + + +def test_write_todos_rejects_bad_status_and_empty_content(): + with pytest.raises(ValueError, match="status"): + write_todos.__wrapped__(todos=[{"content": "x", "status": "done"}], context=ctx()) + with pytest.raises(ValueError, match="content"): + write_todos.__wrapped__(todos=[{"content": " ", "status": "pending"}], context=ctx()) + + +def test_write_todos_defaults_status_to_pending(): + out = write_todos.__wrapped__(todos=[{"content": "x"}], context=ctx()) + assert out["todos"][0]["status"] == "pending" + assert "next" not in out + + +def test_stored_todos_handles_missing_or_malformed_state(): + assert stored_todos(None) is None + assert stored_todos({}) is None + assert stored_todos({TODOS_KEY: ["not", "an", "object"]}) is None + assert stored_todos({TODOS_KEY: {"items": [{"content": "x", "status": "pending"}]}}) == [ + {"content": "x", "status": "pending"} + ] + + +def test_write_todos_tells_the_model_when_to_stop(): + c = ctx() + plan = [{"content": "a", "status": "completed"}, {"content": "b", "status": "in_progress"}] + out = write_todos.__wrapped__(todos=plan, context=c) + assert "next" not in out + out = write_todos.__wrapped__(todos=plan, context=c) + assert out["next"].startswith("The plan is unchanged") + out = write_todos.__wrapped__( + todos=[{"content": "a", "status": "completed"}, {"content": "b", "status": "completed"}], + context=c, + ) + assert out["next"].startswith("All steps are completed") + + +def test_write_todos_schema_spells_out_the_item_fields(): + schema = get_tool_def(write_todos).input_schema + todos = schema["properties"]["todos"] + assert todos["type"] == "array" + item = todos["items"] + assert item["required"] == ["content", "status"] + assert item["properties"]["status"]["enum"] == ["pending", "in_progress", "completed"] + assert "context" not in schema["properties"] + + +# ── file tools ────────────────────────────────────────────────────────── + + +def test_file_tools_round_trip(workspace): + c = ctx() + assert ( + write_file.__wrapped__(path="/notes/a.md", content="one\ntwo\nthree\n", context=c)["bytes"] + == 14 + ) + assert (workspace / "exec-1" / "notes" / "a.md").read_text() == "one\ntwo\nthree\n" + + got = read_file.__wrapped__(path="notes/a.md", context=c) + assert got["content"] == "one\ntwo\nthree" + assert got["total_lines"] == 3 + + got = read_file.__wrapped__(path="notes/a.md", offset=1, limit=1, context=c) + assert got["content"] == "two" + assert got["returned_lines"] == 1 + + assert ( + edit_file.__wrapped__(path="notes/a.md", old_string="two", new_string="2", context=c)[ + "replacements" + ] + == 1 + ) + assert read_file.__wrapped__(path="notes/a.md", context=c)["content"] == "one\n2\nthree" + + listing = ls.__wrapped__(path="/", context=c) + assert [(e["name"], e["type"]) for e in listing["entries"]] == [("notes", "dir")] + listing = ls.__wrapped__(path="notes", context=c) + assert listing["entries"][0]["name"] == "a.md" + assert listing["entries"][0]["size"] == 12 + + +def test_edit_file_requires_a_unique_match(workspace): + c = ctx() + write_file.__wrapped__(path="x.txt", content="a b a", context=c) + out = edit_file.__wrapped__(path="x.txt", old_string="a", new_string="z", context=c) + assert out["replacements"] == 0 and "2 times" in out["error"] + out = edit_file.__wrapped__( + path="x.txt", old_string="a", new_string="z", replace_all=True, context=c + ) + assert out["replacements"] == 2 + assert read_file.__wrapped__(path="x.txt", context=c)["content"] == "z b z" + + +def test_missing_file_is_reported_not_raised(workspace): + c = ctx() + assert read_file.__wrapped__(path="nope.txt", context=c)["error"] == "no such file" + assert ( + edit_file.__wrapped__(path="nope.txt", old_string="a", new_string="b", context=c)["error"] + == "no such file" + ) + assert ls.__wrapped__(path="nope", context=c)["error"] == "no such directory" + + +def test_paths_cannot_escape_the_workspace(workspace): + c = ctx() + with pytest.raises(ValueError, match="outside"): + write_file.__wrapped__(path="../escape.txt", content="x", context=c) + with pytest.raises(ValueError, match="outside"): + read_file.__wrapped__(path="/notes/../../escape.txt", context=c) + assert not (workspace / "escape.txt").exists() + + +def test_runs_get_separate_directories(workspace): + write_file.__wrapped__(path="r.md", content="one", context=ctx("exec-1")) + write_file.__wrapped__(path="r.md", content="two", context=ctx("exec-2")) + assert (workspace / "exec-1" / "r.md").read_text() == "one" + assert (workspace / "exec-2" / "r.md").read_text() == "two" + + +def test_no_context_uses_a_local_directory(workspace): + write_file.__wrapped__(path="r.md", content="x", context=None) + assert (workspace / "local" / "r.md").exists() + + +# ── root execution id ─────────────────────────────────────────────────── + + +class FakeWorkflowClient: + def __init__(self, parents): + self.parents = parents + self.calls = [] + + def get_workflow(self, workflow_id, include_tasks=False): + self.calls.append(workflow_id) + if workflow_id not in self.parents: + raise RuntimeError("not found") + wf = type("WF", (), {})() + wf.parent_workflow_id = self.parents[workflow_id] + return wf + + +def test_root_execution_id_walks_the_parent_chain(): + client = FakeWorkflowClient({"child": "mid", "mid": "root", "root": None}) + assert resolve_root_execution_id("child", client) == "root" + assert resolve_root_execution_id("root", client) == "root" + + +def test_root_execution_id_falls_back_on_lookup_failure(): + client = FakeWorkflowClient({"child": "gone"}) + # The parent exists as an id but cannot be fetched; keep the furthest id reached. + assert resolve_root_execution_id("child", client) == "gone" + assert resolve_root_execution_id("unknown", client) == "unknown" + + +def test_root_execution_id_stops_on_a_cycle(): + client = FakeWorkflowClient({"a": "b", "b": "a"}) + assert resolve_root_execution_id("a", client) in ("a", "b") + assert len(client.calls) <= 3 + + +# ── deep_agent ────────────────────────────────────────────────────────── + + +def names(agent): + return [get_tool_def(t).name for t in agent.tools] + + +def test_deep_agent_assembles_tools_and_prompt(tmp_path, monkeypatch): + from conductor.ai.agents import tool + + @tool + def get_weather(city: str) -> dict: + """Weather.""" + return {"city": city} + + analyst = Agent( + name="weather_analyst", model="openai/gpt-4o-mini", instructions="Compare cities." + ) + agent = deep_agent( + name="planner", + model="openai/gpt-4o-mini", + instructions="You are a travel assistant.", + tools=[get_weather], + subagents=[analyst], + workspace=tmp_path / "ws", + approval=["write_file"], + ) + + assert isinstance(agent, Agent) + assert names(agent) == [ + "write_todos", + "ls", + "read_file", + "write_file", + "edit_file", + "get_weather", + "weather_analyst", + ] + assert agent.max_turns == 50 + assert agent.instructions.startswith(DEFAULT_INSTRUCTIONS) + assert "You are a travel assistant." in agent.instructions + assert __import__("os").environ[WORKSPACE_ENV] == str((tmp_path / "ws").resolve()) + + by_name = {get_tool_def(t).name: get_tool_def(t) for t in agent.tools} + assert by_name["write_file"].approval_required is True + assert by_name["write_file"].func is not None + assert by_name["edit_file"].approval_required is False + assert by_name["weather_analyst"].tool_type == "agent_tool" + + # The sub-agent shares the workspace but not the plan. + sub_names = names(analyst) + assert "write_todos" not in sub_names + assert set(("ls", "read_file", "write_file", "edit_file")) <= set(sub_names) + + +def test_deep_agent_applies_approval_to_user_tools(): + from conductor.ai.agents import tool + + @tool + def send_email(to: str) -> dict: + """Send.""" + return {"to": to} + + agent = deep_agent(name="a", model="m", tools=[send_email], approval=["send_email"]) + td = {get_tool_def(t).name: get_tool_def(t) for t in agent.tools}["send_email"] + assert td.approval_required is True + + +def test_deep_agent_without_instructions_uses_the_default_prompt_and_date(): + agent = deep_agent(name="a", model="m") + assert agent.instructions.startswith(DEFAULT_INSTRUCTIONS) + assert "Today is" in agent.instructions + assert "## Your task" not in agent.instructions + assert names(agent) == ["write_todos", "ls", "read_file", "write_file", "edit_file"] From ccbf739d5386b48170c5e89bce829acb9f0461a1 Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Fri, 20 Nov 2026 14:31:54 -0800 Subject: [PATCH 2/4] feat(agents): store the deep agent workspace through the file API Sub-agents get their own directories, retried writes are idempotent, and a coding agent example is added. --- examples/agents/deep_agent_coding.py | 199 +++++++++ examples/agents/deep_agent_research.py | 32 +- src/conductor/ai/agents/deep.py | 253 +++++++---- src/conductor/ai/agents/deep_files.py | 335 +++++++++++++++ src/conductor/ai/agents/runtime/_dispatch.py | 8 +- src/conductor/ai/agents/tool.py | 3 + src/conductor/client/file_client.py | 420 +++++++++++++++++++ tests/unit/ai/test_deep_agent.py | 90 +++- tests/unit/ai/test_deep_files.py | 241 +++++++++++ tests/unit/test_file_client.py | 314 ++++++++++++++ 10 files changed, 1781 insertions(+), 114 deletions(-) create mode 100644 examples/agents/deep_agent_coding.py create mode 100644 src/conductor/ai/agents/deep_files.py create mode 100644 src/conductor/client/file_client.py create mode 100644 tests/unit/ai/test_deep_files.py create mode 100644 tests/unit/test_file_client.py diff --git a/examples/agents/deep_agent_coding.py b/examples/agents/deep_agent_coding.py new file mode 100644 index 00000000..850a09da --- /dev/null +++ b/examples/agents/deep_agent_coding.py @@ -0,0 +1,199 @@ +"""Deep agent — build a small module from a specification until its tests pass. + +A coding agent of the kind people use every day, built from the deep agent +harness. The main agent plans, delegates the tests to a test-writer sub-agent +that never sees the implementation, writes the code, runs the tests and fixes +the code until they pass, then asks a reviewer sub-agent for a review. + +Demonstrates: + - A plan the main agent keeps up to date with write_todos + - Sub-agents with their own context: the test writer sees only the + specification, the reviewer sees only the code + - One shared workspace stored through Conductor's file API: the tests run in + a separate worker process that pulls the code and the tests from the + server, not from a local disk + - Ownership by structure: the tests live in the test writer's directory, and + the main agent cannot change them, only make them pass + - Long output kept out of the conversation: the full test log goes to a file + and the model gets a short summary + +Requirements: + - Conductor server with LLM support + - conductor.file-storage.enabled=true on the server, and its GET /api/files listing + - CONDUCTOR_SERVER_URL=http://localhost:8080/api as environment variable + - CONDUCTOR_AGENT_LLM_MODEL=openai/gpt-4o-mini as environment variable + - pytest installed in the Python environment that runs this script + +The tests run in a local subprocess with no sandbox. Use a container-based +executor before running code you do not trust. +""" + +import os +import posixpath +import subprocess +import sys +import tempfile +from pathlib import Path + +from conductor.ai.agents import Agent, AgentRuntime, EventType, tool +from conductor.ai.agents import deep_files +from conductor.ai.agents.deep import deep_agent, export_run, list_run_files, read_run_file +from conductor.ai.agents.tool import ToolContext +from settings import settings + +SPECIFICATION = """\ +Implement a Python module named `semver` in /src/semver.py with three functions. + +parse(text: str) -> tuple + Parse a semantic version such as "1.2.3", "1.2.3-alpha.1" or "1.2.3+build.5". + Return (major, minor, patch, prerelease), where major, minor and patch are + ints and prerelease is a tuple of identifiers: an identifier made only of + digits is an int, any other identifier is a str. prerelease is () when there + is none. Build metadata after "+" is accepted and ignored. + Raise ValueError when the text is not a valid version: missing or extra + numeric parts, an empty identifier, a numeric part or numeric prerelease + identifier with a leading zero (such as "01"), or characters other than + ASCII letters, digits, "-" and ".". + +compare(a: str, b: str) -> int + Return -1, 0 or 1 by semantic version precedence. Compare major, minor and + patch numerically. A version with a prerelease is lower than the same + version without one. Compare prerelease identifiers left to right: numeric + identifiers numerically, other identifiers in ASCII order, and a numeric + identifier is lower than a non-numeric one. When all compared identifiers + are equal, the version with fewer identifiers is lower. Build metadata does + not affect precedence. + +bump(text: str, part: str) -> str + part is "major", "minor" or "patch". Increment that part, set the parts to + its right to 0, and drop the prerelease and build metadata. + bump("1.2.3-rc.1", "minor") == "1.3.0". Raise ValueError for any other part. +""" + + +@tool +def run_tests(context: ToolContext = None) -> dict: + """Run the project's tests: every test_*.py file in the workspace, against the code in /src. + + Returns: + Whether every test passed, pytest's summary line, the end of the output + when something failed, and the path of the file with the full log. + """ + with tempfile.TemporaryDirectory() as tmp: + exported = deep_files.export_files(context, tmp) + tests = [ + p + for p in exported + if posixpath.basename(p).startswith("test_") and p.endswith(".py") + ] + if not tests: + return {"passed": False, "error": "no test_*.py files in the workspace yet"} + proc = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "--no-header", "-p", "no:cacheprovider", + "--import-mode=importlib", *tests], + cwd=tmp, + env={**os.environ, "PYTHONPATH": str(Path(tmp) / "src")}, + capture_output=True, + text=True, + timeout=120, + ) + log = proc.stdout + proc.stderr + lines = [line for line in log.splitlines() if line.strip()] + log_path = f"test-runs/run-{(context.task_id if context else 'local')[:8]}.log" + deep_files.backend().write(context, log_path, log.encode("utf-8")) + result = { + "passed": proc.returncode == 0, + "summary": lines[-1] if lines else "", + "tests": ["/" + t for t in tests], + "log": "/" + log_path, + } + if proc.returncode != 0: + result["output_tail"] = log[-2500:] + return result + + +test_writer = Agent( + name="test_writer", + model=settings.llm_model, + instructions=( + "You write pytest tests from a specification. You never see the " + "implementation, so test the behaviour the specification promises, " + "including its edge cases and its errors. Import the module under test " + "with `from semver import parse, compare, bump`. Write 15 to 25 focused " + "tests to test_semver.py with write_file. Do not implement the module." + ), + metadata={ + "description": ( + "Writes pytest tests for a specification without seeing the " + "implementation. Give it the full specification." + ) + }, +) + +code_reviewer = Agent( + name="code_reviewer", + model=settings.llm_model, + instructions=( + "You review code against a specification. Read the file you are given " + "with read_file. Look for behaviour that contradicts the specification, " + "cases the code does not handle, and code that is hard to follow. Write " + "your findings to review.md, most important first, and reply with the " + "three most important findings and the path of review.md." + ), + metadata={ + "description": ( + "Reviews a source file against a specification. Give it the path of " + "the file and the full specification." + ) + }, +) + +agent = deep_agent( + name="coding_agent", + model=settings.llm_model, + instructions=( + "You are a software engineer. Build what the user's specification " + "describes, in this order:\n" + "1. Ask test_writer for the tests, giving it the full specification.\n" + "2. Write the implementation to the path the specification names.\n" + "3. Call run_tests. While tests fail, read the failures, fix the " + "implementation, and run them again. You cannot change the tests. If you " + "are certain a test contradicts the specification, stop fixing and say " + "which test and why.\n" + "4. Ask code_reviewer to review the implementation against the " + "specification. Apply fixes for real defects it finds and run the tests " + "again.\n" + "5. Reply with the final test result, what you changed after review, and " + "anything you left open." + ), + tools=[run_tests], + subagents=[test_writer, code_reviewer], + max_turns=40, +) + + +if __name__ == "__main__": + with AgentRuntime() as runtime: + handle = runtime.start(agent, f"Build this.\n\n{SPECIFICATION}") + print(f"Started: {handle.execution_id}\n") + + for event in handle.stream(): + if event.type == EventType.TOOL_CALL: + print(f" [tool_call] {event.tool_name}({str(event.args)[:100]})") + elif event.type == EventType.TOOL_RESULT: + print(f" [tool_result] {event.tool_name} -> {str(event.result)[:160]}") + elif event.type == EventType.DONE: + print(f"\nDone: {event.output}") + + print("\nWorkspace files:") + for path, entry in list_run_files(handle.execution_id).items(): + print(f" /{path} {entry.size} bytes written by {entry.owner[:8]}") + + out = Path("workspace") / f"coding-{handle.execution_id}" + export_run(handle.execution_id, out) + print(f"\nExported the workspace to {out.resolve()}") + try: + print("--- /src/semver.py ---") + print(read_run_file(handle.execution_id, "/src/semver.py")) + except FileNotFoundError: + print("(no /src/semver.py)") diff --git a/examples/agents/deep_agent_research.py b/examples/agents/deep_agent_research.py index 4ad482b2..c5c5b5d5 100644 --- a/examples/agents/deep_agent_research.py +++ b/examples/agents/deep_agent_research.py @@ -2,18 +2,20 @@ A travel assistant that plans with write_todos, hands weather lookups to a weather-analyst sub-agent, asks for approval before writing a file, and writes -its recommendation to report.md in the run's workspace. +its recommendation to report.md in the run's workspace. The workspace is stored +through Conductor's file API, so it works across machines. Demonstrates: - deep_agent(): one call gives an Agent with planning, file tools and sub-agents - The plan lives in _agent_state, outside the conversation - - A sub-agent with its own context, sharing the run's workspace - - Human approval before write_file + - A sub-agent with its own context and its own directory in the run's workspace + - Human approval before write_file, for the sub-agent too Requirements: - Conductor server with LLM support - CONDUCTOR_SERVER_URL=http://localhost:8080/api as environment variable - CONDUCTOR_AGENT_LLM_MODEL=openai/gpt-4o-mini as environment variable + - conductor.file-storage.enabled=true on the server, and its GET /api/files listing - Network access to open-meteo.com for the weather tool """ @@ -22,7 +24,7 @@ import urllib.request from conductor.ai.agents import Agent, AgentRuntime, EventType, tool -from conductor.ai.agents.deep import deep_agent, workspace_dir +from conductor.ai.agents.deep import deep_agent, list_run_files, read_run_file from settings import settings _WMO = { @@ -100,8 +102,8 @@ def get_weather(city: str) -> str: model=settings.llm_model, instructions=( "You are a weather analyst. Call get_weather for each city you are given. " - "Write the full forecast for each city to weather-.md in the workspace " - "with write_file, then reply with a two-sentence summary and the file paths." + "Write the full forecast for each city to weather-.md with write_file, " + "then reply with a two-sentence summary and the file paths." ), tools=[get_weather], metadata={ @@ -118,7 +120,6 @@ def get_weather(city: str) -> str: "and tell the user what you chose and why." ), subagents=[weather_analyst], - workspace="./workspace", approval=["write_file"], ) @@ -137,14 +138,17 @@ def get_weather(city: str) -> str: elif event.type == EventType.TOOL_RESULT: print(f" [tool_result] {event.tool_name} -> {str(event.result)[:120]}") elif event.type == EventType.WAITING: - print(" [waiting] approving write_file") - handle.approve() + # Pass the event: a sub-agent's approval waits in its own execution. + print(f" [waiting] approving write_file in {event.execution_id[:8]}") + handle.approve(event=event) elif event.type == EventType.DONE: print(f"\nDone: {event.output}") - ws = workspace_dir(handle.execution_id, "./workspace") - report = ws / "report.md" - print(f"\nWorkspace: {ws}") - if report.exists(): + print("\nWorkspace files:") + for path, entry in list_run_files(handle.execution_id).items(): + print(f" /{path} {entry.size} bytes written by {entry.owner[:8]}") + try: print("--- report.md ---") - print(report.read_text()) + print(read_run_file(handle.execution_id, "/report.md")) + except FileNotFoundError: + print("(no report.md)") diff --git a/src/conductor/ai/agents/deep.py b/src/conductor/ai/agents/deep.py index 62261879..15a92bab 100644 --- a/src/conductor/ai/agents/deep.py +++ b/src/conductor/ai/agents/deep.py @@ -5,13 +5,14 @@ - the plan, in ``_agent_state`` through :func:`write_todos`, private to the agent that wrote it and shown to it on every turn; -- large data, in files under a per-run workspace directory, shared with the - sub-agents of the same run; +- large data, in files of a run-scoped workspace shared with the sub-agents of + the same run, stored through Conductor's file API (see + :mod:`conductor.ai.agents.deep_files`); - delegated work, in sub-agents that return a short result and never see the parent's plan or history. -This is a proof of concept on the Python SDK alone: the todo and file tools are -ordinary ``@tool`` workers. See ``examples/agents/deep_agent_research.py``. +This is a proof of concept: the todo and file tools are ordinary ``@tool`` +workers. See ``examples/agents/deep_agent_research.py``. Example:: @@ -30,27 +31,30 @@ from __future__ import annotations +import copy import dataclasses +import json import datetime as dt import os import threading from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Sequence, Union +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union +from conductor.ai.agents import deep_files from conductor.ai.agents.agent import Agent +from conductor.ai.agents.deep_files import ( + DEFAULT_WORKSPACE, + STORAGE_ENV, + SUBAGENTS_ENV, + WORKSPACE_ENV, + export_run, + list_run_files, + read_run_file, +) from conductor.ai.agents.tool import ToolContext, agent_tool, get_tool_def, tool from conductor.client.configuration.configuration import Configuration from conductor.client.orkes_clients import OrkesClients -WORKSPACE_ENV = "CONDUCTOR_DEEP_AGENT_WORKSPACE" -"""Environment variable naming the workspace root directory. - -Tool workers run in separate processes, so the root is passed through the -environment rather than captured in a closure. -""" - -DEFAULT_WORKSPACE = "./workspace" - TODOS_KEY = "todos" """Key under which the plan lives in ``ToolContext.state`` (``_agent_state``). @@ -77,19 +81,23 @@ task. Your current plan is shown to you under "Context" on every turn. ## Files -The workspace is private to this run. Paths are relative to it; "/report.md" -and "report.md" are the same file. Write anything long or anything worth -keeping to a file instead of repeating it in the conversation: research notes, -drafts, the final report. Read a file only when you need its content. Use -edit_file for small changes; use write_file to create or replace a whole file. -Use the file names your instructions ask for. +Every agent in this run shares one file workspace. Paths that start with "/" +are absolute; a relative path is resolved from your own directory, which for you +is the root, so "/report.md" and "report.md" are the same file. Write anything +long or anything worth keeping to a file instead of repeating it in the +conversation: research notes, drafts, the final report. Read a file only when +you need its content. Use edit_file for small changes; use write_file to create +or replace a whole file. Use the file names your instructions ask for. Sub-agents +write in their own directories under /subagents/; read their results from the +paths they return. A sub-agent's directory belongs to it: you can read it but +not change it. If two agents write the same path, the last write wins. ## Sub-agents Each sub-agent is a tool. Delegate a self-contained piece of work with a clear request and everything it needs to know; it does not see your plan or this -conversation. Ask it to write long results to a file in the workspace and to -return a short summary plus the file path. Delegate when a step would otherwise -fill your context with material you only need a summary of. +conversation. It writes long results to a file in its own directory and returns +a short summary plus the file path. Delegate when a step would otherwise fill +your context with material you only need a summary of. ## Finishing When every step is completed, give the user the result they asked for in your @@ -99,6 +107,17 @@ the plan to work around it. """ +SUBAGENT_INSTRUCTIONS = """\ +## Files +You are a sub-agent. You share a file workspace with the agent that called you, +and you have your own directory in it. A relative path such as "notes.md" is +inside your directory, and you can only write there. Read other files with +absolute paths such as "/report.md". Put long results in a file in your +directory. The calling agent sees only your final message, so finish with a +short summary and the full path of every file you wrote, as write_file returned +it. +""" + # ── Plan ──────────────────────────────────────────────────────────────── @@ -189,6 +208,10 @@ def stored_todos(state: Optional[Dict[str, Any]]) -> Optional[List[Dict[str, str # ── Workspace ─────────────────────────────────────────────────────────── +# Used by the local backend, which keys a run's directory by its root execution +# id. The conductor backend needs none of this: the server scopes file access to +# the workflow family. + _root_cache: Dict[str, str] = {} _root_lock = threading.Lock() @@ -242,50 +265,54 @@ def _run_id(context: Optional[ToolContext]) -> str: return _root_execution_id(context.execution_id) -def _workspace_root(context: Optional[ToolContext]) -> Path: - base = Path(os.environ.get(WORKSPACE_ENV) or DEFAULT_WORKSPACE) - root = (base / _run_id(context)).resolve() - root.mkdir(parents=True, exist_ok=True) - return root +# ── File tools ────────────────────────────────────────────────────────── -def _resolve(root: Path, path: str) -> Path: - """Map a model-supplied path onto the workspace, refusing escapes.""" - rel = (path or "").strip().lstrip("/") - target = (root / rel).resolve() if rel else root - if target != root and root not in target.parents: - raise ValueError(f"path {path!r} is outside the workspace") - return target +def _display(path: str) -> str: + return "/" + path -# ── File tools ────────────────────────────────────────────────────────── +def _writer(entry: deep_files.FileEntry, context: Optional[ToolContext]) -> str: + if not entry.owner: + return "" + if context is not None and entry.owner == context.execution_id: + return "you" + return f"another agent ({entry.owner[:8]})" @tool -def ls(path: str = "/", context: ToolContext = None) -> dict: +def ls(path: str = "", context: ToolContext = None) -> dict: """List the files and directories at a path in the workspace. Args: - path: Directory to list, relative to the workspace root. Default is the root. + path: Directory to list. Relative paths start at your own directory; + "/" is the root of the workspace. Default is your own directory. Returns: - The entries with their type and size in bytes. + The entries with their type, size in bytes, and who wrote each file. """ - root = _workspace_root(context) - target = _resolve(root, path) - if not target.exists(): - return {"path": path, "entries": [], "error": "no such directory"} - if not target.is_dir(): - return {"path": path, "entries": [], "error": "not a directory"} - entries = [ - { - "name": p.name, - "type": "dir" if p.is_dir() else "file", - "size": p.stat().st_size if p.is_file() else 0, - } - for p in sorted(target.iterdir()) - ] - return {"path": path, "entries": entries} + target = deep_files.resolve(context, path) + files = deep_files.backend().list(context) + if target in files: + return {"path": _display(target), "entries": [], "error": "not a directory"} + prefix = target + "/" if target else "" + entries: Dict[str, dict] = {} + for p, entry in files.items(): + if not p.startswith(prefix): + continue + name, _, rest = p[len(prefix) :].partition("/") + if rest: + entries.setdefault(name, {"name": name, "type": "dir", "size": 0}) + else: + entries[name] = { + "name": name, + "type": "file", + "size": entry.size, + "written_by": _writer(entry, context), + } + if not entries and target: + return {"path": _display(target), "entries": [], "error": "no such directory"} + return {"path": _display(target), "entries": [entries[k] for k in sorted(entries)]} @tool @@ -293,21 +320,21 @@ def read_file(path: str, offset: int = 0, limit: int = 2000, context: ToolContex """Read a text file from the workspace. Args: - path: File path relative to the workspace root. + path: File path. Relative paths start at your own directory. offset: First line to return, counting from 0. limit: Maximum number of lines to return. Returns: The requested lines and the file's total line count. """ - root = _workspace_root(context) - target = _resolve(root, path) - if not target.is_file(): - return {"path": path, "error": "no such file"} - lines = target.read_text(encoding="utf-8").splitlines() + target = deep_files.resolve(context, path) + entry = deep_files.backend().list(context).get(target) + if entry is None: + return {"path": _display(target), "error": "no such file"} + lines = deep_files.backend().read(context, entry).decode("utf-8").splitlines() chunk = lines[offset : offset + limit] return { - "path": path, + "path": _display(target), "content": "\n".join(chunk), "total_lines": len(lines), "offset": offset, @@ -315,25 +342,28 @@ def read_file(path: str, offset: int = 0, limit: int = 2000, context: ToolContex } +def _store(context: Optional[ToolContext], target: str, text: str) -> dict: + entry, replaced = deep_files.backend().write(context, target, text.encode("utf-8")) + out: Dict[str, Any] = {"path": _display(target), "bytes": entry.size} + if replaced: + out["note"] = f"This replaced a version written by another agent ({replaced[:8]})." + return out + + @tool def write_file(path: str, content: str, context: ToolContext = None) -> dict: """Create or replace a text file in the workspace. Args: - path: File path relative to the workspace root. Parent directories are created. + path: File path. Relative paths start at your own directory. content: The full file content. Returns: - The path written and its size in bytes. + The absolute path written and its size in bytes. """ - root = _workspace_root(context) - target = _resolve(root, path) - if target == root: - raise ValueError("path must name a file") - target.parent.mkdir(parents=True, exist_ok=True) - data = content.encode("utf-8") - target.write_bytes(data) - return {"path": path, "bytes": len(data)} + target = deep_files.resolve(context, path) + deep_files.check_writable(context, target) + return _store(context, target, content) @tool @@ -347,7 +377,7 @@ def edit_file( """Replace text in a workspace file. Args: - path: File path relative to the workspace root. + path: File path. Relative paths start at your own directory. old_string: The exact text to find. It must occur once unless replace_all is set. new_string: The replacement text. replace_all: Replace every occurrence instead of requiring a unique match. @@ -355,19 +385,25 @@ def edit_file( Returns: The path edited and how many replacements were made. """ - root = _workspace_root(context) - target = _resolve(root, path) - if not target.is_file(): - return {"path": path, "error": "no such file"} + target = deep_files.resolve(context, path) + deep_files.check_writable(context, target) if not old_string: raise ValueError("old_string is empty") - text = target.read_text(encoding="utf-8") + store = deep_files.backend() + done = store.written_by_task(context, target) + if done is not None: + # A retried delivery of a call that already applied its edit. + return {"path": _display(target), "replacements": None, "note": "already applied"} + entry = store.list(context).get(target) + if entry is None: + return {"path": _display(target), "error": "no such file"} + text = store.read(context, entry).decode("utf-8") count = text.count(old_string) if count == 0: - return {"path": path, "error": "old_string not found", "replacements": 0} + return {"path": _display(target), "error": "old_string not found", "replacements": 0} if count > 1 and not replace_all: return { - "path": path, + "path": _display(target), "error": f"old_string occurs {count} times; include more context or set replace_all", "replacements": 0, } @@ -376,8 +412,9 @@ def edit_file( if replace_all else text.replace(old_string, new_string, 1) ) - target.write_text(updated, encoding="utf-8") - return {"path": path, "replacements": count if replace_all else 1} + out = _store(context, target, updated) + out["replacements"] = count if replace_all else 1 + return out FILE_TOOLS = (ls, read_file, write_file, edit_file) @@ -416,6 +453,24 @@ def _describe(sub: Agent) -> str: return f"Delegate a self-contained task to the {sub.name} agent. Give it everything it needs." +def _as_subagent(sub: Agent, approval: Tuple[str, ...]) -> Agent: + """A copy of *sub* with the file tools and sub-agent file rules added. + + The caller's object is left unchanged, so the same agent can be used + elsewhere without the harness tools. + """ + copied = copy.copy(sub) + existing = _tool_names(sub.tools or []) + missing = [ft for ft in FILE_TOOLS if get_tool_def(ft).name not in existing] + copied.tools = [ + *(_with_approval(t, approval) for t in sub.tools or []), + *(_with_approval(t, approval) for t in missing), + ] + if isinstance(sub.instructions, str): + copied.instructions = f"{sub.instructions.rstrip()}\n\n{SUBAGENT_INSTRUCTIONS}" + return copied + + def deep_agent( name: str, *, @@ -423,6 +478,7 @@ def deep_agent( instructions: str = "", tools: Optional[Sequence[Any]] = None, subagents: Optional[Sequence[Agent]] = None, + storage: str = "conductor", workspace: Optional[Union[str, Path]] = None, approval: Iterable[str] = (), max_turns: int = 50, @@ -436,19 +492,32 @@ def deep_agent( instructions: Your instructions. They follow the harness's default prompt, which explains planning, files and delegation. tools: Your own tools, in addition to the harness tools. - subagents: Agents to expose as tools. Each gets the file tools so it - can write to the shared workspace, and does not get ``write_todos``. - workspace: Root directory for run workspaces. Each run gets a + subagents: Agents to expose as tools. Each is copied and the copy gets + the file tools and a directory of its own in the shared workspace. + Sub-agents do not get ``write_todos``. + storage: Where the workspace lives: ``"conductor"`` (default) uses + Conductor's file API, which works across machines and needs + ``conductor.file-storage.enabled=true`` on the server; + ``"local"`` uses a directory on the worker's disk, for development. + workspace: Root directory for the ``local`` storage. Each run gets a sub-directory named by its root execution id. Defaults to ``./workspace``. approval: Names of tools that pause for human approval before they - run, for example ``["write_file"]``. + run, for example ``["write_file"]``. Applies to sub-agents too. max_turns: Turn limit for the agent loop; deep agents need more than the ``Agent`` default. **agent_kwargs: Passed through to :class:`Agent`. """ + if storage not in ("conductor", "local"): + raise ValueError(f"storage must be 'conductor' or 'local', not {storage!r}") + # Tool workers run in separate processes and read their configuration from + # the environment, so set it before the runtime starts them. + os.environ[STORAGE_ENV] = storage if workspace is not None: os.environ[WORKSPACE_ENV] = str(Path(workspace).resolve()) + known = set(deep_files.subagent_names()) + known.update(sub.name for sub in subagents or ()) + os.environ[SUBAGENTS_ENV] = json.dumps(sorted(known)) approval = tuple(approval) agent_tools: List[Any] = [write_todos] @@ -456,11 +525,9 @@ def deep_agent( agent_tools.extend(_with_approval(t, approval) for t in tools or ()) for sub in subagents or (): - existing = _tool_names(sub.tools or []) - missing = [ft for ft in FILE_TOOLS if get_tool_def(ft).name not in existing] - if missing: - sub.tools = [*(sub.tools or []), *missing] - agent_tools.append(agent_tool(sub, name=sub.name, description=_describe(sub))) + agent_tools.append( + agent_tool(_as_subagent(sub, approval), name=sub.name, description=_describe(sub)) + ) # The model's idea of today comes from its training data, which makes it # argue with dated tool results; tell it the date. @@ -492,12 +559,18 @@ def workspace_dir(execution_id: str, workspace: Optional[Union[str, Path]] = Non __all__ = [ "DEFAULT_INSTRUCTIONS", "FILE_TOOLS", + "STORAGE_ENV", + "SUBAGENTS_ENV", + "SUBAGENT_INSTRUCTIONS", "TODOS_KEY", "WORKSPACE_ENV", "deep_agent", "edit_file", + "export_run", + "list_run_files", "ls", "read_file", + "read_run_file", "resolve_root_execution_id", "stored_todos", "workspace_dir", diff --git a/src/conductor/ai/agents/deep_files.py b/src/conductor/ai/agents/deep_files.py new file mode 100644 index 00000000..e024bda6 --- /dev/null +++ b/src/conductor/ai/agents/deep_files.py @@ -0,0 +1,335 @@ +"""Run-scoped file storage behind the deep agent's file tools. + +Every agent in a run sees one file tree. Two backends implement it: + +- ``conductor`` (default): Conductor's file API. Each write uploads a new + immutable file owned by the writing agent's execution, with the logical path + in the file name. A path resolves to its newest version, so the last writer + wins. The server lets a workflow read files owned by its ancestors and + descendants, so a parent sees its sub-agents' files and a sub-agent sees its + parent's. Sibling sub-agents do not see each other's files. Needs + ``conductor.file-storage.enabled=true`` and the ``GET /api/files`` listing. +- ``local``: a directory per run on the worker's disk. For development only: + workers on different machines do not share it. + +Paths: ``/a/b.md`` is run-global. A relative path resolves to the calling +agent's home directory: the root for the top-level agent, and +``subagents/-`` for a sub-agent. A sub-agent may write only inside +its home directory; it may read any path it can see. +""" + +from __future__ import annotations + +import json +import os +import posixpath +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import quote, unquote + +from conductor.ai.agents.tool import ToolContext + +STORAGE_ENV = "CONDUCTOR_DEEP_AGENT_STORAGE" +"""Selects the backend: ``conductor`` (default) or ``local``.""" + +SUBAGENTS_ENV = "CONDUCTOR_DEEP_AGENT_SUBAGENTS" +"""JSON list of sub-agent names, set by ``deep_agent()``. + +Tool workers run in separate processes, so configuration travels through the +environment rather than a closure. +""" + +WORKSPACE_ENV = "CONDUCTOR_DEEP_AGENT_WORKSPACE" +"""Root directory for the ``local`` backend.""" + +DEFAULT_WORKSPACE = "./workspace" + + +@dataclass +class FileEntry: + """The newest version of one logical path.""" + + path: str + size: int + owner: str = "" + task_id: str = "" + created_at: int = 0 + handle: Optional[str] = None + + +# ── paths ──────────────────────────────────────────────────────────────── + + +def normalize(path: str) -> str: + """Normalize a path to ``a/b.md`` form, refusing escapes from the run tree.""" + raw = (path or "").strip().lstrip("/") + if not raw: + return "" + rel = posixpath.normpath(raw) + if rel == ".": + return "" + if rel == ".." or rel.startswith("../"): + raise ValueError(f"path {path!r} is outside the workspace") + return rel + + +def subagent_names() -> List[str]: + try: + names = json.loads(os.environ.get(SUBAGENTS_ENV) or "[]") + except ValueError: + return [] + return [n for n in names if isinstance(n, str)] + + +def home_of(context: Optional[ToolContext]) -> str: + """The calling agent's home directory: ``""`` for the top-level agent.""" + if context is None or context.agent_name not in subagent_names(): + return "" + return f"subagents/{context.agent_name}-{(context.execution_id or 'local')[:8]}" + + +def resolve(context: Optional[ToolContext], path: str) -> str: + """Resolve a tool path: absolute paths are run-global, relative ones start at home.""" + if (path or "").strip().startswith("/"): + return normalize(path) + return normalize(posixpath.join(home_of(context), (path or "").strip())) + + +SUBAGENTS_DIR = "subagents" + + +def check_writable(context: Optional[ToolContext], path: str) -> None: + """Refuse writes outside the caller's own area. + + A sub-agent writes only inside its home directory. Every other agent, + including the top-level one, may write anywhere except another sub-agent's + directory, so what a sub-agent produced stays as it wrote it. + """ + home = home_of(context) + if not path: + raise ValueError("path must name a file") + if home: + if path != home and not path.startswith(home + "/"): + raise ValueError( + f"a sub-agent may only write inside its own directory /{home}; " + f"use a relative path such as {posixpath.basename(path)!r}" + ) + elif path == SUBAGENTS_DIR or path.startswith(SUBAGENTS_DIR + "/"): + raise ValueError( + f"/{path} belongs to a sub-agent and only that sub-agent can change it; " + "write your own file instead" + ) + + +# ── backends ───────────────────────────────────────────────────────────── + + +class _Backend: + def list(self, context: Optional[ToolContext]) -> Dict[str, FileEntry]: + raise NotImplementedError + + def read(self, context: Optional[ToolContext], entry: FileEntry) -> bytes: + raise NotImplementedError + + def write( + self, context: Optional[ToolContext], path: str, data: bytes + ) -> Tuple[FileEntry, Optional[str]]: + """Store *data* at *path*. Returns the entry and the previous owner if it differs.""" + raise NotImplementedError + + def written_by_task(self, context: Optional[ToolContext], path: str) -> Optional[FileEntry]: + """The version of *path* this tool call already wrote, if it is a retried delivery.""" + return None + + +class LocalBackend(_Backend): + """One directory per run, keyed by the root execution id.""" + + def _root(self, context: Optional[ToolContext]) -> Path: + from conductor.ai.agents.deep import _run_id # root-id lookup lives with the harness + + base = Path(os.environ.get(WORKSPACE_ENV) or DEFAULT_WORKSPACE) + root = (base / _run_id(context)).resolve() + root.mkdir(parents=True, exist_ok=True) + return root + + def list(self, context): + root = self._root(context) + out = {} + for p in sorted(root.rglob("*")): + if p.is_file(): + rel = p.relative_to(root).as_posix() + out[rel] = FileEntry(path=rel, size=p.stat().st_size) + return out + + def read(self, context, entry): + return (self._root(context) / entry.path).read_bytes() + + def write(self, context, path, data): + target = self._root(context) / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + return FileEntry(path=path, size=len(data)), None + + +class ConductorBackend(_Backend): + """Conductor's file API. Every write is a new version; the newest one wins.""" + + def __init__(self): + self._client = None + self._lock = threading.Lock() + + def _files(self): + with self._lock: + if self._client is None: + from conductor.client.configuration.configuration import Configuration + from conductor.client.file_client import FileClient + + self._client = FileClient(Configuration()) + return self._client + + def _execution(self, context: Optional[ToolContext]) -> str: + if context is None or not context.execution_id: + raise ValueError("the conductor file backend needs a running execution") + return context.execution_id + + def versions(self, context: Optional[ToolContext]) -> List[Dict[str, Any]]: + """Every uploaded version visible to the caller, oldest first.""" + client = self._files() + items = client._api_client.call_api( # listing is not in FileClient yet + "/files", + "GET", + {}, + [("workflowId", self._execution(context))], + {"Accept": "application/json"}, + response_type="object", + auth_settings=["api_key"], + _return_http_data_only=True, + _preload_content=True, + ) + return list(items or []) + + def list(self, context): + out: Dict[str, FileEntry] = {} + for item in self.versions(context): + path = unquote(item.get("fileName") or "") + if not path: + continue + out[path] = _entry(path, item) # oldest first, so the newest version wins + return out + + def read(self, context, entry): + return self._files().download_bytes(self._execution(context), entry.handle) + + def written_by_task(self, context, path): + task_id = context.task_id if context else "" + if not task_id: + return None + execution = self._execution(context) + for item in self.versions(context): + if ( + unquote(item.get("fileName") or "") == path + and item.get("taskId") == task_id + and item.get("workflowId") == execution + ): + return _entry(path, item) + return None + + def write(self, context, path, data): + from conductor.client.file_client import FileUploadOptions + + execution = self._execution(context) + task_id = context.task_id if context else "" + previous = None + for item in self.versions(context): + if unquote(item.get("fileName") or "") != path: + continue + # A retried delivery of the same tool call: return what it already wrote. + if task_id and item.get("taskId") == task_id and item.get("workflowId") == execution: + return _entry(path, item), None + previous = item + handle = self._files().upload_bytes( + execution, + data, + FileUploadOptions( + file_name=quote(path, safe=""), + content_type="text/plain; charset=utf-8", + task_id=task_id or None, + ), + ) + entry = FileEntry( + path=path, size=len(data), owner=execution, task_id=task_id, handle=handle + ) + replaced = None + if previous is not None and previous.get("workflowId") != execution: + replaced = previous.get("workflowId") + return entry, replaced + + +def _entry(path: str, item: Dict[str, Any]) -> FileEntry: + return FileEntry( + path=path, + size=int(item.get("fileSize") or 0), + owner=item.get("workflowId") or "", + task_id=item.get("taskId") or "", + created_at=int(item.get("createdAt") or 0), + handle=item.get("fileHandleId"), + ) + + +_conductor_backend = ConductorBackend() +_local_backend = LocalBackend() + + +def backend() -> _Backend: + if (os.environ.get(STORAGE_ENV) or "conductor").strip().lower() == "local": + return _local_backend + return _conductor_backend + + +# ── helpers for code outside the agent ────────────────────────────────── + + +def list_run_files(execution_id: str) -> Dict[str, FileEntry]: + """Every file in a run, newest version per path. Pass the root execution id.""" + return backend().list(ToolContext(execution_id=execution_id)) + + +def read_run_file(execution_id: str, path: str) -> str: + """Read one file of a run as text. Pass the root execution id.""" + context = ToolContext(execution_id=execution_id) + entry = backend().list(context).get(normalize(path)) + if entry is None: + raise FileNotFoundError(path) + return backend().read(context, entry).decode("utf-8") + + +def export_files( + context: Optional[ToolContext], dest: "os.PathLike[str] | str", prefix: str = "" +) -> List[str]: + """Copy the newest version of every file the caller can see into *dest*. + + Paths keep their workspace layout under *dest*. Pass *prefix* (a workspace + path such as ``"/src"``) to export one directory only. Returns the exported + workspace paths. Used to run tools such as test runners over the workspace, + and to inspect a run's files on disk. + """ + root = Path(dest) + want = normalize(prefix) + store = backend() + exported = [] + for path, entry in store.list(context).items(): + if want and path != want and not path.startswith(want + "/"): + continue + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(store.read(context, entry)) + exported.append(path) + return exported + + +def export_run(execution_id: str, dest: "os.PathLike[str] | str", prefix: str = "") -> List[str]: + """Copy a run's files into *dest* for inspection. Pass the root execution id.""" + return export_files(ToolContext(execution_id=execution_id), dest, prefix) diff --git a/src/conductor/ai/agents/runtime/_dispatch.py b/src/conductor/ai/agents/runtime/_dispatch.py index 647e60ad..1058df60 100644 --- a/src/conductor/ai/agents/runtime/_dispatch.py +++ b/src/conductor/ai/agents/runtime/_dispatch.py @@ -328,7 +328,7 @@ def run_tool_task(task, *, tool_name, tool_func, guardrails=None, credential_nam from conductor.client.http.models import Task, TaskResult from conductor.client.http.models.task_result_status import TaskResultStatus - def _execute(kwargs, execution_id="", agent_state=None): + def _execute(kwargs, execution_id="", agent_state=None, agent_name="", task_id=""): """Core execution logic shared by both Task-based and kwargs-based paths.""" # Circuit breaker: disable tool after N consecutive failures if _tool_error_counts.get(tool_name, 0) >= _CIRCUIT_BREAKER_THRESHOLD: @@ -344,11 +344,13 @@ def _execute(kwargs, execution_id="", agent_state=None): state = dict(agent_state) if agent_state else {} ctx = ToolContext( execution_id=execution_id, - agent_name=_current_context.get("agent_name", ""), + # The task's workflow is the agent's own execution, named after the agent. + agent_name=agent_name or _current_context.get("agent_name", ""), session_id=_current_context.get("session_id", ""), metadata=_current_context.get("metadata", {}), dependencies=_current_context.get("dependencies", {}), state=state, + task_id=task_id, ) kwargs["context"] = ctx @@ -508,6 +510,8 @@ def _invoke_with_context(): fn_kwargs, execution_id=task.workflow_instance_id or "", agent_state=agent_state, + agent_name=task.workflow_type or "", + task_id=task.task_id or "", ) finally: if resolved_secrets: diff --git a/src/conductor/ai/agents/tool.py b/src/conductor/ai/agents/tool.py index e06a70f9..432d4e41 100644 --- a/src/conductor/ai/agents/tool.py +++ b/src/conductor/ai/agents/tool.py @@ -32,6 +32,8 @@ class ToolContext: agent_name: The name of the agent executing this tool. metadata: Arbitrary metadata from the agent. dependencies: User-provided dependencies (DB connections, API clients, etc.) + task_id: The Conductor task ID of this tool call. A retried delivery of the + same call keeps the same ID, so tools can use it to stay idempotent. """ session_id: str = "" @@ -40,6 +42,7 @@ class ToolContext: metadata: Dict[str, Any] = field(default_factory=dict) dependencies: Dict[str, Any] = field(default_factory=dict) state: Dict[str, Any] = field(default_factory=dict) + task_id: str = "" # ── ToolDef (serialisable tool definition) ────────────────────────────── diff --git a/src/conductor/client/file_client.py b/src/conductor/client/file_client.py new file mode 100644 index 00000000..314e6466 --- /dev/null +++ b/src/conductor/client/file_client.py @@ -0,0 +1,420 @@ +"""Client for Conductor file storage (``/api/files``). + +Workflows exchange files as opaque handles of the form ``conductor://file/``. +The server stores file records and enforces workflow ownership; the configured +backend (``conductor``, ``s3``, ``azure-blob`` or ``gcs``) stores the bytes. +Workers upload and download through short-lived URLs the server issues, so they +need no storage credentials. + +This is a port of the Java SDK ``FileClient``. It covers single-request uploads, +downloads and metadata. Multipart uploads, which only S3 and Azure Blob Storage +support, are not implemented yet; every backend accepts a single-request upload. + +Example:: + + from conductor.client.file_client import FileClient, FileUploadOptions + + files = FileClient(configuration) + handle = files.upload(workflow_id, "report.pdf") + files.download(workflow_id, handle, "/tmp/report.pdf") +""" + +from __future__ import annotations + +import os +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Dict, Optional, TypeVar, Union + +import requests + +from conductor.client.configuration.configuration import Configuration +from conductor.client.http.api_client import ApiClient +from conductor.client.http.rest import ApiException + +FILE_HANDLE_PREFIX = "conductor://file/" + +_MAX_FILENAME_LENGTH = 255 +_FILE_ID_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,255}") +_UNSAFE_FILENAME = re.compile(r'[\x00-\x1f:*?"<>|/\\]') +_DEFAULT_CONTENT_TYPE = "application/octet-stream" +_AUTH_SETTINGS = ["api_key"] +# Statuses worth retrying. A signed URL may have expired (401/403); the upload +# path asks the server for a fresh one before the next attempt. +_RETRYABLE_TRANSFER_STATUSES = {401, 403, 408, 425, 429} +_RETRYABLE_SERVER_STATUSES = {0, 408, 425, 429} +_SERVER_ERROR = 500 +_HTTP_OK_MIN, _HTTP_OK_MAX = 200, 300 + +T = TypeVar("T") + + +class FileStorageError(Exception): + """A file operation failed. The message never contains a signed URL.""" + + +class _TransferError(Exception): + def __init__(self, operation: str, status: Optional[int] = None): + super().__init__(f"{operation} failed" + (f" with HTTP {status}" if status else "")) + self.status = status + + +@dataclass +class FileUploadOptions: + """Optional metadata recorded with an upload.""" + + file_name: Optional[str] = None + content_type: Optional[str] = None + task_id: Optional[str] = None + + +@dataclass +class FileMetadata: + """Server-side metadata for an uploaded file.""" + + file_handle_id: str + file_name: Optional[str] = None + content_type: Optional[str] = None + file_size: Optional[int] = None + content_hash: Optional[str] = None + storage_type: Optional[str] = None + upload_status: Optional[str] = None + workflow_id: Optional[str] = None + task_id: Optional[str] = None + created_at: Optional[int] = None + updated_at: Optional[int] = None + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "FileMetadata": + return cls( + file_handle_id=data.get("fileHandleId"), + file_name=data.get("fileName"), + content_type=data.get("contentType"), + file_size=data.get("fileSize"), + content_hash=data.get("contentHash"), + storage_type=data.get("storageType"), + upload_status=data.get("uploadStatus"), + workflow_id=data.get("workflowId"), + task_id=data.get("taskId"), + created_at=data.get("createdAt"), + updated_at=data.get("updatedAt"), + ) + + +class FileClient: + """Upload, download and inspect files owned by workflow executions. + + Server calls go through the shared :class:`ApiClient`, so authentication is + the same as every other client. Transfers to the issued URLs use a plain + HTTP session with no Conductor credentials attached. + """ + + def __init__( + self, + configuration: Optional[Configuration] = None, + *, + retry_count: int = 3, + api_client: Optional[ApiClient] = None, + session: Optional[requests.Session] = None, + timeout: float = 60.0, + ): + if retry_count < 0: + raise FileStorageError("retry_count must not be negative") + self._api_client = api_client or ApiClient(configuration or Configuration()) + self._session = session or requests.Session() + self._retry_count = retry_count + self._timeout = timeout + + # ── public API ─────────────────────────────────────────────────────── + + def upload( + self, + workflow_id: str, + source: Union[str, os.PathLike], + options: Optional[FileUploadOptions] = None, + ) -> str: + """Upload a local file and return its ``conductor://file/`` handle.""" + _validate_workflow_id(workflow_id) + path = Path(source) + if not path.is_file() or not os.access(path, os.R_OK): + raise FileStorageError("Upload source must be a readable regular file") + resolved = _resolve_options(options, default_name=path.name) + return self._upload(workflow_id, resolved, lambda: path.read_bytes()) + + def upload_bytes(self, workflow_id: str, data: bytes, options: FileUploadOptions) -> str: + """Upload in-memory content. ``options.file_name`` is required.""" + _validate_workflow_id(workflow_id) + if data is None: + raise FileStorageError("Upload data is required") + if options is None or not options.file_name: + raise FileStorageError("A filename is required when uploading bytes") + resolved = _resolve_options(options, default_name=None) + payload = bytes(data) + return self._upload(workflow_id, resolved, lambda: payload) + + def download( + self, workflow_id: str, file_handle_id: str, destination: Union[str, os.PathLike] + ) -> Path: + """Download a file to *destination*, replacing it only after a complete transfer.""" + target = _validate_destination(destination) + data = self.download_bytes(workflow_id, file_handle_id) + target.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp( + prefix=f".{target.name}.", suffix=".part", dir=str(target.parent) + ) + try: + with os.fdopen(fd, "wb") as out: + out.write(data) + os.replace(temporary, target) + except OSError as e: + raise FileStorageError("File download failed") from e + finally: + if os.path.exists(temporary): + os.unlink(temporary) + return target + + def download_bytes(self, workflow_id: str, file_handle_id: str) -> bytes: + """Download a file's content into memory.""" + _validate_workflow_id(workflow_id) + file_id = _to_file_id(file_handle_id) + + def attempt(_: int) -> bytes: + url = self._server( + "GET", "/files/{workflowId}/{fileId}/download-url", workflow_id, file_id + ).get("downloadUrl") + if not url: + raise FileStorageError("Server returned an empty download URL response") + return self._transfer("GET", url, None, "File download") + + return self._with_transfer_retries("File download", attempt) + + def get_metadata(self, workflow_id: str, file_handle_id: str) -> FileMetadata: + """Return metadata for a file visible to *workflow_id*'s workflow family.""" + _validate_workflow_id(workflow_id) + file_id = _to_file_id(file_handle_id) + data = self._with_server_retries( + lambda: self._server("GET", "/files/{workflowId}/{fileId}", workflow_id, file_id) + ) + if not data: + raise FileStorageError("Server returned empty file metadata") + return FileMetadata.from_dict(data) + + # ── upload internals ───────────────────────────────────────────────── + + def _upload( + self, workflow_id: str, options: FileUploadOptions, read: Callable[[], bytes] + ) -> str: + body = { + "workflowId": workflow_id, + "fileName": options.file_name, + "contentType": options.content_type, + "taskId": options.task_id, + } + created = self._server("POST", "/files", body=body) + if not created: + raise FileStorageError("Server returned an empty file creation response") + handle = created.get("fileHandleId") + file_id = _to_file_id(handle) + owner = created.get("workflowId") + if owner and owner != workflow_id: + raise FileStorageError("The server returned a file owned by a different workflow") + storage_type = created.get("storageType") + + def attempt(n: int) -> None: + url = created.get("uploadUrl") + if n > 0: + url = self._server( + "GET", "/files/{workflowId}/{fileId}/upload-url", workflow_id, file_id + ).get("uploadUrl") + if not url: + raise FileStorageError("Server returned an empty upload URL response") + self._transfer("PUT", url, read(), "File upload", _upload_headers(storage_type)) + + self._with_transfer_retries("File upload", attempt) + self._complete_upload(workflow_id, file_id) + return handle + + def _complete_upload(self, workflow_id: str, file_id: str) -> None: + """Confirm the upload, treating an already-confirmed file as success.""" + last: Optional[Exception] = None + for _ in range(self._retry_count + 1): + try: + self._server( + "POST", "/files/{workflowId}/{fileId}/upload-complete", workflow_id, file_id + ) + return + except ApiException as e: + if not _retryable_server(e): + raise + last = e + if self._already_uploaded(workflow_id, file_id): + return + raise FileStorageError("Unable to confirm file upload completion") from last + + def _already_uploaded(self, workflow_id: str, file_id: str) -> bool: + try: + data = self._server("GET", "/files/{workflowId}/{fileId}", workflow_id, file_id) + except ApiException: + return False + return bool(data) and data.get("uploadStatus") == "UPLOADED" + + # ── transport ──────────────────────────────────────────────────────── + + def _server( + self, + method: str, + path: str, + workflow_id: Optional[str] = None, + file_id: Optional[str] = None, + body: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + path_params = {} + if workflow_id is not None: + path_params["workflowId"] = workflow_id + if file_id is not None: + path_params["fileId"] = file_id + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + } + return self._api_client.call_api( + path, + method, + path_params, + [], + headers, + body=body, + post_params=[], + files={}, + response_type="object", + auth_settings=_AUTH_SETTINGS, + _return_http_data_only=True, + _preload_content=True, + ) + + def _transfer( + self, + method: str, + url: str, + data: Optional[bytes], + operation: str, + headers: Optional[Dict[str, str]] = None, + ) -> bytes: + if not url.startswith(("http://", "https://")): + raise FileStorageError("Signed URL must use HTTP(S)") + try: + response = self._session.request( + method, url, data=data, headers=headers or {}, timeout=self._timeout + ) + except requests.RequestException: + # Drop the original exception text: it can contain the signed URL. + raise _TransferError(operation) from None + if not _HTTP_OK_MIN <= response.status_code < _HTTP_OK_MAX: + raise _TransferError(operation, response.status_code) + return response.content + + def _with_transfer_retries(self, operation: str, attempt: Callable[[int], T]) -> T: + last: Optional[Exception] = None + for n in range(self._retry_count + 1): + try: + return attempt(n) + except _TransferError as e: + retryable = ( + e.status is None + or e.status in _RETRYABLE_TRANSFER_STATUSES + or e.status >= _SERVER_ERROR + ) + if not retryable: + raise FileStorageError(str(e)) from None + last = e + except ApiException as e: + if not _retryable_server(e): + raise + last = e + raise FileStorageError( + f"{operation} failed after {self._retry_count + 1} attempts" + ) from last + + def _with_server_retries(self, read: Callable[[], T]) -> T: + last: Optional[Exception] = None + for _ in range(self._retry_count + 1): + try: + return read() + except ApiException as e: + if not _retryable_server(e): + raise + last = e + raise FileStorageError("Server request failed") from last + + +# ── helpers ────────────────────────────────────────────────────────────── + + +def _retryable_server(error: ApiException) -> bool: + status = getattr(error, "status", None) or 0 + return status in _RETRYABLE_SERVER_STATUSES or status >= _SERVER_ERROR + + +def _upload_headers(storage_type: Optional[str]) -> Dict[str, str]: + # Azure rejects a single-request PUT without the blob type. + if (storage_type or "").strip().upper() == "AZURE_BLOB": + return {"x-ms-blob-type": "BlockBlob"} + return {} + + +def _resolve_options( + options: Optional[FileUploadOptions], default_name: Optional[str] +) -> FileUploadOptions: + options = options or FileUploadOptions() + name = options.file_name or default_name + _validate_file_name(name) + return FileUploadOptions( + file_name=name, + content_type=options.content_type or _DEFAULT_CONTENT_TYPE, + task_id=options.task_id, + ) + + +def _validate_workflow_id(workflow_id: str) -> None: + if not workflow_id or not str(workflow_id).strip(): + raise FileStorageError("workflow_id is required") + + +def _validate_file_name(name: Optional[str]) -> None: + if ( + not name + or not name.strip() + or len(name) > _MAX_FILENAME_LENGTH + or name in (".", "..") + or _UNSAFE_FILENAME.search(name) + ): + raise FileStorageError("Filename is unsafe") + + +def _validate_destination(destination: Union[str, os.PathLike]) -> Path: + if destination is None: + raise FileStorageError("Download destination is required") + target = Path(destination).absolute() + if target.exists() and (target.is_dir() or target.is_symlink() or not target.is_file()): + raise FileStorageError("Download destination must name a regular file") + return target + + +def _to_file_id(file_handle_id: Optional[str]) -> str: + if not file_handle_id or not file_handle_id.startswith(FILE_HANDLE_PREFIX): + raise FileStorageError("Malformed file handle") + file_id = file_handle_id[len(FILE_HANDLE_PREFIX) :] + if not _FILE_ID_PATTERN.fullmatch(file_id): + raise FileStorageError("Malformed file handle") + return file_id + + +__all__ = [ + "FILE_HANDLE_PREFIX", + "FileClient", + "FileMetadata", + "FileStorageError", + "FileUploadOptions", +] diff --git a/tests/unit/ai/test_deep_agent.py b/tests/unit/ai/test_deep_agent.py index 1dea3c56..1d5037a9 100644 --- a/tests/unit/ai/test_deep_agent.py +++ b/tests/unit/ai/test_deep_agent.py @@ -6,6 +6,9 @@ from conductor.ai.agents import deep from conductor.ai.agents.deep import ( DEFAULT_INSTRUCTIONS, + STORAGE_ENV, + SUBAGENT_INSTRUCTIONS, + SUBAGENTS_ENV, TODOS_KEY, WORKSPACE_ENV, deep_agent, @@ -22,15 +25,19 @@ @pytest.fixture def workspace(tmp_path, monkeypatch): - """A workspace root in tmp, with root-id resolution short-circuited.""" + """A local workspace root in tmp, with root-id resolution short-circuited.""" + monkeypatch.setenv(STORAGE_ENV, "local") + monkeypatch.setenv(SUBAGENTS_ENV, '["analyst"]') monkeypatch.setenv(WORKSPACE_ENV, str(tmp_path)) monkeypatch.setattr(deep, "_root_execution_id", lambda eid: eid) return tmp_path -def ctx(execution_id="exec-1", state=None): +def ctx(execution_id="exec-1", state=None, agent_name="t"): return ToolContext( - execution_id=execution_id, agent_name="t", state=state if state is not None else {} + execution_id=execution_id, + agent_name=agent_name, + state=state if state is not None else {}, ) @@ -178,6 +185,57 @@ def test_no_context_uses_a_local_directory(workspace): assert (workspace / "local" / "r.md").exists() +def test_subagent_relative_paths_resolve_to_its_own_directory(workspace): + sub = ctx(agent_name="analyst", execution_id="exec-1") + out = write_file.__wrapped__(path="notes.md", content="x", context=sub) + assert out["path"] == "/subagents/analyst-exec-1/notes.md" + assert (workspace / "exec-1" / "subagents" / "analyst-exec-1" / "notes.md").exists() + assert ls.__wrapped__(context=sub)["entries"][0]["name"] == "notes.md" + + +def test_subagent_reads_anywhere_but_writes_only_at_home(workspace): + write_file.__wrapped__(path="/brief.md", content="brief", context=ctx()) + sub = ctx(agent_name="analyst") + assert read_file.__wrapped__(path="/brief.md", context=sub)["content"] == "brief" + with pytest.raises(ValueError, match="own directory"): + write_file.__wrapped__(path="/report.md", content="x", context=sub) + with pytest.raises(ValueError, match="own directory"): + edit_file.__wrapped__(path="/brief.md", old_string="b", new_string="c", context=sub) + + +def test_top_level_agent_sees_subagent_files(workspace): + sub = ctx(agent_name="analyst") + write_file.__wrapped__(path="result.md", content="r", context=sub) + parent = ctx() + listing = ls.__wrapped__(path="/subagents", context=parent) + assert listing["entries"] == [{"name": "analyst-exec-1", "type": "dir", "size": 0}] + got = read_file.__wrapped__(path="/subagents/analyst-exec-1/result.md", context=parent) + assert got["content"] == "r" + + +def test_top_level_agent_cannot_change_a_subagent_directory(workspace): + sub = ctx(agent_name="analyst") + path = write_file.__wrapped__(path="tests.py", content="assert True", context=sub)["path"] + parent = ctx() + with pytest.raises(ValueError, match="belongs to a sub-agent"): + write_file.__wrapped__(path=path, content="", context=parent) + with pytest.raises(ValueError, match="belongs to a sub-agent"): + edit_file.__wrapped__(path=path, old_string="True", new_string="1", context=parent) + assert read_file.__wrapped__(path=path, context=parent)["content"] == "assert True" + + +def test_export_files_copies_the_workspace_layout(workspace, tmp_path): + from conductor.ai.agents import deep_files + + c = ctx() + write_file.__wrapped__(path="/src/m.py", content="x = 1", context=c) + write_file.__wrapped__(path="/notes.md", content="n", context=c) + out = tmp_path / "export" + assert deep_files.export_files(c, out, prefix="/src") == ["src/m.py"] + assert (out / "src" / "m.py").read_text() == "x = 1" + assert not (out / "notes.md").exists() + + # ── root execution id ─────────────────────────────────────────────────── @@ -255,7 +313,10 @@ def get_weather(city: str) -> dict: assert agent.max_turns == 50 assert agent.instructions.startswith(DEFAULT_INSTRUCTIONS) assert "You are a travel assistant." in agent.instructions - assert __import__("os").environ[WORKSPACE_ENV] == str((tmp_path / "ws").resolve()) + env = __import__("os").environ + assert env[WORKSPACE_ENV] == str((tmp_path / "ws").resolve()) + assert env[STORAGE_ENV] == "conductor" + assert "weather_analyst" in __import__("json").loads(env[SUBAGENTS_ENV]) by_name = {get_tool_def(t).name: get_tool_def(t) for t in agent.tools} assert by_name["write_file"].approval_required is True @@ -263,10 +324,23 @@ def get_weather(city: str) -> dict: assert by_name["edit_file"].approval_required is False assert by_name["weather_analyst"].tool_type == "agent_tool" - # The sub-agent shares the workspace but not the plan. - sub_names = names(analyst) - assert "write_todos" not in sub_names - assert set(("ls", "read_file", "write_file", "edit_file")) <= set(sub_names) + # The caller's sub-agent is left unchanged; the harness wires a copy. + assert analyst.tools in (None, []) + assert analyst.instructions == "Compare cities." + wired = by_name["weather_analyst"].config["agent"] + assert wired is not analyst + assert wired.instructions.startswith("Compare cities.") + assert wired.instructions.endswith(SUBAGENT_INSTRUCTIONS) + # The copy shares the workspace but not the plan, and approval applies to it. + sub_tools = {get_tool_def(t).name: get_tool_def(t) for t in wired.tools} + assert "write_todos" not in sub_tools + assert {"ls", "read_file", "write_file", "edit_file"} <= set(sub_tools) + assert sub_tools["write_file"].approval_required is True + + +def test_deep_agent_rejects_unknown_storage(): + with pytest.raises(ValueError, match="storage"): + deep_agent(name="a", model="m", storage="s3") def test_deep_agent_applies_approval_to_user_tools(): diff --git a/tests/unit/ai/test_deep_files.py b/tests/unit/ai/test_deep_files.py new file mode 100644 index 00000000..671144e9 --- /dev/null +++ b/tests/unit/ai/test_deep_files.py @@ -0,0 +1,241 @@ +"""Tests for the deep agent's Conductor file backend. + +Uses a real in-process HTTP server (no mocks, per repo test policy) that emulates +the file API, including ``GET /api/files?workflowId=`` and the workflow-family +access rule: a workflow sees files owned by itself, its ancestors and its +descendants. +""" + +import json +import re +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import parse_qs, urlparse + +import pytest + +from conductor.ai.agents import deep_files +from conductor.ai.agents.deep import ( + STORAGE_ENV, + SUBAGENTS_ENV, + edit_file, + ls, + read_file, + write_file, +) +from conductor.ai.agents.tool import ToolContext + +# root has two sub-agent executions; they are siblings. +PARENTS = {"sub-a": "root", "sub-b": "root"} + + +def family(wf): + out = {wf} + p = wf + while p in PARENTS: + p = PARENTS[p] + out.add(p) + out.update(c for c, parent in PARENTS.items() if parent == wf) + return out + + +class _State: + def reset(self): + self.records = {} + self.content = {} + self.clock = 0 + self.base = "" + + +STATE = _State() + + +class _Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def _json(self, status, body): + data = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _body(self): + n = int(self.headers.get("Content-Length") or 0) + return self.rfile.read(n) if n else b"" + + def do_POST(self): + if self.path == "/api/files": + req = json.loads(self._body()) + STATE.clock += 1 + file_id = f"f{STATE.clock}" + STATE.records[file_id] = { + "fileHandleId": f"conductor://file/{file_id}", + "fileName": req.get("fileName"), + "workflowId": req["workflowId"], + "taskId": req.get("taskId"), + "storageType": "CONDUCTOR", + "uploadStatus": "UPLOADING", + "createdAt": STATE.clock, + } + return self._json(201, { + **STATE.records[file_id], + "uploadUrl": f"{STATE.base}/api/files/content/{req['workflowId']}/{file_id}", + }) + m = re.fullmatch(r"/api/files/([^/]+)/([^/]+)/upload-complete", self.path) + if m: + rec = STATE.records[m.group(2)] + rec["uploadStatus"] = "UPLOADED" + rec["fileSize"] = len(STATE.content.get(m.group(2), b"")) + return self._json(200, {"uploadStatus": "UPLOADED"}) + return self._json(404, {}) + + def do_PUT(self): + m = re.fullmatch(r"/api/files/content/([^/]+)/([^/]+)", self.path) + STATE.content[m.group(2)] = self._body() + self.send_response(204) + self.send_header("Content-Length", "0") + self.end_headers() + + def do_GET(self): + url = urlparse(self.path) + if url.path == "/api/files": + caller = parse_qs(url.query)["workflowId"][0] + visible = family(caller) + items = [r for r in STATE.records.values() + if r["uploadStatus"] == "UPLOADED" and r["workflowId"] in visible] + return self._json(200, sorted(items, key=lambda r: r["createdAt"])) + m = re.fullmatch(r"/api/files/content/([^/]+)/([^/]+)", url.path) + if m: + data = STATE.content[m.group(2)] + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + m = re.fullmatch(r"/api/files/([^/]+)/([^/]+)/download-url", url.path) + if m: + caller, file_id = m.groups() + if STATE.records[file_id]["workflowId"] not in family(caller): + return self._json(403, {"message": "forbidden"}) + return self._json(200, {"downloadUrl": + f"{STATE.base}/api/files/content/{caller}/{file_id}"}) + return self._json(404, {}) + + +@pytest.fixture(scope="module") +def server(): + httpd = HTTPServer(("127.0.0.1", 0), _Handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + yield httpd + httpd.shutdown() + + +@pytest.fixture +def files(server, monkeypatch): + STATE.reset() + STATE.base = f"http://127.0.0.1:{server.server_port}" + monkeypatch.setenv("CONDUCTOR_SERVER_URL", f"{STATE.base}/api") + monkeypatch.setenv(STORAGE_ENV, "conductor") + monkeypatch.setenv(SUBAGENTS_ENV, '["analyst"]') + # A fresh client per test so it picks up this server's URL. + monkeypatch.setattr(deep_files, "_conductor_backend", deep_files.ConductorBackend()) + return STATE + + +def ctx(execution_id="root", agent_name="planner", task_id="t1"): + return ToolContext(execution_id=execution_id, agent_name=agent_name, task_id=task_id) + + +def test_write_then_read_and_list(files): + out = write_file.__wrapped__(path="notes/a.md", content="one\ntwo", context=ctx()) + assert out == {"path": "/notes/a.md", "bytes": 7} + + assert read_file.__wrapped__(path="/notes/a.md", context=ctx())["content"] == "one\ntwo" + listing = ls.__wrapped__(path="notes", context=ctx()) + assert listing["entries"] == [ + {"name": "a.md", "type": "file", "size": 7, "written_by": "you"} + ] + # The logical path travels in the file name, encoded so it has no slash. + assert next(iter(files.records.values()))["fileName"] == "notes%2Fa.md" + + +def test_newest_version_wins_and_versions_are_kept(files): + write_file.__wrapped__(path="r.md", content="v1", context=ctx(task_id="t1")) + write_file.__wrapped__(path="r.md", content="v2", context=ctx(task_id="t2")) + + assert read_file.__wrapped__(path="r.md", context=ctx())["content"] == "v2" + assert len(files.records) == 2 + + +def test_retried_write_does_not_create_another_version(files): + write_file.__wrapped__(path="r.md", content="v1", context=ctx(task_id="t1")) + write_file.__wrapped__(path="r.md", content="v1", context=ctx(task_id="t1")) + + assert len(files.records) == 1 + + +def test_retried_edit_is_not_applied_twice(files): + write_file.__wrapped__(path="r.md", content="a b", context=ctx(task_id="t1")) + first = edit_file.__wrapped__( + path="r.md", old_string="a", new_string="z", context=ctx(task_id="t2") + ) + again = edit_file.__wrapped__( + path="r.md", old_string="a", new_string="z", context=ctx(task_id="t2") + ) + + assert first["replacements"] == 1 + assert again["note"] == "already applied" + assert read_file.__wrapped__(path="r.md", context=ctx())["content"] == "z b" + assert len(files.records) == 2 + + +def test_parent_and_subagent_see_each_others_files(files): + sub = ctx(execution_id="sub-a", agent_name="analyst", task_id="s1") + out = write_file.__wrapped__(path="result.md", content="r", context=sub) + assert out["path"] == "/subagents/analyst-sub-a/result.md" + + parent = ctx() + got = read_file.__wrapped__(path=out["path"], context=parent) + assert got["content"] == "r" + assert ls.__wrapped__(path=out["path"].rsplit("/", 1)[0], context=parent)["entries"][0][ + "written_by" + ] == "another agent (sub-a)" + + write_file.__wrapped__(path="/brief.md", content="b", context=parent) + assert read_file.__wrapped__(path="/brief.md", context=sub)["content"] == "b" + + +def test_sibling_subagents_do_not_see_each_others_files(files): + a = ctx(execution_id="sub-a", agent_name="analyst", task_id="s1") + b = ctx(execution_id="sub-b", agent_name="analyst", task_id="s2") + path = write_file.__wrapped__(path="result.md", content="r", context=a)["path"] + + assert read_file.__wrapped__(path=path, context=b)["error"] == "no such file" + + +def test_overwriting_another_agents_version_is_reported(files): + write_file.__wrapped__(path="/shared.md", content="sub", context=ctx("sub-a", "x", "s1")) + out = write_file.__wrapped__(path="/shared.md", content="parent", context=ctx()) + + assert out["note"] == "This replaced a version written by another agent (sub-a)." + + +def test_read_run_file_and_list_run_files(files): + write_file.__wrapped__(path="report.md", content="final", context=ctx()) + + assert deep_files.read_run_file("root", "/report.md") == "final" + assert list(deep_files.list_run_files("root")) == ["report.md"] + + +def test_export_run_downloads_every_visible_file(files, tmp_path): + write_file.__wrapped__(path="/src/m.py", content="x = 1", context=ctx()) + sub = ctx(execution_id="sub-a", agent_name="analyst", task_id="s1") + tests = write_file.__wrapped__(path="test_m.py", content="t", context=sub)["path"] + + exported = deep_files.export_run("root", tmp_path) + + assert sorted(exported) == sorted(["src/m.py", tests.lstrip("/")]) + assert (tmp_path / "src" / "m.py").read_text() == "x = 1" diff --git a/tests/unit/test_file_client.py b/tests/unit/test_file_client.py new file mode 100644 index 00000000..24f93f37 --- /dev/null +++ b/tests/unit/test_file_client.py @@ -0,0 +1,314 @@ +"""Tests for FileClient, the /api/files client. + +Uses a real in-process HTTP server (no mocks, per repo test policy) that emulates +the file API with the Conductor-managed backend: file records, the raw content +endpoints the server hands out as upload and download URLs, and the +upload-complete transition. +""" + +import json +import re +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from conductor.client.configuration.configuration import Configuration +from conductor.client.file_client import ( + FileClient, + FileStorageError, + FileUploadOptions, +) + + +class _FileServer: + """In-memory state shared by the handler.""" + + def __init__(self): + self.reset() + + def reset(self): + self.records = {} + self.content = {} + self.next_id = 0 + self.requests = [] + # Failure knobs: number of times to fail, and with which status. + self.fail_put = 0 + self.fail_put_status = 503 + self.fail_complete = 0 + self.storage_type = "CONDUCTOR" + self.base = "" + + +STATE = _FileServer() + + +class _Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def _json(self, status, body): + data = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _empty(self, status): + self.send_response(status) + self.send_header("Content-Length", "0") + self.end_headers() + + def _body(self): + n = int(self.headers.get("Content-Length") or 0) + return self.rfile.read(n) if n else b"" + + def _record(self): + STATE.requests.append((self.command, self.path, dict(self.headers))) + + def do_POST(self): + self._record() + if self.path == "/api/files": + req = json.loads(self._body()) + STATE.next_id += 1 + file_id = f"f{STATE.next_id}" + STATE.records[file_id] = { + "fileHandleId": f"conductor://file/{file_id}", + "fileName": req.get("fileName"), + "contentType": req.get("contentType"), + "workflowId": req["workflowId"], + "taskId": req.get("taskId"), + "storageType": STATE.storage_type, + "uploadStatus": "UPLOADING", + } + return self._json( + 201, + { + **STATE.records[file_id], + "uploadUrl": f"{STATE.base}/api/files/content/{req['workflowId']}/{file_id}", + }, + ) + m = re.fullmatch(r"/api/files/([^/]+)/([^/]+)/upload-complete", self.path) + if m: + wf, file_id = m.groups() + rec = STATE.records.get(file_id) + if rec is None or rec["workflowId"] != wf: + return self._json(403, {"message": "forbidden"}) + if STATE.fail_complete: + STATE.fail_complete -= 1 + # The server applied the change but the response was lost. + rec["uploadStatus"] = "UPLOADED" + return self._json(503, {"message": "unavailable"}) + if rec["uploadStatus"] == "UPLOADED": + return self._json(409, {"message": "already uploaded"}) + rec["uploadStatus"] = "UPLOADED" + rec["fileSize"] = len(STATE.content.get(file_id, b"")) + return self._json( + 200, {"fileHandleId": rec["fileHandleId"], "uploadStatus": "UPLOADED"} + ) + return self._json(404, {"message": "not found"}) + + def do_PUT(self): + self._record() + m = re.fullmatch(r"/api/files/content/([^/]+)/([^/]+)", self.path) + if not m: + return self._json(404, {}) + body = self._body() + if STATE.fail_put: + STATE.fail_put -= 1 + return self._empty(STATE.fail_put_status) + STATE.content[m.group(2)] = body + return self._empty(204) + + def do_GET(self): + self._record() + m = re.fullmatch(r"/api/files/content/([^/]+)/([^/]+)", self.path) + if m: + data = STATE.content.get(m.group(2)) + if data is None: + return self._json(404, {}) + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + m = re.fullmatch(r"/api/files/([^/]+)/([^/]+)(/[a-z-]+)?", self.path) + if not m: + return self._json(404, {}) + wf, file_id, suffix = m.groups() + rec = STATE.records.get(file_id) + if rec is None: + return self._json(404, {"message": "not found"}) + if suffix is None: + return self._json(200, rec) + if suffix == "/upload-url": + return self._json( + 200, + { + "fileHandleId": rec["fileHandleId"], + "uploadUrl": f"{STATE.base}/api/files/content/{wf}/{file_id}", + }, + ) + if suffix == "/download-url": + if rec["uploadStatus"] != "UPLOADED": + return self._json(400, {"message": "not uploaded"}) + return self._json( + 200, + { + "fileHandleId": rec["fileHandleId"], + "downloadUrl": f"{STATE.base}/api/files/content/{wf}/{file_id}", + }, + ) + return self._json(404, {}) + + +@pytest.fixture(scope="module") +def server(): + httpd = HTTPServer(("127.0.0.1", 0), _Handler) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + STATE.base = f"http://127.0.0.1:{httpd.server_port}" + yield httpd + httpd.shutdown() + + +@pytest.fixture +def client(server): + STATE.reset() + STATE.base = f"http://127.0.0.1:{server.server_port}" + config = Configuration(server_api_url=f"{STATE.base}/api") + return FileClient(config, retry_count=2) + + +def test_upload_and_download_bytes_round_trip(client): + handle = client.upload_bytes( + "wf-1", b"hello", FileUploadOptions(file_name="a.txt", content_type="text/plain") + ) + + assert handle == "conductor://file/f1" + assert client.download_bytes("wf-1", handle) == b"hello" + meta = client.get_metadata("wf-1", handle) + assert meta.upload_status == "UPLOADED" + assert meta.file_name == "a.txt" + assert meta.content_type == "text/plain" + assert meta.workflow_id == "wf-1" + + +def test_upload_path_uses_file_name_and_default_content_type(client, tmp_path): + source = tmp_path / "report.md" + source.write_text("# report") + + handle = client.upload("wf-1", source) + + meta = client.get_metadata("wf-1", handle) + assert meta.file_name == "report.md" + assert meta.content_type == "application/octet-stream" + + +def test_upload_records_task_id(client): + handle = client.upload_bytes( + "wf-1", b"x", FileUploadOptions(file_name="a.txt", task_id="task-9") + ) + + assert client.get_metadata("wf-1", handle).task_id == "task-9" + + +def test_download_to_path_replaces_destination(client, tmp_path): + handle = client.upload_bytes("wf-1", b"new", FileUploadOptions(file_name="a.txt")) + target = tmp_path / "out" / "a.txt" + target.parent.mkdir() + target.write_bytes(b"old") + + result = client.download("wf-1", handle, target) + + assert result == target + assert target.read_bytes() == b"new" + assert [p.name for p in target.parent.iterdir()] == ["a.txt"] + + +def test_upload_retries_transient_put_failure_with_fresh_url(client): + STATE.fail_put = 1 + + handle = client.upload_bytes("wf-1", b"data", FileUploadOptions(file_name="a.txt")) + + assert client.download_bytes("wf-1", handle) == b"data" + paths = [p for (m, p, _) in STATE.requests] + assert "/api/files/wf-1/f1/upload-url" in paths + + +def test_upload_fails_fast_on_non_retryable_put_status(client): + STATE.fail_put = 1 + STATE.fail_put_status = 400 + + with pytest.raises(FileStorageError) as err: + client.upload_bytes("wf-1", b"data", FileUploadOptions(file_name="a.txt")) + + assert "HTTP 400" in str(err.value) + + +def test_upload_gives_up_after_retry_count(client): + STATE.fail_put = 10 + + with pytest.raises(FileStorageError): + client.upload_bytes("wf-1", b"data", FileUploadOptions(file_name="a.txt")) + + puts = [p for (m, p, _) in STATE.requests if m == "PUT"] + assert len(puts) == 3 + + +def test_lost_completion_response_is_reconciled(client): + STATE.fail_complete = 1 + + handle = client.upload_bytes("wf-1", b"data", FileUploadOptions(file_name="a.txt")) + + assert client.get_metadata("wf-1", handle).upload_status == "UPLOADED" + + +def test_azure_upload_sends_block_blob_header(client): + STATE.storage_type = "AZURE_BLOB" + + client.upload_bytes("wf-1", b"data", FileUploadOptions(file_name="a.txt")) + + put_headers = [h for (m, _, h) in STATE.requests if m == "PUT"][0] + assert put_headers.get("x-ms-blob-type") == "BlockBlob" + + +def test_transfers_do_not_send_conductor_credentials(client): + client.upload_bytes("wf-1", b"data", FileUploadOptions(file_name="a.txt")) + + put_headers = [h for (m, _, h) in STATE.requests if m == "PUT"][0] + assert "X-Authorization" not in put_headers + + +@pytest.mark.parametrize("name", ["", ".", "..", "a/b.txt", "a\\b.txt", "a:b", "x" * 256]) +def test_unsafe_file_names_are_rejected(client, name): + with pytest.raises(FileStorageError): + client.upload_bytes("wf-1", b"x", FileUploadOptions(file_name=name)) + assert STATE.requests == [] + + +@pytest.mark.parametrize( + "handle", [None, "", "f1", "s3://file/f1", "conductor://file/", "conductor://file/../x"] +) +def test_malformed_handles_are_rejected(client, handle): + with pytest.raises(FileStorageError): + client.download_bytes("wf-1", handle) + assert STATE.requests == [] + + +def test_workflow_id_is_required(client): + with pytest.raises(FileStorageError): + client.upload_bytes("", b"x", FileUploadOptions(file_name="a.txt")) + + +def test_upload_bytes_requires_file_name(client): + with pytest.raises(FileStorageError): + client.upload_bytes("wf-1", b"x", FileUploadOptions()) + + +def test_download_destination_must_not_be_a_directory(client, tmp_path): + handle = client.upload_bytes("wf-1", b"x", FileUploadOptions(file_name="a.txt")) + + with pytest.raises(FileStorageError): + client.download("wf-1", handle, tmp_path) From 1cdb4c8750d95dfbb93397574bffaa9cd5499c28 Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Fri, 20 Nov 2026 14:46:19 -0800 Subject: [PATCH 3/4] feat(agents): read the file list from its SearchResult wrapper --- src/conductor/ai/agents/deep_files.py | 2 +- tests/unit/ai/test_deep_files.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/conductor/ai/agents/deep_files.py b/src/conductor/ai/agents/deep_files.py index e024bda6..76a3cee2 100644 --- a/src/conductor/ai/agents/deep_files.py +++ b/src/conductor/ai/agents/deep_files.py @@ -209,7 +209,7 @@ def versions(self, context: Optional[ToolContext]) -> List[Dict[str, Any]]: _return_http_data_only=True, _preload_content=True, ) - return list(items or []) + return list((items or {}).get("results") or []) def list(self, context): out: Dict[str, FileEntry] = {} diff --git a/tests/unit/ai/test_deep_files.py b/tests/unit/ai/test_deep_files.py index 671144e9..fa003b78 100644 --- a/tests/unit/ai/test_deep_files.py +++ b/tests/unit/ai/test_deep_files.py @@ -1,7 +1,7 @@ """Tests for the deep agent's Conductor file backend. Uses a real in-process HTTP server (no mocks, per repo test policy) that emulates -the file API, including ``GET /api/files?workflowId=`` and the workflow-family +the file API, including ``GET /api/files?workflowId=`` (a ``SearchResult``) and the workflow-family access rule: a workflow sees files owned by itself, its ancestors and its descendants. """ @@ -106,7 +106,8 @@ def do_GET(self): visible = family(caller) items = [r for r in STATE.records.values() if r["uploadStatus"] == "UPLOADED" and r["workflowId"] in visible] - return self._json(200, sorted(items, key=lambda r: r["createdAt"])) + results = sorted(items, key=lambda r: r["createdAt"]) + return self._json(200, {"totalHits": len(results), "results": results}) m = re.fullmatch(r"/api/files/content/([^/]+)/([^/]+)", url.path) if m: data = STATE.content[m.group(2)] From 4bb45060ca335a10fc53a3f7560e7058c68f0b05 Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Fri, 20 Nov 2026 16:41:15 -0800 Subject: [PATCH 4/4] refactor(agents): export only the deep agent's user-facing API --- src/conductor/ai/agents/deep.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/conductor/ai/agents/deep.py b/src/conductor/ai/agents/deep.py index 15a92bab..8b8b690b 100644 --- a/src/conductor/ai/agents/deep.py +++ b/src/conductor/ai/agents/deep.py @@ -556,24 +556,12 @@ def workspace_dir(execution_id: str, workspace: Optional[Union[str, Path]] = Non return (base / _root_execution_id(execution_id)).resolve() +# Public API. The tools, prompts and settings are internal: users get them +# through deep_agent() and should not attach them to other agents. __all__ = [ - "DEFAULT_INSTRUCTIONS", - "FILE_TOOLS", - "STORAGE_ENV", - "SUBAGENTS_ENV", - "SUBAGENT_INSTRUCTIONS", - "TODOS_KEY", - "WORKSPACE_ENV", "deep_agent", - "edit_file", "export_run", "list_run_files", - "ls", - "read_file", "read_run_file", - "resolve_root_execution_id", - "stored_todos", "workspace_dir", - "write_file", - "write_todos", ]