diff --git a/.github/workflows/cog.yml b/.github/workflows/cog.yml index d46c0f3b0..778e0230b 100644 --- a/.github/workflows/cog.yml +++ b/.github/workflows/cog.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: ref: ${{ github.head_ref }} - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ab73b83af..e99c0ab6f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,13 +14,13 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | pip install . --group dev @@ -34,13 +34,13 @@ jobs: id-token: write needs: [test] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.13' cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | pip install setuptools wheel build diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml index 84b829e96..3ae9d9447 100644 --- a/.github/workflows/stable-docs.yml +++ b/.github/workflows/stable-docs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v7 with: fetch-depth: 0 # We need all commits to find docs/ changes - name: Set up Git user diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0c6fccaae..03ff77e92 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,13 +13,13 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | pip install . --group dev @@ -27,7 +27,7 @@ jobs: run: | python -m pytest -vv - name: Check if cog needs to be run - if: matrix.os != 'windows-latest' + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' run: | cog --check \ -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \ @@ -54,7 +54,7 @@ jobs: ./tests/test-llm-load-plugins.sh - name: Upload artifact of builds if: matrix.python-version == '3.13' && matrix.os == 'ubuntu-latest' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: dist-${{ matrix.os }}-${{ matrix.python-version }} path: dist/* diff --git a/.gitignore b/.gitignore index aa1fee1f0..7e07dc883 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,10 @@ venv .eggs .pytest_cache *.egg-info +build/ .DS_Store .idea/ .vscode/ -uv.lock \ No newline at end of file +uv.lock +*.db +backups-do-not-delete/ diff --git a/Justfile b/Justfile index 626ff09c4..f85fddc58 100644 --- a/Justfile +++ b/Justfile @@ -27,10 +27,10 @@ @cog: uv run cog -r -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" docs/**/*.md docs/*.md README.md -# Serve live docs on localhost:8000 -@docs: cog +# Serve live docs on localhost (default port: 8000) +@docs port="8000": cog rm -rf docs/_build - cd docs && uv run make livehtml + cd docs && uv run make livehtml SPHINXOPTS="--port {{port}}" # Apply Black @black: diff --git a/README.md b/README.md index 92cde72d1..1dcee9860 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ cog.out(readme_markdown) [![Discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord-llm) [![Homebrew](https://img.shields.io/homebrew/installs/dy/llm?color=yellow&label=homebrew&logo=homebrew)](https://formulae.brew.sh/formula/llm) -A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Meta’s Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. +A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Qwen**, **Gemma**, **Kimi**, **DeepSeek**, **Mistral**, and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/). @@ -69,13 +69,27 @@ Or with [uv](https://docs.astral.sh/uv/guides/tools/) uv tool install llm ``` +Use LLM to run prompts or start chats against an arbitrary OpenAI-compatible Chat Completions endpoint, such as [LM Studio](https://lmstudio.ai). With `uvx`, you can do this without installing LLM first: + +```bash +uvx llm openai endpoint http://localhost:1234/v1 \ + -m google/gemma-4-12b \ + "What is the capital of France?" + +uvx llm openai endpoint http://localhost:1234/v1 \ + -m google/gemma-4-12b \ + --chat +``` + +Add `--key your-api-key` if the endpoint requires authentication. See [Run against an endpoint without configuring it](https://llm.datasette.io/en/stable/other-models.html#openai-endpoint) for more options. + If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: ```bash # Paste your OpenAI API key into this llm keys set openai -# Run a prompt (with the default gpt-4o-mini model) +# Run a prompt (with the default gpt-5.6-luna model) llm "Ten fun names for a pet pelican" # Extract text from an image @@ -91,12 +105,12 @@ Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic]( llm install llm-gemini llm keys set gemini # Paste Gemini API key here -llm -m gemini-2.0-flash 'Tell me fun facts about Mountain View' +llm -m gemini-3.5-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here -llm -m claude-4-opus 'Impress me with wild facts about turnips' +llm -m claude-sonnet-5 'Impress me with wild facts about turnips' ``` You can also [install a plugin](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#installing-plugins) to access models that can run on your local device. If you use [Ollama](https://ollama.com/): @@ -128,17 +142,24 @@ Why don't pelicans like to tip waiters? Because they always have a big bill! ``` -More background on this project: - -- [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) -- [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) -- [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) -- [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) -- [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) -- [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) -- [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) - -See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. +## Project news + +- 29th April 2026: [LLM 0.32a0 is a major backwards-compatible refactor](https://simonwillison.net/2026/Apr/29/llm/) +- 11th August 2025: [LLM 0.27, the annotated release notes: GPT-5 and improved tool calling](https://simonwillison.net/2025/Aug/11/llm-027/) +- 27th May 2025: [Large Language Models can run tools in your terminal with LLM 0.26](https://simonwillison.net/2025/May/27/llm-tools/) +- 5th May 2025: [Feed a video to a vision LLM as a sequence of JPEG frames on the CLI (also LLM 0.25)](https://simonwillison.net/2025/May/5/llm-video-frames/) +- 7th April 2025: [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) +- 28th February 2025: [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) +- 17th February 2025: [LLM 0.22, the annotated release notes](https://simonwillison.net/2025/Feb/17/llm/) +- 29th October 2024: [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) +- 26th January 2024: [LLM 0.13: The annotated release notes](https://simonwillison.net/2024/Jan/26/llm/) +- 12th September 2023: [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) +- 4th September 2023: [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) +- 12th July 2023: [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) +- 18th May 2023: [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) +- 4th April 2023: [The original announcement of the llm CLI tool](https://simonwillison.net/2023/Apr/4/llm/) + +For everything else, see [the llm tag](https://simonwillison.net/tags/llm/) on my blog. ## Contents @@ -163,6 +184,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [System prompts](https://llm.datasette.io/en/stable/usage.html#system-prompts) * [Tools](https://llm.datasette.io/en/stable/usage.html#tools) * [Extracting fenced code blocks](https://llm.datasette.io/en/stable/usage.html#extracting-fenced-code-blocks) + * [JSON output](https://llm.datasette.io/en/stable/usage.html#json-output) * [Schemas](https://llm.datasette.io/en/stable/usage.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/usage.html#fragments) * [Continuing a conversation](https://llm.datasette.io/en/stable/usage.html#continuing-a-conversation) @@ -175,12 +197,17 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Configuration](https://llm.datasette.io/en/stable/openai-models.html#configuration) * [OpenAI language models](https://llm.datasette.io/en/stable/openai-models.html#openai-language-models) * [Model features](https://llm.datasette.io/en/stable/openai-models.html#model-features) + * [Web Search](https://llm.datasette.io/en/stable/openai-models.html#web-search) + * [Code Interpreter](https://llm.datasette.io/en/stable/openai-models.html#code-interpreter) + * [Fast mode and service tiers](https://llm.datasette.io/en/stable/openai-models.html#fast-mode-and-service-tiers) * [OpenAI embedding models](https://llm.datasette.io/en/stable/openai-models.html#openai-embedding-models) * [OpenAI completion models](https://llm.datasette.io/en/stable/openai-models.html#openai-completion-models) * [Adding more OpenAI models](https://llm.datasette.io/en/stable/openai-models.html#adding-more-openai-models) * [Other models](https://llm.datasette.io/en/stable/other-models.html) * [Installing and using a local model](https://llm.datasette.io/en/stable/other-models.html#installing-and-using-a-local-model) * [OpenAI-compatible models](https://llm.datasette.io/en/stable/other-models.html#openai-compatible-models) + * [Run against an endpoint without configuring it](https://llm.datasette.io/en/stable/other-models.html#run-against-an-endpoint-without-configuring-it) + * [Configure an OpenAI-compatible model](https://llm.datasette.io/en/stable/other-models.html#configure-an-openai-compatible-model) * [Extra HTTP headers](https://llm.datasette.io/en/stable/other-models.html#extra-http-headers) * [Tools](https://llm.datasette.io/en/stable/tools.html) * [How tools work](https://llm.datasette.io/en/stable/tools.html#how-tools-work) @@ -238,6 +265,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Retrieving similar items](https://llm.datasette.io/en/stable/embeddings/python-api.html#retrieving-similar-items) * [SQL schema](https://llm.datasette.io/en/stable/embeddings/python-api.html#sql-schema) * [Writing plugins to add new embedding models](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html) + * [`EmbeddingModel`](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#llm.EmbeddingModel) * [Embedding binary content](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#embedding-binary-content) * [Embedding storage format](https://llm.datasette.io/en/stable/embeddings/storage.html) * [Plugins](https://llm.datasette.io/en/stable/plugins/index.html) @@ -254,7 +282,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Just for fun](https://llm.datasette.io/en/stable/plugins/directory.html#just-for-fun) * [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html) * [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli) - * [register_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register) + * [register_models(register, model_aliases)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register-model-aliases) * [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register) * [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register) * [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register) @@ -279,7 +307,12 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Async models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#async-models) * [Supporting schemas](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-schemas) * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) + * [Supporting server-side tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-server-side-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) + * [Structured messages and streaming events](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#structured-messages-and-streaming-events) + * [Condensing logged payloads with json_replacements](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#condensing-logged-payloads-with-json-replacements) + * [Consuming prompt.messages in build_messages](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#consuming-prompt-messages-in-build-messages) + * [Restoring opaque metadata on subsequent requests](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#restoring-opaque-metadata-on-subsequent-requests) * [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage) * [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names) * [LLM_RAISE_ERRORS](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#llm-raise-errors) @@ -301,7 +334,9 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Accessing the underlying JSON](https://llm.datasette.io/en/stable/python-api.html#accessing-the-underlying-json) * [Token usage](https://llm.datasette.io/en/stable/python-api.html#token-usage) * [Streaming responses](https://llm.datasette.io/en/stable/python-api.html#streaming-responses) + * [Structured messages and streaming events](https://llm.datasette.io/en/stable/python-api.html#structured-messages-and-streaming-events) * [Async models](https://llm.datasette.io/en/stable/python-api.html#async-models) + * [`AsyncResponse`](https://llm.datasette.io/en/stable/python-api.html#llm.AsyncResponse) * [Tool functions can be sync or async](https://llm.datasette.io/en/stable/python-api.html#tool-functions-can-be-sync-or-async) * [Tool use for async models](https://llm.datasette.io/en/stable/python-api.html#tool-use-for-async-models) * [Conversations](https://llm.datasette.io/en/stable/python-api.html#conversations) @@ -326,6 +361,16 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Browsing data collected using schemas](https://llm.datasette.io/en/stable/logging.html#browsing-data-collected-using-schemas) * [Browsing logs using Datasette](https://llm.datasette.io/en/stable/logging.html#browsing-logs-using-datasette) * [Backing up your database](https://llm.datasette.io/en/stable/logging.html#backing-up-your-database) + * [The message store](https://llm.datasette.io/en/stable/logging.html#the-message-store) + * [Threads, turns, messages and parts](https://llm.datasette.io/en/stable/logging.html#threads-turns-messages-and-parts) + * [A worked example](https://llm.datasette.io/en/stable/logging.html#a-worked-example) + * [Content addressing as a contract](https://llm.datasette.io/en/stable/logging.html#content-addressing-as-a-contract) + * [Forking and shared history](https://llm.datasette.io/en/stable/logging.html#forking-and-shared-history) + * [Storage by reference](https://llm.datasette.io/en/stable/logging.html#storage-by-reference) + * [The raw provider payload](https://llm.datasette.io/en/stable/logging.html#the-raw-provider-payload) + * [Table by table](https://llm.datasette.io/en/stable/logging.html#table-by-table) + * [Querying the message store](https://llm.datasette.io/en/stable/logging.html#querying-the-message-store) + * [Logging from Python](https://llm.datasette.io/en/stable/logging.html#logging-from-python) * [SQL schema](https://llm.datasette.io/en/stable/logging.html#sql-schema) * [Related tools](https://llm.datasette.io/en/stable/related-tools.html) * [strip-tags](https://llm.datasette.io/en/stable/related-tools.html#strip-tags) diff --git a/docs/aliases.md b/docs/aliases.md index ced77a570..a685f4d33 100644 --- a/docs/aliases.md +++ b/docs/aliases.md @@ -20,7 +20,6 @@ cog.out("```\n{}```".format(result.output)) ]]] --> ``` 4o : gpt-4o -chatgpt-4o : chatgpt-4o-latest 4o-mini : gpt-4o-mini 4.1 : gpt-4.1 4.1-mini : gpt-4.1-mini @@ -31,11 +30,9 @@ chatgpt-16k : gpt-3.5-turbo-16k 3.5-16k : gpt-3.5-turbo-16k 4 : gpt-4 gpt4 : gpt-4 -4-32k : gpt-4-32k gpt-4-turbo-preview : gpt-4-turbo 4-turbo : gpt-4-turbo 4t : gpt-4-turbo -gpt-4.5 : gpt-4.5-preview 3.5-instruct : gpt-3.5-turbo-instruct chatgpt-instruct : gpt-3.5-turbo-instruct ada : text-embedding-ada-002 (embedding) @@ -69,15 +66,15 @@ Example output: The `llm aliases set ` command can be used to add a new alias: ```bash -llm aliases set mini gpt-4o-mini +llm aliases set luna gpt-5.6-luna ``` You can also pass one or more `-q search` options to set an alias on the first model matching those search terms: ```bash -llm aliases set mini -q 4o -q mini +llm aliases set luna -q gpt -q luna ``` -Now you can run the `gpt-4o-mini` model using the `mini` alias like this: +Now you can run the `gpt-5.6-luna` model using the `luna` alias like this: ```bash -llm -m mini 'An epic Greek-style saga about a cheesecake that builds a SQL database from scratch' +llm -m luna 'An epic Greek-style saga about a cheesecake that builds a SQL database from scratch' ``` Aliases can be set for both regular models and {ref}`embedding models ` using the same command. To set an alias of `oai` for the OpenAI `ada-002` embedding model use this: ```bash @@ -113,4 +110,4 @@ To view the content of that file, run this: ```bash cat "$(llm aliases path)" -``` \ No newline at end of file +``` diff --git a/docs/changelog.md b/docs/changelog.md index 24bf2711a..f9c9e16cc 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,220 @@ # Changelog +(unreleased)= +## Unreleased + +- Reasoning-capable Responses API models now support a `reasoning_summary` option with `auto`, `concise`, and `detailed` values. This can be used with {ref}`llm openai endpoint --responses `. [#1600](https://github.com/simonw/llm/issues/1600) + +(v0_32)= +## 0.32 (2026-08-04) + +LLM 0.32 is a major, backwards-compatible update to the way prompts, responses, tools and logs are represented. It adds structured messages and parts throughout the Python API, adopts the OpenAI Responses API for reasoning-capable models, substantially expands control over pausable and resumable tool loops and introduces a new content-addressed SQLite logging schema. Reasoning traces are now displayed on standard error by the `llm` command, for models that support them. + +### Structured messages and richer responses + +Prompt inputs and model outputs are now represented as lists of `Message` objects, each containing typed `Part` objects for text, reasoning, tool calls, tool results and attachments. + +- New {ref}`messages= keyword argument ` on the prompt, conversation and chain APIs, including their asynchronous equivalents. For example, `model.prompt(messages=[llm.user("Hello"), llm.assistant("Hi!"), llm.user("What can you do?")])`. Existing `prompt=`, `system=`, `attachments=` and `tool_results=` arguments continue to work and are converted into the same structured representation. +- New {ref}`structured streaming methods ` `response.stream_events()` and `response.astream_events()` expose mixed streams of text, reasoning, tool calls and tool results. Iterating over a response directly continues to yield text strings. +- `response.messages()` returns the assembled structured output. `response.reply()` continues from any response and can automatically execute pending tool calls before the next turn. +- `response.to_dict()` and `Response.from_dict()` provide JSON-safe persistence of complete turns, including reasoning and provider metadata, with matching `TypedDict` definitions in the new `llm.serialization` module. +- `response.prompt.messages` is now the canonical record of exactly what was sent to the model, including the full preceding conversation chain. + +These APIs were introduced in {ref}`0.32a0 `. They are described in the {ref}`Advanced model plugins ` documentation. + +### OpenAI Responses API and reasoning + +- Most reasoning-capable OpenAI models now use the `/v1/responses` endpoint by default, enabling interleaved reasoning across tool calls. The existing Chat Completions classes remain available, and `-o chat_completions 1` selects that older path for an individual prompt. See {ref}`0.32a2 ` for the full list of affected models. +- OpenAI Responses API models now provide {ref}`WebSearch ` and {ref}`CodeInterpreter ` server-side tools, available from the CLI using `-T WebSearch` or `-T 'CodeInterpreter(memory_limit="4g")'`. +- {ref}`Visible reasoning summaries ` are streamed to standard error by `llm prompt` and `llm chat`. Use `-R/--hide-reasoning` or the new `hide_reasoning=True` Python argument to hide them. Encrypted reasoning metadata is preserved for subsequent turns. +- The default model for users who have not selected one is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna), replacing GPT-4o mini. New built-in models include `gpt-5.6-sol`, `gpt-5.6-terra` and `gpt-5.6-luna`; models that are no longer available from OpenAI have been removed. See {ref}`0.32rc1 ` and {ref}`0.32rc2 ` for details. +- New {ref}`llm openai endpoint ` command runs prompts and chats, or lists models, against an arbitrary OpenAI-compatible endpoint without configuring it first. These calls are not logged. +- OpenAI models now support a `service_tier` option. Use `-o service_tier fast` for faster responses at a higher price, or `-o service_tier flex` for slower, cheaper processing on supported models. See {ref}`Fast mode and service tiers `. [#1585](https://github.com/simonw/llm/pull/1585) +- New `llm -m model --options` flag lists the options supported by a model. The Python prompt APIs now accept an explicit {ref}`options= dictionary ` as well as the previous keyword-argument form. + +### More controllable tool loops + +- Every tool call now has a unique `tool_call_id`, synthesized when the provider does not supply one. Tool implementations can {ref}`accept an llm_tool_call parameter ` to inspect the current call and its ID. +- Tools can {ref}`raise llm.PauseChain ` to pause execution for human approval or another external event. Chains can later {ref}`resume from a message history ending in unresolved tool calls `, without repeating calls that already have results. +- {ref}`Conversations that use configured tools ` can be continued with `llm -c` or `llm chat -c` without repeating the toolbox configuration. +- `llm tools` now shows constructor signatures and docstrings for {ref}`dynamic toolboxes `. Passing a toolbox specification instantiates it and lists its runtime-generated tools, while `llm tools --json` identifies dynamic toolboxes with a `"dynamic"` boolean. [#1580](https://github.com/simonw/llm/issues/1580) +- Models can now {ref}`declare the server-side tools they support ` using the instance-level `supported_server_side_tools` property and the new `llm.ServerSideTool` base class. Server-executed calls and results are captured as structured message parts, and `llm -c` restores configured server-side tools for continued conversations. [#1592](https://github.com/simonw/llm/issues/1592), [#1593](https://github.com/simonw/llm/pull/1593) +- {ref}`llm tools -m MODEL ` lists the server-side tools supported by that model. `llm models --json` returns model aliases, capability flags, attachment types and server-side tools, with option schemas included when combined with `--options`. +- {ref}`OpenAI-compatible Responses endpoints ` can use provider-specific server-side tools with `ServerSideTool(spec={...})`, including OpenRouter's web search implementation. + +See {ref}`0.32a3 ` for more detail on pausing, resuming and inspecting tool calls. + +### New SQLite logging schema + +LLM now logs prompts and responses using a new schema built around threads, turns and a **content-addressed message store**. Existing records in the legacy `responses` table are left untouched, and `llm logs` combines both generations of data. You can {ref}`create a backup ` before upgrading using: + + llm logs backup logs-backup.db + +- Messages are stored once and referenced by their content hash, preserving structured text, reasoning, attachments and tool activity without duplicating repeated conversation history. See {ref}`the message store documentation `. +- Raw provider payloads are stored in `turns.response_json`, {ref}`condensed ` using [condense-json](https://github.com/simonw/condense-json). `llm logs --json` expands them back to their original shape, and `LogStore.turn_response_json(turn_id)` returns them from Python. [#1586](https://github.com/simonw/llm/pull/1586) +- Model plugins can define {ref}`json_replacements ` dictionaries to further improve payload compression. +- {ref}`Full-text search `, {ref}`model ` and {ref}`tool filters `, and {ref}`conversation views ` work across both the legacy and new tables. Logs now record which configured toolbox instance supplied each tool. +- New {ref}`Response.log_to_db() ` Python API writes a response to a logs database. `llm prompt --json` outputs the same structured representation as `llm logs --json`, even when persistent logging is disabled. +- New {ref}`message_tree SQL view ` renders conversation threads as indented text outlines for direct SQL exploration. +- LLM now requires [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0) or higher and no longer depends on `sqlite-migrate`. + +See {ref}`0.32rc1 ` for the detailed migration notes and complete list of logging changes. + +### Fixes since 0.32rc2 + +- Fixed streamed OpenAI Responses API calls recording two different ciphertexts of the same reasoning: the part's `encrypted_content` was harvested from the `response.output_item.done` event while `response_json` came from `response.completed`, and OpenAI encrypts per event. Reasoning metadata is now re-emitted from the final payload, so both records agree on one blob. +- Attachments loaded from URLs now follow up to three redirects when detecting their content type or fetching their bytes. Thanks, [Ojas Sharma](https://github.com/ojassharma7). [#1046](https://github.com/simonw/llm/issues/1046), [#1579](https://github.com/simonw/llm/pull/1579) +- Fixed a bug where `llm openai endpoint --schema` was ignored if the selected template also defined a schema. Thanks, [ikatyal2110](https://github.com/ikatyal2110). [#1588](https://github.com/simonw/llm/pull/1588) +- `llm logs status` now counts records in the new `threads` and `turns` tables, with legacy conversation and response counts shown separately when present. +- `Response.to_dict()` now executes an unconsumed synchronous response before serializing it instead of producing an empty assistant message list. +- `Response.from_dict()` now restores pending client-side tool calls so they can be inspected, executed or continued using `response.reply(tools=[...])`. +- `Response.reply()` now correctly passes {ref}`attachments returned by tools ` to the next model call, for both synchronous and asynchronous responses. + +(v0_32_rc2)= +## 0.32rc2 (2026-07-30) + +- The default model for users who have not set their own default is now [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna). It was previously [GPT-4o mini](https://developers.openai.com/api/docs/models/gpt-4o-mini). Luna is a much better and more recent model, albeit slightly more expensive - $0.20 per million input tokens and $1.20 per million output tokens, compared to $0.15/$0.60 for 4o mini. You can switch back to 4o mini using `llm models default gpt-4o-mini`, or switch to [GPT-5 nano](https://developers.openai.com/api/docs/models/gpt-5-nano), an even cheaper default model ($0.05/$0.40), using `llm models default gpt-5-nano`. [#1576](https://github.com/simonw/llm/issues/1576) +- New {ref}`llm openai endpoint ` command for running prompts, chats and model listings against arbitrary OpenAI-compatible endpoints without first configuring a model. These calls are not logged. [#1565](https://github.com/simonw/llm/issues/1565) +- No longer depends on [sqlite-migrate](https://github.com/simonw/sqlite-migrate), since that functionality is now handled by [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0). [#1577](https://github.com/simonw/llm/issues/1577) +- Fixed a bug where server-executed tool calls were incorrectly reported as pending by `LogStore.pending_tool_calls()`. Thanks, [ikatyal2110](https://github.com/ikatyal2110). [#1574](https://github.com/simonw/llm/issues/1574) +- `schema_dsl()` now raises a descriptive `ValueError` instead of an `IndexError` for malformed fields with no name before the colon. [#1466](https://github.com/simonw/llm/issues/1466) + +(v0_32_rc1)= +## 0.32rc1 (2026-07-30) + +This release candidate for 0.32 introduces a new database schema for logging prompts and responses that captures full details of the interaction with the underlying LLM, and de-duplicates those records using a **content-addressed message store**. + +Upgrading to this RC will create those new tables and start logging to them. Existing data in the `responses` table will be left unaffected, and the `logs` command will read from both old and new tables. New interactions will only be written to the new tables. + +You can create a backup of your logs database prior to upgrading using: + + llm logs backup logs-backup.db + +### Changes to SQLite logging + +- See {ref}`the message store documentation ` for details of the new logging schema. +- `llm prompt` logs are now written to a new set of tables, not the legacy tables. +- New documented {ref}`Response.log_to_db() ` Python API for writing to a SQLite logs database. +- `llm logs` now reads both generations of log tables: history recorded by older versions of LLM in the legacy `responses` table is merged into the output alongside new turns, with rows from the dual-write era deduplicated by id. The `-m`, `-c`, `-f`, `-T`, `--tools` and `--schema` filters work across both. [#1562](https://github.com/simonw/llm/pull/1562) +- Raw provider payloads (the old `prompt_json` and `response_json` columns) are no longer persisted - the stored message chain is the record of what was sent and returned. This means data that only ever lived in those raw payloads is no longer stored, notably the log probabilities returned by OpenAI completion models with `-o logprobs`. +- `llm logs -q` full-text search works against the new log tables. Search covers the prompt text you typed and the model's text responses - system prompts, fragment contents, tool activity and reasoning traces are excluded from the index. See {ref}`logging-search`. +- `llm logs` now shows which configured toolbox instance provided each tool - in the tools list for a prompt and on each tool result, so a `SQLite_query` result records that it ran against `SQLite("mydb.db")`. +- New {ref}`message_tree SQL view ` that renders each conversation thread as an indented text outline, for exploring the message store directly with SQL. [#1562](https://github.com/simonw/llm/pull/1562) + +### Other changes + +- New OpenAI models: `gpt-5.6-sol`, `gpt-5.6-terra` and `gpt-5.6-luna`. +- Removed OpenAI models that are no longer available via the OpenAI API: `chatgpt-4o-latest`, the `gpt-4o` and `gpt-4o-mini` audio preview models, `gpt-4-32k`, `gpt-4-1106-preview`, `gpt-4-0125-preview`, the `gpt-4.5` preview models, `o1-preview`, `o1-mini` and `gpt-5.1-chat-latest`. [#1553](https://github.com/simonw/llm/issues/1553) +- Continuing a conversation that used a configured toolbox now works: `llm -c` and `llm chat -c` reconstruct each toolbox instance from its recorded configuration, so a conversation started with `-T 'Datasette("https://datasette.io/content")'` can be continued without repeating the `-T` option. +- New `llm prompt --json` option which outputs a JSON array describing the prompt and the response, in the same format as `llm logs --json`. This works even with `--no-log` or logging turned off, in which case the response is logged to a temporary in-memory database to build the JSON. [#1566](https://github.com/simonw/llm/pull/1566) +- New `response.execute_tool_call(tool_call)` method (awaitable on async responses) that executes a single tool call and returns its `ToolResult`. This is designed for plugins wrapping provider SDKs that orchestrate the tool loop themselves, described in {ref}`the advanced model plugins documentation `. +- Documentation for the `responses: true` setting in `extra-openai-models.yaml`, which opts a custom OpenAI model into the Responses API instead of Chat Completions. See {ref}`openai-extra-models`. +- Fixed a bug where the system prompt was omitted from the pre-computed `prompt.messages` on the first turn of a conversation, so models that build their request from that message list never received it. Thanks, [Niall Smart](https://github.com/niallsmart). [#1478](https://github.com/simonw/llm/issues/1478) +- Fixed a bug on Windows where a fragment argument holding an absolute path such as `C:\Users\demo\notes.txt` was mistaken for a reference to a `c:` fragment loader plugin. Fragment arguments that match an existing file path now always resolve as files. [#1563](https://github.com/simonw/llm/issues/1563) +- LLM now requires [sqlite-utils 4.0](https://sqlite-utils.datasette.io/en/stable/changelog.html#v4-0) or higher. + +(v0_31_1)= +## 0.31.1 (2026-07-09) + +- Fix for a bug with OpenAI Chat Completion endpoints where a tool call with empty arguments could result in a JSON error from some providers. [#1521](https://github.com/simonw/llm/issues/1521) + +(v0_32_a3)= +## 0.32a3 (2026-06-09) + +Driven by the needs of [Datasette Agent](https://github.com/datasette/datasette-agent)'s human-in-the-loop `ask_user()` feature, made the following improvements to how tool calls work: + +- Tool implementations can declare a parameter named `llm_tool_call` in order to be passed the `llm.ToolCall` object for the current invocation. This allows them to access the current `llm_tool_call.tool_call_id`. See {ref}`python-api-tools-llm-tool-call`. [#1480](https://github.com/simonw/llm/pull/1480) +- Every tool call is now guaranteed a unique `tool_call_id` - providers that do not supply one get a synthesized `tc_`-prefixed ULID. [#1481](https://github.com/simonw/llm/pull/1481) +- Tools can raise a `llm.PauseChain` exception to cleanly pause the tool chain, useful for things like waiting for human approval. The exception propagates to the caller with `.tool_call` and `.tool_results` (completed sibling results) attached, and no model call is made with a placeholder result. See {ref}`python-api-tools-pause`. [#1482](https://github.com/simonw/llm/pull/1482) +- Failure semantics for concurrent tool execution: async sibling tool calls always run to completion before a pause or hook exception propagates. [#1482](https://github.com/simonw/llm/pull/1482) +- Chains can now resume from a `messages=` history ending in unresolved tool calls: the calls are executed through the normal `before_call`/`after_call` machinery before the first model call, skipping any that already have results. The `execute_tool_calls()` method also accepts a new optional `tool_calls_list=` argument for executing an explicit list of `ToolCall` objects in place of the calls requested by the response. See {ref}`python-api-tools-resume`. [#1482](https://github.com/simonw/llm/pull/1482) +- Fixed a bug where the async tool executor silently dropped calls to tools not present in `tools=` - these now return `Error: tool "..." does not exist` results, matching the sync executor. [#1483](https://github.com/simonw/llm/pull/1483) + +(v0_32_a2)= +## 0.32a2 (2026-05-12) + +### Support for the OpenAI Responses API + +Most reasoning-capable OpenAI models now use the [`/v1/responses`](https://developers.openai.com/api/reference/responses/overview) endpoint instead of `/v1/chat/completions`. This enables interleaved reasoning across tool calls for GPT-5 class models. [#1435](https://github.com/simonw/llm/pull/1435) + +- New `Responses` and `AsyncResponses` model classes driving the OpenAI Responses API. The existing `Chat` and `AsyncChat` classes are unchanged so other plugins that import them keep working. +- The following models now use the Responses API by default: `o1`, `o3-mini`, `o3`, `o4-mini`, `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5.1`, `gpt-5.2`, `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano`, `gpt-5.5` (and their pinned date variants). +- Use `-o chat_completions 1` to fall back to the older `/v1/chat/completions` code path for any of these models. +- Encrypted reasoning items are captured as `provider_metadata` on `ReasoningPart` objects and round-tripped back to OpenAI on subsequent turns. +- Reasoning summaries are now requested with `"summary": "auto"` so visible reasoning text is streamed back where the model produces it, unless `--hide-reasoning` or `hide_reasoning=` is set. +- This means OpenAI prompts run using `llm prompt` that return reasoning tokens will display those on standard error. + +### CLI + +- New `llm -m model --options` flag to list the options supported by a given model. [#1441](https://github.com/simonw/llm/pull/1441) +- The `-R/--no-reasoning` option has been renamed to `-R/--hide-reasoning`. + +### Python API + +- New `hide_reasoning=True` keyword argument on `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, and their async counterparts, exposed to model plugins as `prompt.hide_reasoning`. Model plugins can {ref}`use this to decide ` if they should request visible reasoning summaries from their providers. [#1442](https://github.com/simonw/llm/pull/1442) +- New `options=` dict keyword argument on `Model.prompt()`, `Conversation.prompt()`, `Response.reply()`, and their async equivalents, matching the pattern already used by `.chain()`. The previous `**kwargs` form continues to work for backwards compatibility but is no longer documented, and will be removed in the future. [#1432](https://github.com/simonw/llm/pull/1432) + +### Bug fixes + +- `add_tool_call()` calls that were not also recorded as stream events are now correctly emitted as `ToolCallPart` objects when assembling response parts, so they survive serialization via `response.to_dict()`. [#1433](https://github.com/simonw/llm/issues/1433) + +(v0_32_a1)= +## 0.32a1 (2026-04-29) + +- Fixed a bug in 0.32a0 where tool-calling conversations were not correctly reinflated from SQLite. [#1426](https://github.com/simonw/llm/issues/1426) + +(v0_32_a0)= +## 0.32a0 (2026-04-28) + +This alpha introduces a major backwards-compatible refactor. Models can now be prompted with a list of messages, OpenAI Chat Completions style, and the response can now be iterated over as a sequence of mixed types of content, for example reasoning tokens mixed with text tokens mixed with tool calls. + +For more background on this release take a look at [the annotated release notes](https://simonwillison.net/2026/Apr/29/llm/) on my blog. + +Prompt inputs and response outputs are now expressed as a list of `Message` objects, each containing typed `Part` objects (text, reasoning, tool calls, tool results, attachments). + +The `llm` CLI tool can now display reasoning tokens while executing a prompt. + +Plugin authors should read the expanded {ref}`Advanced model plugins ` documentation, which now covers `StreamEvent`, consuming `prompt.messages`, and round-tripping opaque provider metadata such as Anthropic extended-thinking signatures and Gemini `thoughtSignature` values. + +### Structured messages and streaming events + +- New `llm.Message` value type and constructor helpers `llm.user()`, `llm.assistant()`, `llm.system()`, and `llm.tool_message()` for building structured prompt inputs. The helpers accept strings, `Attachment` instances, or nested `Part` lists. +- New `messages=` keyword argument on `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, and their async counterparts. The `prompt=`, `system=`, `attachments=`, and `tool_results=` keywords still work and synthesize into the same `Message` list internally. +- New `response.stream_events()` and `response.astream_events()` methods yielding typed `StreamEvent` objects (`type` is one of `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, `"tool_result"`, plus a `redacted=True` marker for opaque reasoning). Iterating against `response` directly continues to yield only text strings. +- New `response.messages()` method (async: `await response.messages()`) returning the assembled `list[Message]` produced by the model. Calling it forces execution if the response prompt has not yet been executed. +- New `response.reply(prompt=None, **kwargs)` method that continues the conversation from any `Response`, regardless of origin. When the previous response made tool calls and `tool_results=` was not passed, `reply()` automatically executes the pending tool calls and threads the results into the next turn. On async responses `reply()` is awaitable. +- New `response.to_dict()` and `Response.from_dict(data, *, model=None)` for JSON-safe serialization of a full conversation turn — model id, input chain, assembled output (including reasoning parts and provider metadata), options, and audit fields. Reasoning signatures and `thoughtSignature` values round-trip via `provider_metadata`, so multi-turn extended thinking works across process boundaries. +- New `llm/serialization.py` module exposing `MessageDict`, `PartDict`, `ResponseDict`, `PromptDict`, `UsageDict`, `AttachmentDict`, and the per-Part TypedDicts. Every `to_dict()` / `from_dict()` method is annotated with the matching TypedDict. +- `Response.prompt.messages` is now the canonical structured input across the entire conversation chain. `Conversation.prompt` and `AsyncConversation.prompt` pre-compute the full chain (prior input + prior output + new turn) before constructing the next `Prompt`, so `response.prompt.messages` is always exactly what the model was sent. + +### CLI + +- `llm prompt` and `llm chat` now display visible reasoning text to stderr in a dim style while the response streams. +- New `-R/--hide-reasoning` flag for `llm prompt` and `llm chat` to hide the reasoning stream. +- `llm logs` now renders any visible reasoning emitted during a response under a `## Reasoning` heading above the response. +- New `reasoning` column on the `responses` table populated from the visible-reasoning text. + +(v0_31)= +## 0.31 (2026-04-24) + +- New GPT-5.5 OpenAI model: `llm -m gpt-5.5`. [#1418](https://github.com/simonw/llm/issues/1418) +- New option to set the [text verbosity level](https://developers.openai.com/cookbook/examples/gpt-5/gpt-5_new_params_and_tools#1-verbosity-parameter) for GPT-5+ OpenAI models: `-o verbosity low`. Values are `low`, `medium`, `high`. +- New option for setting the [image detail level](https://developers.openai.com/api/docs/guides/images-vision#choose-an-image-detail-level) used for image attachments to OpenAI models: `-o image_detail low` - values are `low`, `high` and `auto`, and GPT-5.4 and 5.5 also accept `original`. +- Models listed in `extra-openai-models.yaml` are now also registered as asynchronous. [#1395](https://github.com/simonw/llm/issues/1395) + +(v0_30)= +## 0.30 (2026-03-31) + +- The {ref}`register_models() plugin hook ` now takes an optional `model_aliases` parameter listing all of the models, async models and aliases that have been registered so far by other plugins. A plugin with `@hookimpl(trylast=True)` can use this to take previously registered models into account. [#1389](https://github.com/simonw/llm/issues/1389) +- Added docstrings to public classes and methods and included those directly in the documentation. + +(v0_29)= +## 0.29 (2026-03-17) + +- The `-t/--template` option now works correctly with the `-x/--extract` and `--xl/--extract-last` flags. +- `llm logs` now shows any additional model options in the Markdown output. [#1322](https://github.com/simonw/llm/issues/1322) +- New OpenAI models: `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano`. [#1376](https://github.com/simonw/llm/issues/1376) + (v0_28)= ## 0.28 (2025-12-12) @@ -415,7 +630,7 @@ response = model.prompt( ``` Plugins that provide alternative models can support attachments, see {ref}`advanced-model-plugins-attachments` for details. -The latest **[llm-claude-3](https://github.com/simonw/llm-claude-3)** plugin now supports attachments for Anthropic's Claude 3 and 3.5 models. The **[llm-gemini](https://github.com/simonw/llm-gemini)** plugin supports attachments for Google's Gemini 1.5 models. +The latest **[llm-claude-3](https://github.com/simonw/llm-claude-3)** plugin now supports attachments for Anthropic's Claude 3 and 3.5 models. The **[llm-gemini](https://github.com/simonw/llm-gemini)** plugin supports attachments for Google's `gemini-flash-latest` model. Also in this release: OpenAI models now record their `"usage"` data in the database even when the response was streamed. These records can be viewed using `llm logs --json`. [#591](https://github.com/simonw/llm/issues/591) diff --git a/docs/conf.py b/docs/conf.py index f4a2d953c..7783599a2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - from subprocess import PIPE, Popen # This file is execfile()d with the current directory set to its diff --git a/docs/contributing.md b/docs/contributing.md index 3459258fc..63e2e4ec5 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -27,7 +27,7 @@ The default OpenAI plugin has a debugging mechanism for showing the exact reques Set the `LLM_OPENAI_SHOW_RESPONSES` environment variable like this: ```bash -LLM_OPENAI_SHOW_RESPONSES=1 uv run llm -m chatgpt 'three word slogan for an an otter-run bakery' +LLM_OPENAI_SHOW_RESPONSES=1 uv run llm -m chatgpt 'three word slogan for an otter-run bakery' ``` This will output details of the API requests and responses to the console. @@ -35,7 +35,7 @@ Use `--no-stream` to see a more readable version of the body that avoids streami ```bash LLM_OPENAI_SHOW_RESPONSES=1 uv run llm -m chatgpt --no-stream \ - 'three word slogan for an an otter-run bakery' + 'three word slogan for an otter-run bakery' ``` ## Documentation diff --git a/docs/embeddings/python-api.md b/docs/embeddings/python-api.md index ca586991c..fe97da3db 100644 --- a/docs/embeddings/python-api.md +++ b/docs/embeddings/python-api.md @@ -13,8 +13,10 @@ vector = embedding_model.embed("my happy hound") ``` If the embedding model can handle binary input, you can call `.embed()` with a byte string instead. You can check the `supports_binary` property to see if this is supported: ```python +from pathlib import Path + if embedding_model.supports_binary: - vector = embedding_model.embed(open("my-image.jpg", "rb").read()) + vector = embedding_model.embed(Path("my-image.jpg").read_bytes()) ``` The `embedding_model.supports_text` property indicates if the model supports text input. @@ -122,12 +124,12 @@ A collection instance has the following properties and methods: - `model_id` - the string ID of the embedding model used for this collection - `model()` - returns the `EmbeddingModel` instance, based on that `model_id` - `count()` - returns the integer number of items in the collection -- `embed(id: str, text: str, metadata: dict=None, store: bool=False)` - embeds the given string and stores it in the collection under the given ID. Can optionally include metadata (stored as JSON) and store the text content itself in the database table. -- `embed_multi(entries: Iterable, store: bool=False, batch_size: int=100)` - see above -- `embed_multi_with_metadata(entries: Iterable, store: bool=False, batch_size: int=100)` - see above -- `similar(query: str, number: int=10)` - returns a list of entries that are most similar to the embedding of the given query string -- `similar_by_id(id: str, number: int=10)` - returns a list of entries that are most similar to the embedding of the item with the given ID -- `similar_by_vector(vector: List[float], number: int=10, skip_id: str=None)` - returns a list of entries that are most similar to the given embedding vector, optionally skipping the entry with the given ID +- `embed(id: str, value: str | bytes, metadata: dict[str, Any] | None = None, store: bool = False)` - embeds the given value and stores it in the collection under the given ID. Can optionally include metadata (stored as JSON) and store the text or binary content itself in the database table. +- `embed_multi(entries: Iterable[tuple[str, str | bytes]], store: bool = False, batch_size: int = 100)` - see above +- `embed_multi_with_metadata(entries: Iterable[tuple[str, str | bytes, dict[str, Any] | None]], store: bool = False, batch_size: int = 100)` - see above +- `similar(value: str | bytes, number: int = 10, prefix: str | None = None)` - returns a list of entries that are most similar to the embedding of the given value +- `similar_by_id(id: str, number: int = 10, prefix: str | None = None)` - returns a list of entries that are most similar to the embedding of the item with the given ID +- `similar_by_vector(vector: list[float], number: int = 10, skip_id: str | None = None, prefix: str | None = None)` - returns a list of entries that are most similar to the given embedding vector, optionally skipping the entry with the given ID - `delete()` - deletes the collection and its embeddings from the database There is also a `Collection.exists(db, name)` class method which returns a boolean value and can be used to determine if a collection exists or not in a database: @@ -191,21 +193,21 @@ for table in ("collections", "embeddings"): cog.out("```\n") ]]] --> ```sql -CREATE TABLE [collections] ( - [id] INTEGER PRIMARY KEY, - [name] TEXT, - [model] TEXT +CREATE TABLE "collections" ( + "id" INTEGER PRIMARY KEY, + "name" TEXT, + "model" TEXT ) CREATE TABLE "embeddings" ( - [collection_id] INTEGER REFERENCES [collections]([id]), - [id] TEXT, - [embedding] BLOB, - [content] TEXT, - [content_blob] BLOB, - [content_hash] BLOB, - [metadata] TEXT, - [updated] INTEGER, - PRIMARY KEY ([collection_id], [id]) + "collection_id" INTEGER REFERENCES "collections"("id"), + "id" TEXT, + "embedding" BLOB, + "content" TEXT, + "content_blob" BLOB, + "content_hash" BLOB, + "metadata" TEXT, + "updated" INTEGER, + PRIMARY KEY ("collection_id", "id") ) ``` diff --git a/docs/embeddings/writing-plugins.md b/docs/embeddings/writing-plugins.md index 0e5cd0f7d..ec8ff8ae8 100644 --- a/docs/embeddings/writing-plugins.md +++ b/docs/embeddings/writing-plugins.md @@ -14,6 +14,11 @@ There are two components to an embedding model plugin: The following example uses the [sentence-transformers](https://github.com/UKPLab/sentence-transformers) package to provide access to the [MiniLM-L6](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) embedding model. +```{eval-rst} +.. autoclass:: llm.EmbeddingModel + :members: embed, embed_multi, embed_batch +``` + ```python import llm from sentence_transformers import SentenceTransformer diff --git a/docs/fragments.md b/docs/fragments.md index 572f84049..366505b55 100644 --- a/docs/fragments.md +++ b/docs/fragments.md @@ -30,7 +30,7 @@ from importlib.metadata import version llm_version = version("llm") cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.') ]]]--> -The URL will be fetched with the user-agent `llm/0.28 (https://llm.datasette.io/)`. +The URL will be fetched with the user-agent `llm/0.32 (https://llm.datasette.io/)`. The `-f` option can be used multiple times to combine together multiple fragments. diff --git a/docs/help.md b/docs/help.md index a5a63566e..391900318 100644 --- a/docs/help.md +++ b/docs/help.md @@ -80,7 +80,7 @@ Commands: keys Manage stored API keys for different models logs Tools for exploring logged prompts and responses models Manage available models - openai Commands for working directly with the OpenAI API + openai Commands for working with OpenAI and OpenAI-compatible APIs plugins List installed plugins schemas Manage stored schemas similar Return top N similar IDs from a collection using cosine... @@ -101,7 +101,7 @@ Usage: llm prompt [OPTIONS] [PROMPT] Examples: llm 'Capital of France?' - llm 'Capital of France?' -m gpt-4o + llm 'Capital of France?' -m gpt-5.5 llm 'Capital of France?' -s 'answer in Spanish' Multi-modal models can be called with attachments like this: @@ -134,6 +134,7 @@ Options: --cl, --chain-limit INTEGER How many chained tool responses to allow, default 5, set 0 for unlimited -o, --option ... key/value options for the model + --options Show options for the selected model --schema TEXT JSON schema, filepath or ID --schema-multi TEXT JSON schema to use for multiple results -f, --fragment TEXT Fragment (alias, URL, hash or file path) to @@ -144,6 +145,7 @@ Options: --no-stream Do not stream output -n, --no-log Don't log to database --log Log prompt and response to the database + -R, --hide-reasoning Hide reasoning output -c, --continue Continue the most recent conversation. --cid, --conversation TEXT Continue the conversation with the given ID. --key TEXT API key to use @@ -152,6 +154,8 @@ Options: -u, --usage Show token usage -x, --extract Extract first fenced code block --xl, --extract-last Extract last fenced code block + --json Output the response as JSON, same format as + llm logs --json -h, --help Show this message and exit. ``` @@ -175,6 +179,7 @@ Options: -o, --option ... key/value options for the model -d, --database FILE Path to log database --no-stream Do not stream output + -R, --hide-reasoning Hide reasoning output --key TEXT API key to use -T, --tool TEXT Name of a tool to make available to the model --functions TEXT Python code block or file path defining @@ -397,6 +402,7 @@ Options: --async List async models --schemas List models that support schemas --tools List models that support tools + --json Output as JSON -q, --query TEXT Search for models matching these strings -m, --model TEXT Specific model IDs -h, --help Show this message and exit. @@ -454,7 +460,7 @@ Usage: llm models options show [OPTIONS] MODEL Example usage: - llm models options show gpt-4o + llm models options show gpt-4.1 Options: -h, --help Show this message and exit. @@ -469,7 +475,7 @@ Usage: llm models options set [OPTIONS] MODEL KEY VALUE Example usage: - llm models options set gpt-4o temperature 0.5 + llm models options set gpt-4.1 temperature 0.5 Options: -h, --help Show this message and exit. @@ -484,9 +490,9 @@ Usage: llm models options clear [OPTIONS] MODEL [KEY] Example usage: - llm models options clear gpt-4o + llm models options clear gpt-4.1 # Or for a single option - llm models options clear gpt-4o temperature + llm models options clear gpt-4.1 temperature Options: -h, --help Show this message and exit. @@ -634,7 +640,7 @@ Options: -h, --help Show this message and exit. Commands: - list* List available tools that have been provided by plugins + list* List available tools, optionally including tools supported by a model ``` (help-tools-list)= @@ -642,10 +648,11 @@ Commands: ``` Usage: llm tools list [OPTIONS] [TOOL_DEFS]... - List available tools that have been provided by plugins + List available tools, optionally including tools supported by a model Options: --json Output as JSON + -m, --model TEXT List tools supported by this model --functions TEXT Python code block or file path defining functions to register as tools -h, --help Show this message and exit. @@ -689,12 +696,12 @@ Usage: llm aliases set [OPTIONS] ALIAS [MODEL_ID] Example usage: - llm aliases set mini gpt-4o-mini + llm aliases set luna gpt-5.6-luna Alternatively you can omit the model ID and specify one or more -q options. The first model matching all of those query strings will be used. - llm aliases set mini -q 4o -q mini + llm aliases set luna -q gpt -q luna Options: -q, --query TEXT Set alias for model matching these strings @@ -1061,13 +1068,57 @@ Options: ``` Usage: llm openai [OPTIONS] COMMAND [ARGS]... - Commands for working directly with the OpenAI API + Commands for working with OpenAI and OpenAI-compatible APIs Options: -h, --help Show this message and exit. Commands: - models List models available to you from the OpenAI API + endpoint Run against an OpenAI-compatible endpoint without logging. + models List models available to you from the OpenAI API +``` + +(help-openai-endpoint)= +#### llm openai endpoint --help +``` +Usage: llm openai endpoint [OPTIONS] URL [PROMPT] + + Run against an OpenAI-compatible endpoint without logging. + + PROMPT or stdin is executed once. If neither is provided, wait for input on + stdin. Use --chat to start an interactive chat. Templates run once by default; + use --chat to apply one interactively. Use --models to list the available + model IDs without running a prompt. + +Options: + -m, --model TEXT Model ID (required unless --models or provided + by template) + -s, --system TEXT System prompt to use + -t, --template TEXT Template to use + -p, --param ... Parameters for template + -o, --option ... key/value options for the model + --schema TEXT JSON schema, filepath or ID + --schema-multi TEXT JSON schema to use for multiple results + -a, --attachment ATTACHMENT Attachment path or URL or - + --at, --attachment-type ... + Attachment with explicit mimetype, + --at image.jpg image/jpeg + -T, --tool TEXT Name of a tool to make available to the model + --functions TEXT Python code block or file path defining + functions to register as tools + --td, --tools-debug Show full details of tool executions + --ta, --tools-approve Manually approve every tool execution + --cl, --chain-limit INTEGER How many chained tool responses to allow, + default 5, set 0 for unlimited + --key TEXT API key or stored key alias to send + -H, --header ... Additional HTTP header + --responses Use the Responses API instead of Chat + Completions + --chat Start an interactive chat + --models List model IDs from the endpoint and exit + --no-stream Do not stream output + -R, --hide-reasoning Hide reasoning output + -h, --help Show this message and exit. ``` (help-openai-models)= @@ -1082,4 +1133,4 @@ Options: --key TEXT OpenAI API key -h, --help Show this message and exit. ``` - \ No newline at end of file + diff --git a/docs/index.md b/docs/index.md index 4536f5d1a..939ef3807 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,7 @@ [![Discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord-llm) [![Homebrew](https://img.shields.io/homebrew/installs/dy/llm?color=yellow&label=homebrew&logo=homebrew)](https://formulae.brew.sh/formula/llm) -A CLI tool and Python library for interacting with **OpenAI**, **Anthropic's Claude**, **Google's Gemini**, **Meta's Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. +A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Qwen**, **Gemma**, **Kimi**, **DeepSeek**, **Mistral**, and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/). @@ -39,12 +39,24 @@ Or with [uv](https://docs.astral.sh/uv/guides/tools/) ```bash uv tool install llm ``` +Use LLM to run prompts or start chats against an arbitrary OpenAI-compatible Chat Completions endpoint, such as [LM Studio](https://lmstudio.ai). With `uvx`, you can do this without installing LLM first: +```bash +uvx llm openai endpoint http://localhost:1234/v1 \ + -m google/gemma-4-12b \ + "What is the capital of France?" + +uvx llm openai endpoint http://localhost:1234/v1 \ + -m google/gemma-4-12b \ + --chat +``` +Add `--key your-api-key` if the endpoint requires authentication. See {ref}`Run against an endpoint without configuring it ` for more options. + If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: ```bash # Paste your OpenAI API key into this llm keys set openai -# Run a prompt (with the default gpt-4o-mini model) +# Run a prompt (with the default gpt-5.6-luna model) llm "Ten fun names for a pet pelican" # Extract text from an image @@ -58,12 +70,12 @@ Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic]( llm install llm-gemini llm keys set gemini # Paste Gemini API key here -llm -m gemini-2.0-flash 'Tell me fun facts about Mountain View' +llm -m gemini-3.5-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here -llm -m claude-4-opus 'Impress me with wild facts about turnips' +llm -m claude-sonnet-5 'Impress me with wild facts about turnips' ``` You can also {ref}`install a plugin ` to access models that can run on your local device. If you use [Ollama](https://ollama.com/): ```bash @@ -90,17 +102,24 @@ Why don't pelicans like to tip waiters? Because they always have a big bill! ``` -More background on this project: - -- [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) -- [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) -- [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) -- [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) -- [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) -- [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) -- [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) - -See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. +## Project news + +- 29th April 2026: [LLM 0.32a0 is a major backwards-compatible refactor](https://simonwillison.net/2026/Apr/29/llm/) +- 11th August 2025: [LLM 0.27, the annotated release notes: GPT-5 and improved tool calling](https://simonwillison.net/2025/Aug/11/llm-027/) +- 27th May 2025: [Large Language Models can run tools in your terminal with LLM 0.26](https://simonwillison.net/2025/May/27/llm-tools/) +- 5th May 2025: [Feed a video to a vision LLM as a sequence of JPEG frames on the CLI (also LLM 0.25)](https://simonwillison.net/2025/May/5/llm-video-frames/) +- 7th April 2025: [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) +- 28th February 2025: [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) +- 17th February 2025: [LLM 0.22, the annotated release notes](https://simonwillison.net/2025/Feb/17/llm/) +- 29th October 2024: [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) +- 26th January 2024: [LLM 0.13: The annotated release notes](https://simonwillison.net/2024/Jan/26/llm/) +- 12th September 2023: [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) +- 4th September 2023: [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) +- 12th July 2023: [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) +- 18th May 2023: [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) +- 4th April 2023: [The original announcement of the llm CLI tool](https://simonwillison.net/2023/Apr/4/llm/) + +For everything else, see [the llm tag](https://simonwillison.net/tags/llm/) on my blog. ## Contents diff --git a/docs/logging.md b/docs/logging.md index d1a46dcd3..ad0826b95 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -41,8 +41,8 @@ Example output: ``` Logging is ON for all prompts Found log database at /Users/simon/Library/Application Support/io.datasette.llm/logs.db -Number of conversations logged: 33 -Number of responses logged: 48 +Number of threads logged: 33 +Number of turns logged: 48 Database file size: 19.96MB ``` @@ -173,17 +173,26 @@ llm logs --cid 01h82n0q9crqtnzmf13gkyxawg ### Searching the logs -You can search the logs for a search term in the `prompt` or the `response` columns. +You can search the logs for a search term across your prompts and the model's responses. ```bash llm logs -q 'cheesecake' ``` The most relevant results will be shown first. +Search covers the text you typed and the text the model produced, and nothing else. System prompts, {ref}`fragment ` contents, tool calls and their output, and reasoning traces are all excluded from the index - a query only matches words that appeared in a prompt or a response. If a prompt used fragments, the fragment text is not searchable but the question you typed alongside it is. + +Ranking uses [SQLite FTS5](https://www.sqlite.org/fts5.html) relevance scores, with matches in your prompt weighted well above matches in the response - what you asked is usually a stronger signal of what a conversation was about than what came back. The full [FTS5 query syntax](https://www.sqlite.org/fts5.html#full_text_query_syntax) is available, including phrase queries: +```bash +llm logs -q '"pet pelican"' +``` + To switch to sorting with most recent first, add `-l/--latest`. This can be combined with `-n` to limit the number of results shown: ```bash llm logs -q 'cheesecake' -l -n 3 ``` +Search covers both new conversations and history recorded by older versions of LLM. + (logging-filter-id)= ### Filtering past a specific ID @@ -276,6 +285,396 @@ llm logs backup /tmp/backup.db ``` This uses SQLite [VACUUM INTO](https://sqlite.org/lang_vacuum.html#vacuum_with_an_into_clause) under the hood. +(logging-message-store)= + +## The message store + +The `logs.db` database contains two generations of tables. Databases created by older versions of LLM recorded everything in a `responses` table, with companion tables such as `prompt_attachments` and `tool_calls` hanging off it. Current versions write to a set of **content-addressed** tables instead: `threads`, `turns`, `messages` and `parts`. Content-addressed means that rows are identified by a hash of their content rather than an assigned id, so identical content is stored exactly once. + +The legacy tables are read-only history now. The `llm logs` command merges the two generations: rows that only exist in the legacy `responses` table are combined with rows from the new tables, so history recorded by an older version stays visible after an upgrade. All new logging writes only the content-addressed tables. + +(logging-message-store-vocabulary)= + +### Threads, turns, messages and parts + +From the top down: + +- A **thread** is a conversation. It is a named pointer at the message at the head of that conversation, and its id is the conversation id displayed by `llm logs`. +- A **turn** is a single model call within a thread. It records everything specific to that call - which model answered, options, token counts, timings - and points at the messages that were its input and output. +- A **message** is one entry in a conversation, with a role of `system`, `user` or `assistant`. Each message links to its parent, forming a chain, and is identified by a hash of its content combined with that parent link. +- A **part** is a piece of content within a message: text, reasoning, a tool call, a tool result or an attachment. A message's parts are ordered by their `position`. + +(logging-message-store-example)= + +### A worked example + +Here is a two turn conversation, logged to a fresh database and then dumped. It was generated with a small scripted model that returns canned replies - every model plugin logs through the same code path, so rows written by a real model have exactly the same shape, but the canned replies keep this example deterministic. + + +``` +messages and their parts: + +user b2:d6b0cd4e7a65ea90423c50fadb3f5704 + parent: null + part 0: type=text text='Suggest a name for a pet pelican' payload=null + +assistant b2:0f2c02ad982050b623b7e034199c8c61 + parent: b2:d6b0cd4e7a65ea90423c50fadb3f5704 + part 0: type=text text='How about Percy? Pelicans suit a dignified name.' payload=null + +user b2:c785dd6c77540150c2647f406cacc76f + parent: b2:0f2c02ad982050b623b7e034199c8c61 + part 0: type=text text='Now one for a pet walrus' payload=null + +assistant b2:a30a236e0d1b717c592e826d06e3c9d2 + parent: b2:c785dd6c77540150c2647f406cacc76f + part 0: type=text text='Wallace. It pairs nicely with Percy.' payload=null + +turns: + +turn 01kf2rw8jj3nfd5t7w9y1a3c5e + thread_id: 01kf2rw8jhv1x9c2m4p6q8s0tv + parent_message_hash: b2:d6b0cd4e7a65ea90423c50fadb3f5704 + tip_message_hash: b2:0f2c02ad982050b623b7e034199c8c61 + model: scripted + +turn 01kf2rw8jkq7h9k2m4n6p8r0t2 + thread_id: 01kf2rw8jhv1x9c2m4p6q8s0tv + parent_message_hash: b2:c785dd6c77540150c2647f406cacc76f + tip_message_hash: b2:a30a236e0d1b717c592e826d06e3c9d2 + model: scripted + +thread: + +thread 01kf2rw8jhv1x9c2m4p6q8s0tv + name: Suggest a name for a pet pelican + tip_message_hash: b2:a30a236e0d1b717c592e826d06e3c9d2 +``` + + +Things to notice: + +- The four messages form a chain: the first has a `null` parent and each subsequent message names the hash of the one before it. +- The `part N:` lines show each part's storage columns. A part whose text is pure literal keeps it in the `text` column - raw, unescaped and never parsed, so text that happens to look like JSON is safe - with a `null` payload. The `payload` column holds any remaining structure as JSON: fragment references, tool call fields, provider metadata. The part's type lives only in the `type` column. +- Each turn brackets one model call. Its `parent_message_hash` is the tip of the chain that was sent to the model - the last input message, usually that turn's user prompt - and its `tip_message_hash` is the chain tip after the model's reply was appended. The prompt and response that `llm logs` displays are derived by splitting the chain at the parent, rather than being stored a second time. +- The thread's id is the conversation id, its name is derived from the first prompt and its `tip_message_hash` follows the head of the conversation as new turns are logged. +- Turn and thread ids are ULIDs: sortable identifiers issued in time order. The ids shown here are illustrative, since fresh ones are generated on every run. The hashes are not - they depend only on the message content and its position in the chain, so replaying this conversation produces these exact four hashes. + +(logging-message-store-hashes)= + +### Content addressing as a contract + +A message's hash is calculated like this: + +1. Build the object `{"parent": parent_hash, "message": message}`, where `parent_hash` is the hash of the previous message in the chain (or `null` for the first message) and `message` is the message's dictionary representation - its role, its parts and any provider metadata. +2. Serialize that object to canonical JSON: keys sorted, compact `,` and `:` separators with no extra whitespace, non-ASCII characters left unescaped. +3. Hash the UTF-8 encoding of that string with [BLAKE2b](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) using a 16 byte digest, and prefix the hex digest with `b2:`. + +The `b2:` prefix names the algorithm that produced the hash, so any future change to it will be detectable. + +Two design decisions matter here: + +- **The hash covers resolved content.** {ref}`Fragment ` references are expanded to their full text before hashing, and attachments are represented by the SHA-256 hash of their bytes together with their media type - the model sees the type, so identical bytes sent as `image/png` and as `text/plain` are different requests. An attachment supplied as a URL is hashed by that URL: the log records which URL was sent, not whatever it served that day. Attachments loaded from a filesystem path are stored by reference to that file rather than copied into the database, so their fidelity depends on the file staying put - `LogStore.verify()` re-reads the actual bytes when it re-derives every hash, and reports a changed or deleted file as a broken hash rather than letting it pass silently. +- **The parent hash participates in the hash.** The same content appearing at a different point in a conversation is a different node. This is what makes two conversations that share a prefix collapse to shared rows, with no explicit comparison required: replaying the same messages produces the same hashes, and a hash that is already present needs nothing written. + +A consequence of the second decision is that a stateless client - one that holds its own conversation history and re-sends the whole thing with every call - writes only the new tail on each request. It also makes forks cheap, as described next. Here the first conversation from the worked example is logged again, followed by a second conversation that starts with the same prompt and then diverges: + + +``` +message rows after the first conversation: 4 +message rows after both conversations: 6 + +rows added by the second conversation: + +user b2:371ceec468c0ec797ac7042970992c0b + parent: b2:0f2c02ad982050b623b7e034199c8c61 + +assistant b2:4ad19ddf94ea086aa9a8aa6fd8b0af05 + parent: b2:371ceec468c0ec797ac7042970992c0b +``` + + +Two conversations of two turns each - eight messages sent to the model in total - produced six rows. The second conversation's first turn hashed to rows that were already present, so only its second turn was written, and the new user message's parent is the assistant reply both conversations share. + +(logging-message-store-forking)= + +### Forking and shared history + +Because messages form a parent-linked tree, a conversation can fork: a new thread can point at any existing message and continue from there, sharing its entire history with the thread it came from until the two diverge. Nothing is copied when this happens. The `threads.forked_from` column records which thread a fork came from. + +Shared rows cut both ways. Deleting a conversation is not the same as deleting its rows, because another thread may reach the same messages - removing them would silently corrupt that thread's history. For this reason LLM does not currently delete message rows at all; garbage collection of unreachable messages is deliberately left as future work. + +(logging-message-store-references)= + +### Storage by reference + +The hash of a message covers its resolved content, but storage is by reference. A text part whose content borrows from a fragment does not store a copy of that fragment. Instead of a filled `text` column its payload holds a `text_ref` list of fragment references and literal segments: + +```json +{"text_ref": [{"fragment": 1}, {"literal": "\nquestion about it"}]} +``` + +Here fragment `1` is an id in the existing `fragments` table. Reading the part concatenates the fragment content and the literal back together, reproducing the exact text that was hashed. Ask a hundred questions about a novel and the novel is stored once. + +Attachments work the same way: the binary content lives in the `attachments` table, keyed by a SHA-256 hash of the bytes, and the part payload stores that id in place of the data. + +(logging-message-store-response-json)= + +### The raw provider payload + +The parts of a response are a normalized view of what the provider returned. The raw `response.json()` dictionary is also recorded in the `turns.response_json` column, storing details that have no part equivalent - usage breakdowns, system fingerprints, logprobs, and more. + +The JSON is in a condensed form using [condense-json](https://github.com/simonw/condense-json). This library allows values that are stored in the other database rows to be replaced in the JSON as special references such as `{"$": "tool.NAME.description"}`. + +Model plugins can contribute a *dictionary* of their own recurring boilerplate - the same idea as a zstandard custom dictionary - by defining a `json_replacements` class attribute. See {ref}`the plugin author documentation ` for guidance on declaring these. + +`llm logs --json` resolves the stored payload back to the original sen tby the provider as the `response_json` + +(logging-message-store-tables)= + +### Table by table + +The full schema for these tables appears in {ref}`the SQL schema section ` below. + +- `messages` - one row per unique message. `hash` is the content address described above, `parent_hash` links to the previous message in the chain and `role` is `system`, `user` or `assistant`. `provider_metadata` holds any provider-specific data carried by the message; it participates in the hash. +- `parts` - the content of each message, ordered by `position`. When a part's text is stored inline in full it lives in the `text` column - raw and never parsed, so `select text from parts` reads as prose. Text that references fragments is stored in `payload` as `text_ref` instead. `payload` holds any remaining structure as JSON (fragment references, tool call fields, provider metadata), or NULL when the text column carries the whole part. `type` and `tool_name` are their own columns for direct filtering; the type never appears inside the payload. +- `part_attachments` and `part_fragments` - junction tables recording which rows in `attachments` and `fragments` a part's payload references, in order. +- `threads` - one row per conversation. `id` is the conversation id, `tip_message_hash` points at the current head of the conversation and `forked_from` records the thread a fork came from. +- `turns` - one row per model call. `parent_message_hash` and `tip_message_hash` bracket the call's input and output as described above, and the remaining columns record provenance: model, options, schema, token counts, timings and the {ref}`condensed raw provider payload ` in `response_json`. Turn ids are ULIDs, in the same id space as legacy response ids, which is how the two generations of tables sort together in `llm logs`. +- `turn_tools` - which {ref}`tool ` definitions were available to a turn, referencing the `tools` table. For toolbox-derived tools, `instance_id` references the `tool_instances` row recording which configured instance provided them - so the tools list in `llm logs` shows that `SQLite_query` came from `SQLite("mydb.db")` before any call has run. +- `turn_fragments` - which fragments a turn was given, with their `kind` (`prompt` or `system`) and order. Provenance lives here rather than on the shared message rows, and this table is what powers `llm logs -f`. +- `turn_search` - the searchable text of each turn: the literal prompt the user typed (fragment content excluded, via the `text_ref` literals described above) and the assistant's text output. An FTS5 index over this table, `turn_search_fts`, is what powers {ref}`llm logs -q `. Derived from the stored parts when a turn is logged; a turn with no prompt or response text, such as a pure tool call, gets no row. +- `tool_instantiations` - which configured {ref}`toolbox ` instance served a tool call, as a reference into the shared `tool_instances` table (each distinct configuration is stored once), keyed by `(turn_id, tool_call_id)` - call ids supplied by providers are not guaranteed unique across turns. Message rows are shared between conversations and so cannot carry this kind of local execution provenance; this table joins to the chain from outside it. + +(logging-message-store-queries)= + +### Querying the message store + +These queries can be pasted into [Datasette](https://datasette.io/) or `sqlite3` against your `logs.db`. + +The database includes a `message_tree` view that renders every conversation tree as indented text, one row per message, depth-first with forks shown as siblings: + +```sql +select * from message_tree +``` + +Its columns: + +- `root_hash` - the hash of the tree's root message, shared by every message in the tree. Filter or facet on this to isolate a single conversation and its forks. +- `datetime` - when the message was first logged, derived from the earliest turn that recorded it (shared message rows carry no timestamp of their own). +- `message` - the message's text, indented to show its depth in the tree. Text stored as fragment references is resolved back to the fragment content, and messages with no text show a placeholder such as `[tool_result]`. +- `tools` - names of any tools that were executed at that message, comma-separated. +- `message_hash` and `path` - the message's own hash, and the sort key that produces the tree ordering. The rows only read as trees while sorted by `path`, so re-sorting by another column will scramble the indentation. + +
The SQL query behind the message_tree view + +```sql +with recursive msg as ( + select m.hash, m.parent_hash, m.role, m.rowid as rid, + replace(coalesce( + nullif(p.text, ''), + (select f.content from part_fragments pf + join fragments f on f.id = pf.fragment_id + where pf.part_id = p.id + order by pf."order" limit 1), + '[' || coalesce(p.type, 'empty') || ']' + ), char(10), ' ') as text, + (select group_concat(p2.tool_name, ', ') from parts p2 + where p2.message_hash = m.hash and p2.type = 'tool_result' + and p2.tool_name is not null) as tools + from messages m + left join parts p on p.message_hash = m.hash and p.position = 0 +), +tree as ( + select hash, text, tools, 0 as depth, + printf('%012d', rid) as path, hash as root_hash + from msg where parent_hash is null + union all + select msg.hash, msg.text, msg.tools, t.depth + 1, + t.path || '/' || printf('%012d', msg.rid), + t.root_hash + from msg join tree t on msg.parent_hash = t.hash +), +turn_chain as ( + select t.id as turn_id, t.datetime_utc, m.hash, m.parent_hash + from turns t join messages m on m.hash = t.tip_message_hash + union all + select tc.turn_id, tc.datetime_utc, m.hash, m.parent_hash + from turn_chain tc join messages m on m.hash = tc.parent_hash +) +select + t.root_hash, + strftime('%Y-%m-%d %H:%M:%S', + (select min(tc.datetime_utc) from turn_chain tc where tc.hash = t.hash) + ) as datetime, + replace(hex(zeroblob(t.depth)), '00', ' ') || substr(t.text, 1, 60) + as message, + coalesce(t.tools, '') as tools, + t.hash as message_hash, + t.path +from tree t +order by t.path +``` + +
+ +Every turn that was given a specific fragment, via the `turn_fragments` provenance table - set `:fragment_hash` to a hash from `llm fragments`: + +```sql +select turns.id, turns.model, turns.datetime_utc, turn_fragments.kind +from turns +join turn_fragments on turn_fragments.turn_id = turns.id +join fragments on fragments.id = turn_fragments.fragment_id +where fragments.hash = :fragment_hash +order by turns.id; +``` + +The most recently active conversations, with a count of their turns: + +```sql +select + threads.id, + threads.name, + count(turns.id) as num_turns, + max(turns.datetime_utc) as last_used +from threads +left join turns on turns.thread_id = threads.id +group by threads.id +order by last_used desc +limit 10; +``` + +(logging-message-store-python)= + +### Logging from Python + +The supported way to write to a log database from Python is the `log_to_db()` method on a response, which is also what plugins should call: + +```python +import llm +import sqlite_utils + +db = sqlite_utils.Database("logs.db") +model = llm.get_model("gpt-5.5") +response = model.prompt("A short pelican fact") +print(response.text()) +response.log_to_db(db) +``` + +`log_to_db()` takes a `sqlite_utils.Database` and records the response's thread, turn, messages, parts, fragments, attachments and tools in the content-addressed tables. It applies any outstanding migrations itself, so it is safe to call against a brand new database file or one created by an older version of LLM. The underlying `LogStore` class is internal and its API may change - `log_to_db()` and the table schema documented on this page are the supported interfaces. + (logging-sql-schema)= ## SQL schema @@ -300,7 +699,11 @@ cog.out("```sql\n") for table in ( "conversations", "schemas", "responses", "responses_fts", "attachments", "prompt_attachments", "fragments", "fragment_aliases", "prompt_fragments", "system_fragments", "tools", - "tool_responses", "tool_calls", "tool_results", "tool_instances" + "tool_responses", "tool_calls", "tool_results", "tool_instances", + "tool_results_attachments", + "messages", "parts", "part_attachments", "part_fragments", + "threads", "turns", "turn_tools", "turn_fragments", + "turn_search", "turn_search_fts", "tool_instantiations", ): schema = db[table].schema cog.out(format(cleanup_sql(schema))) @@ -308,117 +711,214 @@ for table in ( cog.out("```\n") ]]] --> ```sql -CREATE TABLE [conversations] ( - [id] TEXT PRIMARY KEY, - [name] TEXT, - [model] TEXT +CREATE TABLE "conversations" ( + "id" TEXT PRIMARY KEY, + "name" TEXT, + "model" TEXT ); -CREATE TABLE [schemas] ( - [id] TEXT PRIMARY KEY, - [content] TEXT +CREATE TABLE "schemas" ( + "id" TEXT PRIMARY KEY, + "content" TEXT ); CREATE TABLE "responses" ( - [id] TEXT PRIMARY KEY, - [model] TEXT, - [prompt] TEXT, - [system] TEXT, - [prompt_json] TEXT, - [options_json] TEXT, - [response] TEXT, - [response_json] TEXT, - [conversation_id] TEXT REFERENCES [conversations]([id]), - [duration_ms] INTEGER, - [datetime_utc] TEXT, - [input_tokens] INTEGER, - [output_tokens] INTEGER, - [token_details] TEXT, - [schema_id] TEXT REFERENCES [schemas]([id]), - [resolved_model] TEXT + "id" TEXT PRIMARY KEY, + "model" TEXT, + "prompt" TEXT, + "system" TEXT, + "prompt_json" TEXT, + "options_json" TEXT, + "response" TEXT, + "response_json" TEXT, + "conversation_id" TEXT REFERENCES "conversations"("id"), + "duration_ms" INTEGER, + "datetime_utc" TEXT, + "input_tokens" INTEGER, + "output_tokens" INTEGER, + "token_details" TEXT, + "schema_id" TEXT REFERENCES "schemas"("id"), + "resolved_model" TEXT, + "reasoning" TEXT ); -CREATE VIRTUAL TABLE [responses_fts] USING FTS5 ( - [prompt], - [response], - content=[responses] +CREATE VIRTUAL TABLE "responses_fts" USING FTS5 ( + "prompt", + "response", + content="responses" ); -CREATE TABLE [attachments] ( - [id] TEXT PRIMARY KEY, - [type] TEXT, - [path] TEXT, - [url] TEXT, - [content] BLOB +CREATE TABLE "attachments" ( + "id" TEXT PRIMARY KEY, + "type" TEXT, + "path" TEXT, + "url" TEXT, + "content" BLOB ); -CREATE TABLE [prompt_attachments] ( - [response_id] TEXT REFERENCES [responses]([id]), - [attachment_id] TEXT REFERENCES [attachments]([id]), - [order] INTEGER, - PRIMARY KEY ([response_id], - [attachment_id]) +CREATE TABLE "prompt_attachments" ( + "response_id" TEXT REFERENCES "responses"("id"), + "attachment_id" TEXT REFERENCES "attachments"("id"), + "order" INTEGER, + PRIMARY KEY ("response_id", + "attachment_id") ); -CREATE TABLE [fragments] ( - [id] INTEGER PRIMARY KEY, - [hash] TEXT, - [content] TEXT, - [datetime_utc] TEXT, - [source] TEXT +CREATE TABLE "fragments" ( + "id" INTEGER PRIMARY KEY, + "hash" TEXT, + "content" TEXT, + "datetime_utc" TEXT, + "source" TEXT ); -CREATE TABLE [fragment_aliases] ( - [alias] TEXT PRIMARY KEY, - [fragment_id] INTEGER REFERENCES [fragments]([id]) +CREATE TABLE "fragment_aliases" ( + "alias" TEXT PRIMARY KEY, + "fragment_id" INTEGER REFERENCES "fragments"("id") ); CREATE TABLE "prompt_fragments" ( - [response_id] TEXT REFERENCES [responses]([id]), - [fragment_id] INTEGER REFERENCES [fragments]([id]), - [order] INTEGER, - PRIMARY KEY ([response_id], - [fragment_id], - [order]) + "response_id" TEXT REFERENCES "responses"("id"), + "fragment_id" INTEGER REFERENCES "fragments"("id"), + "order" INTEGER, + PRIMARY KEY ("response_id", + "fragment_id", + "order") ); CREATE TABLE "system_fragments" ( - [response_id] TEXT REFERENCES [responses]([id]), - [fragment_id] INTEGER REFERENCES [fragments]([id]), - [order] INTEGER, - PRIMARY KEY ([response_id], - [fragment_id], - [order]) + "response_id" TEXT REFERENCES "responses"("id"), + "fragment_id" INTEGER REFERENCES "fragments"("id"), + "order" INTEGER, + PRIMARY KEY ("response_id", + "fragment_id", + "order") ); -CREATE TABLE [tools] ( - [id] INTEGER PRIMARY KEY, - [hash] TEXT, - [name] TEXT, - [description] TEXT, - [input_schema] TEXT, - [plugin] TEXT +CREATE TABLE "tools" ( + "id" INTEGER PRIMARY KEY, + "hash" TEXT, + "name" TEXT, + "description" TEXT, + "input_schema" TEXT, + "plugin" TEXT ); -CREATE TABLE [tool_responses] ( - [tool_id] INTEGER REFERENCES [tools]([id]), - [response_id] TEXT REFERENCES [responses]([id]), - PRIMARY KEY ([tool_id], - [response_id]) +CREATE TABLE "tool_responses" ( + "tool_id" INTEGER REFERENCES "tools"("id"), + "response_id" TEXT REFERENCES "responses"("id"), + PRIMARY KEY ("tool_id", + "response_id") ); -CREATE TABLE [tool_calls] ( - [id] INTEGER PRIMARY KEY, - [response_id] TEXT REFERENCES [responses]([id]), - [tool_id] INTEGER REFERENCES [tools]([id]), - [name] TEXT, - [arguments] TEXT, - [tool_call_id] TEXT +CREATE TABLE "tool_calls" ( + "id" INTEGER PRIMARY KEY, + "response_id" TEXT REFERENCES "responses"("id"), + "tool_id" INTEGER REFERENCES "tools"("id"), + "name" TEXT, + "arguments" TEXT, + "tool_call_id" TEXT ); CREATE TABLE "tool_results" ( - [id] INTEGER PRIMARY KEY, - [response_id] TEXT REFERENCES [responses]([id]), - [tool_id] INTEGER REFERENCES [tools]([id]), - [name] TEXT, - [output] TEXT, - [tool_call_id] TEXT, - [instance_id] INTEGER REFERENCES [tool_instances]([id]), - [exception] TEXT + "id" INTEGER PRIMARY KEY, + "response_id" TEXT REFERENCES "responses"("id"), + "tool_id" INTEGER REFERENCES "tools"("id"), + "name" TEXT, + "output" TEXT, + "tool_call_id" TEXT, + "instance_id" INTEGER REFERENCES "tool_instances"("id"), + "exception" TEXT +); +CREATE TABLE "tool_instances" ( + "id" INTEGER PRIMARY KEY, + "plugin" TEXT, + "name" TEXT, + "arguments" TEXT +); +CREATE TABLE "tool_results_attachments" ( + "tool_result_id" INTEGER REFERENCES "tool_results"("id"), + "attachment_id" TEXT REFERENCES "attachments"("id"), + "order" INTEGER, + PRIMARY KEY ("tool_result_id", + "attachment_id") +); +CREATE TABLE "messages" ( + "hash" TEXT PRIMARY KEY, + "parent_hash" TEXT REFERENCES "messages"("hash"), + "role" TEXT, + "provider_metadata" TEXT +); +CREATE TABLE "parts" ( + "id" INTEGER PRIMARY KEY, + "message_hash" TEXT REFERENCES "messages"("hash"), + "position" INTEGER, + "type" TEXT, + "tool_name" TEXT, + "text" TEXT, + "payload" TEXT +); +CREATE TABLE "part_attachments" ( + "part_id" INTEGER REFERENCES "parts"("id"), + "attachment_id" TEXT REFERENCES "attachments"("id"), + "order" INTEGER, + PRIMARY KEY ("part_id", + "attachment_id", + "order") +); +CREATE TABLE "part_fragments" ( + "part_id" INTEGER REFERENCES "parts"("id"), + "fragment_id" INTEGER REFERENCES "fragments"("id"), + "order" INTEGER, + PRIMARY KEY ("part_id", + "fragment_id", + "order") +); +CREATE TABLE "threads" ( + "id" TEXT PRIMARY KEY, + "name" TEXT, + "tip_message_hash" TEXT REFERENCES "messages"("hash"), + "forked_from" TEXT REFERENCES "threads"("id"), + "datetime_utc" TEXT +); +CREATE TABLE "turns" ( + "id" TEXT PRIMARY KEY, + "thread_id" TEXT REFERENCES "threads"("id"), + "parent_message_hash" TEXT REFERENCES "messages"("hash"), + "tip_message_hash" TEXT REFERENCES "messages"("hash"), + "model" TEXT, + "resolved_model" TEXT, + "options_json" TEXT, + "schema_id" TEXT REFERENCES "schemas"("id"), + "input_tokens" INTEGER, + "output_tokens" INTEGER, + "token_details" TEXT, + "duration_ms" INTEGER, + "datetime_utc" TEXT, + "response_json" TEXT +); +CREATE TABLE "turn_tools" ( + "turn_id" TEXT REFERENCES "turns"("id"), + "tool_id" INTEGER REFERENCES "tools"("id"), + "instance_id" INTEGER REFERENCES "tool_instances"("id"), + PRIMARY KEY ("turn_id", + "tool_id") +); +CREATE TABLE "turn_fragments" ( + "turn_id" TEXT REFERENCES "turns"("id"), + "fragment_id" INTEGER REFERENCES "fragments"("id"), + "order" INTEGER, + "kind" TEXT, + PRIMARY KEY ("turn_id", + "fragment_id", + "kind", + "order") +); +CREATE TABLE "turn_search" ( + "id" INTEGER PRIMARY KEY, + "turn_id" TEXT REFERENCES "turns"("id"), + "prompt" TEXT, + "response" TEXT +); +CREATE VIRTUAL TABLE "turn_search_fts" USING FTS5 ( + "prompt", + "response", + content="turn_search" ); -CREATE TABLE [tool_instances] ( - [id] INTEGER PRIMARY KEY, - [plugin] TEXT, - [name] TEXT, - [arguments] TEXT +CREATE TABLE "tool_instantiations" ( + "turn_id" TEXT REFERENCES "turns"("id"), + "tool_call_id" TEXT, + "instance_id" INTEGER REFERENCES "tool_instances"("id"), + PRIMARY KEY ("turn_id", + "tool_call_id") ); ``` -`responses_fts` configures [SQLite full-text search](https://www.sqlite.org/fts5.html) against the `prompt` and `response` columns in the `responses` table. +`responses_fts` configures [SQLite full-text search](https://www.sqlite.org/fts5.html) against the `prompt` and `response` columns in the `responses` table. `turn_search_fts` does the same for the `turn_search` table, which holds the searchable text of each turn in the content-addressed tables - together these are what {ref}`llm logs -q ` queries. diff --git a/docs/openai-models.md b/docs/openai-models.md index ce064c291..13dfa19cd 100644 --- a/docs/openai-models.md +++ b/docs/openai-models.md @@ -32,52 +32,47 @@ cog.out("```\n{}\n```".format("\n".join(models))) ]]] --> ``` OpenAI Chat: gpt-4o (aliases: 4o) -OpenAI Chat: chatgpt-4o-latest (aliases: chatgpt-4o) OpenAI Chat: gpt-4o-mini (aliases: 4o-mini) -OpenAI Chat: gpt-4o-audio-preview -OpenAI Chat: gpt-4o-audio-preview-2024-12-17 -OpenAI Chat: gpt-4o-audio-preview-2024-10-01 -OpenAI Chat: gpt-4o-mini-audio-preview -OpenAI Chat: gpt-4o-mini-audio-preview-2024-12-17 OpenAI Chat: gpt-4.1 (aliases: 4.1) OpenAI Chat: gpt-4.1-mini (aliases: 4.1-mini) OpenAI Chat: gpt-4.1-nano (aliases: 4.1-nano) OpenAI Chat: gpt-3.5-turbo (aliases: 3.5, chatgpt) OpenAI Chat: gpt-3.5-turbo-16k (aliases: chatgpt-16k, 3.5-16k) OpenAI Chat: gpt-4 (aliases: 4, gpt4) -OpenAI Chat: gpt-4-32k (aliases: 4-32k) -OpenAI Chat: gpt-4-1106-preview -OpenAI Chat: gpt-4-0125-preview OpenAI Chat: gpt-4-turbo-2024-04-09 OpenAI Chat: gpt-4-turbo (aliases: gpt-4-turbo-preview, 4-turbo, 4t) -OpenAI Chat: gpt-4.5-preview-2025-02-27 -OpenAI Chat: gpt-4.5-preview (aliases: gpt-4.5) -OpenAI Chat: o1 -OpenAI Chat: o1-2024-12-17 -OpenAI Chat: o1-preview -OpenAI Chat: o1-mini -OpenAI Chat: o3-mini -OpenAI Chat: o3 -OpenAI Chat: o4-mini -OpenAI Chat: gpt-5 -OpenAI Chat: gpt-5-mini -OpenAI Chat: gpt-5-nano -OpenAI Chat: gpt-5-2025-08-07 -OpenAI Chat: gpt-5-mini-2025-08-07 -OpenAI Chat: gpt-5-nano-2025-08-07 -OpenAI Chat: gpt-5.1 -OpenAI Chat: gpt-5.1-chat-latest -OpenAI Chat: gpt-5.2 -OpenAI Chat: gpt-5.2-chat-latest +OpenAI Responses: o1 +OpenAI Responses: o1-2024-12-17 +OpenAI Responses: o3-mini +OpenAI Responses: o3 +OpenAI Responses: o4-mini +OpenAI Responses: gpt-5 +OpenAI Responses: gpt-5-mini +OpenAI Responses: gpt-5-nano +OpenAI Responses: gpt-5-2025-08-07 +OpenAI Responses: gpt-5-mini-2025-08-07 +OpenAI Responses: gpt-5-nano-2025-08-07 +OpenAI Responses: gpt-5.1 +OpenAI Responses: gpt-5.2 +OpenAI Responses: gpt-5.2-chat-latest +OpenAI Responses: gpt-5.4 +OpenAI Responses: gpt-5.4-2026-03-05 +OpenAI Responses: gpt-5.4-mini +OpenAI Responses: gpt-5.4-mini-2026-03-17 +OpenAI Responses: gpt-5.4-nano +OpenAI Responses: gpt-5.4-nano-2026-03-17 +OpenAI Responses: gpt-5.5 +OpenAI Responses: gpt-5.5-2026-04-23 +OpenAI Responses: gpt-5.6-sol +OpenAI Responses: gpt-5.6-terra +OpenAI Responses: gpt-5.6-luna OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instruct) ``` See [the OpenAI models documentation](https://platform.openai.com/docs/models) for details of each of these. -`gpt-4o-mini` (aliased to `4o-mini`) is the least expensive model, and is the default for if you don't specify a model at all. Consult [OpenAI's model documentation](https://platform.openai.com/docs/models) for details of the other models. - -[o1-pro](https://platform.openai.com/docs/models/o1-pro) is not available through the Chat Completions API used by LLM's default OpenAI plugin. You can install the new [llm-openai-plugin](https://github.com/simonw/llm-openai-plugin) plugin to access that model. +`gpt-5.6-luna` is one of the less expensive models, and is the default for if you don't specify a model at all. Consult [OpenAI's model documentation](https://platform.openai.com/docs/models) for details of the other models. ## Model features @@ -88,6 +83,119 @@ The following features work with OpenAI models: - {ref}`Schemas ` can be used to influence the JSON structure of the model output. - {ref}`Model options ` can be used to set parameters like `temperature`. Use `llm models --options` for a full list of supported options. +(openai-models-web-search)= + +## Web Search + +Models that use the OpenAI Responses API can search the web using the `WebSearch` server-side tool: + +```bash +llm -m gpt-5.6-luna -T WebSearch 'Search the web for a positive news story from today' +``` + +The model decides whether to search based on the prompt. The Python API accepts the same tool: + +```python +import llm +from llm.default_plugins.openai_models import WebSearch + +response = llm.get_model("gpt-5.6-luna").prompt( + "Search the web for a positive news story from today", + tools=[WebSearch(include_sources=True)], +) +print(response.text()) +``` + +Domain filters accept up to 100 allowed domains and up to 100 blocked domains. Omit `http://` or `https://`; each entry also covers its subdomains: + +```python +WebSearch( + allowed_domains=["openai.com", "python.org"], + blocked_domains=["example.com"], +) +``` + +Use `search_context_size="low"`, `"medium"` or `"high"` to control how much search-result context is made available to the model. Set `external_web_access=False` for cached/indexed results only. `return_token_budget="unlimited"` removes the standard returned-token limit for longer GPT-5 reasoning searches and can increase latency and cost. + +Approximate geographic context can include a two-letter country code, city, region and IANA timezone. LLM adds the required `"type": "approximate"` field when it is omitted: + +```python +WebSearch( + user_location={ + "country": "GB", + "city": "London", + "timezone": "Europe/London", + } +) +``` + +`include_sources=True` requests every URL consulted during search and records them in the server-executed tool call arguments. For image search, use `search_content_types=["image", "text"]` and `image_settings={"max_results": 3, "caption": True}`. Set `include_results=True` to request and record the raw image result objects in the corresponding server-executed tool result. + +See [OpenAI's Web Search documentation](https://developers.openai.com/api/docs/guides/tools-web-search) for model behavior, current limitations, citations and pricing details. + +(openai-models-code-interpreter)= + +## Code Interpreter + +Models that use the OpenAI Responses API can run Python in an OpenAI-managed container using the `CodeInterpreter` server-side tool: + +```bash +llm -m gpt-5.6-luna -T 'CodeInterpreter(memory_limit="4g")' 'Run this calculation' +``` + +The same tool can be used from Python: + +```python +import llm +from llm.default_plugins.openai_models import CodeInterpreter + +model = llm.get_model("gpt-5.6-luna") +response = model.prompt( + "Use the python tool to calculate 111111 * 333333", + tools=[CodeInterpreter()], +) +print(response.text()) +``` + +OpenAI calls this Code Interpreter, but models know it as the "python tool", so referring to that name in the prompt is the most explicit way to request it. + +By default OpenAI automatically creates a 1 GB container, or reuses an active container from the model's context. Configure a larger automatic container or make existing OpenAI files available to it like this: + +```python +CodeInterpreter( + memory_limit="4g", + file_ids=["file-1", "file-2"], +) +``` + +The accepted memory limits are `1g`, `4g`, `16g` and `64g`. Higher limits cost more. To reuse a container that was created separately, pass its ID: + +```python +CodeInterpreter(container="cntr_abc123") +``` + +An explicit container ID cannot be combined with `memory_limit` or `file_ids`. LLM automatically requests the full Code Interpreter output and records returned code and output as server-executed tool parts; it never tries to run that code locally. + +OpenAI containers are ephemeral and expire after 20 minutes without activity. See [OpenAI's Code Interpreter documentation](https://developers.openai.com/api/docs/guides/tools-code-interpreter) for container behavior, supported files and current pricing details. + +(openai-models-service-tier)= + +## Fast mode and service tiers + +OpenAI models can process requests at different speeds and prices using the `service_tier` API parameter. [Fast mode](https://developers.openai.com/api/docs/guides/fast-mode) runs up to 2.5x faster than standard processing at a higher per-token price, with the biggest speed increase on `gpt-5.6-sol`. [Flex processing](https://developers.openai.com/api/docs/guides/flex-processing) is slower but cheaper. Each tier works with a different subset of models - the Fast and Flex tables on [OpenAI's pricing page](https://developers.openai.com/api/docs/pricing) are the definitive list of which models support which tier. + +All of the OpenAI models supported by LLM expose a `service_tier` option, with the exception of the legacy `gpt-3.5-turbo-instruct` completion model. Use `-o service_tier fast` to enable Fast mode for a prompt: + +```bash +llm -m gpt-5.6-sol -o service_tier fast 'Fast facts about pelicans' +``` + +The value is passed straight to the API, so other tiers such as `priority` (the older name for `fast`) and `flex` ([slower but cheaper processing](https://developers.openai.com/api/docs/guides/flex-processing)) work too: + +```bash +llm -m gpt-5.4 -o service_tier flex 'No rush: facts about pelicans' +``` + (openai-models-embedding)= ## OpenAI embedding models @@ -126,8 +234,6 @@ The vector size of the supported OpenAI embedding models are as follows: The `gpt-3.5-turbo-instruct` model is a little different - it is a completion model rather than a chat model, described in [the OpenAI completions documentation](https://platform.openai.com/docs/api-reference/completions/create). -Completion models can be called with the `-o logprobs 3` option (not supported by chat models) which will cause LLM to store 3 log probabilities for each returned token in the SQLite database. Consult [this issue](https://github.com/simonw/llm/issues/284#issuecomment-1724772704) for details on how to read these values. - (openai-extra-models)= ## Adding more OpenAI models @@ -156,10 +262,14 @@ The `model_id` is the identifier that will be recorded in the LLM logs. You can If the model is a completion model (such as `gpt-3.5-turbo-instruct`) add `completion: true` to the configuration. +If the model should use the OpenAI Responses API rather than Chat Completions, add `responses: true` to the configuration. This is useful for models such as `o1`, `o3-mini` and `gpt-5`-style models that are accessed through `/v1/responses`. + If the model supports structured extraction using json_schema, add `supports_schema: true` to the configuration. For reasoning models like `o1` or `o3-mini` add `reasoning: true`. +If the model supports the `service_tier` parameter - see {ref}`openai-models-service-tier` - add `service_tier: true` to enable the corresponding option. + With this configuration in place, the following command should run a prompt against the new model: ```bash @@ -174,7 +284,6 @@ Example output: OpenAI Chat: gpt-3.5-turbo (aliases: 3.5, chatgpt) OpenAI Chat: gpt-3.5-turbo-16k (aliases: chatgpt-16k, 3.5-16k) OpenAI Chat: gpt-4 (aliases: 4, gpt4) -OpenAI Chat: gpt-4-32k (aliases: 4-32k) OpenAI Chat: gpt-3.5-turbo-0613 (aliases: 0613) ``` Running `llm logs -n 1` should confirm that the prompt and response has been correctly logged to the database. diff --git a/docs/other-models.md b/docs/other-models.md index 69ed41595..662d7af4e 100644 --- a/docs/other-models.md +++ b/docs/other-models.md @@ -28,6 +28,135 @@ Check the {ref}`plugin directory ` for the latest list of avai Projects such as [LocalAI](https://localai.io/) offer a REST API that imitates the OpenAI API but can be used to run other models, including models that can be installed on your own machine. These can be added using the same configuration mechanism. +(openai-endpoint)= +### Run against an endpoint without configuring it + +Use `llm openai endpoint` to run a prompt directly against an OpenAI-compatible base URL: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + "What is the capital of France?" +``` + +This command does not register the model and does not log the prompt or response. It also does not send your configured OpenAI API key to the endpoint. Use `--key` to explicitly provide a key or the alias of a key saved using `llm keys set`: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + --key custom-endpoint \ + "What is the capital of France?" +``` + +List the model IDs advertised by the endpoint using `--models`. This requests the `models` resource relative to the base URL, so a base URL ending in `/v1` will request `/v1/models`: + +```bash +llm openai endpoint https://example.com/v1 --models +``` + +Omit the prompt to read it from stdin. In an interactive terminal the command waits for input until EOF, matching `llm prompt`: + +```bash +llm openai endpoint https://example.com/v1 -m model-id +``` + +Use `--chat` to start an interactive chat: + +```bash +llm openai endpoint https://example.com/v1 -m model-id --chat +``` + +Use `-a` or `--attachment` to attach an image or PDF. Chat Completions endpoints can also receive WAV or MP3 audio attachments: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + -a image.jpg \ + "Describe this image" +``` + +Use `--at path-or-url mimetype` when the attachment type cannot be inferred. Attachments provided when starting an interactive chat are included with the first message. + +Use `-t` or `--template` to apply an existing LLM template. Template prompts, system prompts, defaults, model options, model IDs, schemas, and attachments are supported. Pass template variables using `-p` or `--param`: + +```bash +llm openai endpoint https://example.com/v1 \ + -t summarize \ + -p style concise \ + "Text to summarize" +``` + +The `-m` option can be omitted if the template specifies a model. In an interactive chat started using `--chat`, the template is applied to each turn. Without `--chat`, a template that provides its own prompt runs once even when no prompt argument is supplied. + +Use `--schema` to request structured JSON output or `--schema-multi` to request an array of matching items. These accept the same inline JSON, file paths, stored schema IDs, template references and {ref}`concise schema syntax ` as `llm prompt`: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + --schema 'name, age int' \ + "Invent a dog" +``` + +Reasoning-capable endpoints can be given a reasoning effort using `-o reasoning_effort`, with a value of `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, or `max`: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + -o reasoning_effort high \ + "Solve this problem" +``` + +Responses API endpoints can also be asked for a visible reasoning summary using `-o reasoning_summary`, with a value of `auto`, `concise`, or `detailed`: + +```bash +llm openai endpoint https://example.com/v1 \ + --responses \ + -m model-id \ + -o reasoning_summary auto \ + "Solve this problem" +``` + +This maps to the Responses API `reasoning.summary` request field. `auto` asks the endpoint for the most detailed summary available for that model. Use `-R` or `--hide-reasoning` to suppress summary generation for a request. + +The command does not send reasoning-specific request fields by default and does not request a reasoning summary unless one of these options is used. An endpoint that does not support an option will return its own API error. + +Use `-T` or `--tool` to make an installed LLM tool available to the model, or `--functions` to load Python functions from an inline code block or file: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + -T llm_time \ + --functions tools.py \ + "What time is it?" +``` + +Tool calls are executed locally and their results are sent back to the endpoint until it returns a final answer. `--chain-limit` controls the maximum number of responses, `--tools-debug` shows tool execution details, and `--tools-approve` asks for confirmation before each call. Tools and trusted Python functions declared by local templates are supported too. + +The command uses the Chat Completions API by default. Add `--responses` for an endpoint that implements the Responses API: + +```bash +llm openai endpoint https://example.com/v1 \ + -m model-id \ + --responses \ + "What is the capital of France?" +``` + +Responses endpoints can define their own server-side tool types. Use `ServerSideTool` to pass an endpoint-specific tool specification through without validation. For example, [OpenRouter's web search server tool](https://openrouter.ai/docs/api/reference/responses/web-search) can be used like this: + +```bash +llm openai endpoint https://openrouter.ai/api/v1 \ + -m openai/gpt-oss-20b:free \ + --key openrouter \ + --responses \ + -R \ + -T 'ServerSideTool(spec={"type":"openrouter:web_search","parameters":{"engine":"exa","max_results":2,"max_uses":1}})' \ + "Search for the OpenRouter documentation URL" +``` + +`-R` hides the reasoning text returned by this model so the command displays just its final answer. + +### Configure an OpenAI-compatible model + The `model_id` is the name LLM will use for the model. The `model_name` is the name which needs to be passed to the API - this might differ from the `model_id`, especially if the `model_id` could potentially clash with other installed models. The `api_base` key can be used to point the OpenAI client library at a different API endpoint. @@ -46,6 +175,7 @@ You can set `api_key_name` to the name of a key stored using the {ref}`api-keys` Other keys you can use here: - `completion: true` for completion models that should use the `/completion` endpoint as opposed to `/completion/chat` +- `responses: true` for models that should use the OpenAI Responses API (`/responses`) instead of the Chat Completions API (`/chat/completions`) - `supports_tools: true` for models that support tool calling - `can_stream: false` to disable streaming mode for models that cannot stream - `supports_schema: true` for models that support JSON structured schema output diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index f7b362557..9c4b5a83e 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -58,7 +58,8 @@ The async version of a model subclasses `llm.AsyncModel` instead of `llm.Model`. This example shows a subset of the OpenAI default plugin illustrating how this method might work: ```python -from typing import AsyncGenerator +from collections.abc import AsyncGenerator + import llm class MyAsyncModel(llm.AsyncModel): @@ -105,6 +106,14 @@ def register_models(register): ) ``` +The `prompt` object passed to your `execute()` method is an instance of {class}`~llm.Prompt`: + +```{eval-rst} +.. autoclass:: llm.Prompt + :members: prompt, system + :exclude-members: model, options +``` + (advanced-model-plugins-schemas)= ## Supporting schemas @@ -130,7 +139,7 @@ Adding {ref}`tools support ` involves several steps: 1. Add `supports_tools = True` to your model class. 2. If `prompt.tools` is populated, turn that list of `llm.Tool` objects into the correct format for your model. -3. Look out for requests to call tools in the responses from your model. Call `response.add_tool_call(llm.ToolCall(...))` for each of those. This should work for streaming and non-streaming and async and non-async cases. +3. Look out for requests to call tools in the responses from your model. Call `response.add_tool_call(llm.ToolCall(...))` for each of those. This should work for streaming and non-streaming and async and non-async cases. Pass the provider's tool call ID as `tool_call_id=` if there is one; if you omit it LLM synthesizes a unique `tc_`-prefixed id, since consumers rely on every tool call having one. 4. If your prompt has a `prompt.tool_results` list, pass the information from those `llm.ToolResult` objects to your model. 5. Include `prompt.tools` and `prompt.tool_results` and tool calls from `response.tool_calls_or_raise()` in the conversation history constructed by your plugin. 6. Make sure your code is OK with prompts that do not have `prompt.prompt` set to a value, since they may be carrying exclusively the results of a tool call. @@ -147,12 +156,66 @@ Here are the relevant dataclasses: .. autoclass:: llm.ToolResult ``` +(advanced-model-plugins-server-side-tools)= + +## Supporting server-side tools + +Server-side tools execute inside the provider's infrastructure rather than in LLM's local tool loop. Provider plugins represent these by subclassing {class}`llm.ServerSideTool`: + +```python +class ProviderSearch(llm.ServerSideTool): + "A search tool executed by Example Provider." + + name = "provider_search" + + def __init__(self, allowed_domains=None): + self.allowed_domains = allowed_domains + + def tool_spec(self, model): + spec = {"type": "provider_search"} + if self.allowed_domains: + spec["allowed_domains"] = self.allowed_domains + return spec + + def prepare_request(self, model, kwargs): + # Add any other request fields this tool requires. Merge with existing values instead of replacing them. + include = kwargs.setdefault("include", []) + if "provider_search.results" not in include: + include.append("provider_search.results") +``` + +Model instances expose the server-side tool classes they support using the `supported_server_side_tools` property: + +```python +class MyModel(llm.KeyModel): + @property + def supported_server_side_tools(self): + return (ProviderSearch,) +``` + +This property is evaluated on the model instance, so it can use `self.model_id` or other instance configuration to vary the available tools. It is independent of `supports_tools`: that flag describes locally executed function tools. A model can support server-side tools, function tools, both or neither. + +`prompt.tools` can contain both {class}`llm.Tool` and {class}`llm.ServerSideTool` instances. The adapter should partition that list, put each server-side tool's `tool_spec(model)` result wherever its provider expects it and, after the baseline request is complete, call `tool.prepare_request(model, kwargs)` once for each server-side tool in list order. LLM validates the model declaration before `execute()` runs, but it does not interpret the specification or call either method itself. + +Providers with an OpenAI-compatible tools array can optionally support raw specifications by including `llm.ServerSideTool` in this property. This claims direct instances such as: + +```python +llm.ServerSideTool({"type": "browser_search"}) +``` + +Server-side tool calls returned by the provider should use the `server_executed=True` events described in {ref}`structured-messages-streaming`. + +```{eval-rst} +.. autoclass:: llm.ServerSideTool + :members: tool_spec, prepare_request +``` + (advanced-model-plugins-attachments)= ## Attachments for multi-modal models -Models such as GPT-4o, Claude 3.5 Sonnet and Google's Gemini 1.5 are multi-modal: they accept input in the form of images and maybe even audio, video and other formats. +Models such as GPT-4o, Claude 3.5 Sonnet and Google's `gemini-flash-latest` are multi-modal: they accept input in the form of images and maybe even audio, video and other formats. LLM calls these **attachments**. Models can specify the types of attachments they accept and then implement special code in the `.execute()` method to handle them. @@ -160,17 +223,19 @@ See {ref}`the Python attachments documentation ` for det ### Specifying attachment types -A `Model` subclass can list the types of attachments it accepts by defining a `attachment_types` class attribute: +A `Model` subclass can list the types of attachments it accepts by defining an `attachment_types` class attribute: ```python class NewModel(llm.Model): model_id = "new-model" - attachment_types = { - "image/png", - "image/jpeg", - "image/webp", - "image/gif", - } + attachment_types = frozenset( + { + "image/png", + "image/jpeg", + "image/webp", + "image/gif", + } + ) ``` These content types are detected when an attachment is passed to LLM using `llm -a filename`, or can be specified by the user using the `--attachment-type filename image/png` option. @@ -238,30 +303,391 @@ As you can see, it uses `attachment.url` if that is available and otherwise fall ### Attachments from previous conversations -Models that implement the ability to continue a conversation can reconstruct the previous message JSON using the `response.attachments` attribute. +Conversation history — including attachments from prior turns — is available on the canonical `prompt.messages` list. See the [next section](#structured-messages-streaming) for how that works. + +(structured-messages-streaming)= + +## Structured messages and streaming events + +The 0.32 alpha introduced a richer contract for plugins than "yield strings": + +1. **`execute()` yields `StreamEvent` objects** (or plain `str`, still supported) so text, reasoning (thinking tokens), tool calls, and server-side tool results each surface as their own event type. The framework assembles these into typed `Part` objects. +2. **`build_messages` (or equivalent) reads `prompt.messages`** — a `list[llm.Message]` that is the complete input chain for this turn. +3. **Opaque provider tokens round-trip via `provider_metadata`** — Anthropic thinking signatures, Gemini thought signatures, OpenAI Responses API encrypted reasoning blobs. Plugins stash whatever the API returns, then echo it back on the next request. + +**Older plugins still work.** A plugin that still yields plain `str` from `execute()` works unchanged — each string is wrapped as a `StreamEvent(type="text", chunk=...)` internally. + +### Yielding StreamEvent from execute() + +```python +from llm.parts import StreamEvent + +def execute(self, prompt, stream, response, conversation, key=None): + messages = self.build_messages(prompt, conversation) + ... + + for chunk in provider_sdk.stream(...): + if chunk.type == "text": + yield StreamEvent(type="text", chunk=chunk.text) + elif chunk.type == "thinking": + yield StreamEvent(type="reasoning", chunk=chunk.text) +``` + +A `StreamEvent` has four frequently-used fields: + +- **`type`** — one of `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, `"tool_result"`. +- **`chunk`** — the text fragment. For tool calls this is the tool name (for `tool_call_name`) or a partial JSON string (for `tool_call_args`). +- **`tool_call_id`** — the provider's id for the tool call, set on `tool_call_name` / `tool_call_args` / `tool_result` events. Also the signal the framework uses to group tool-call events into one `ToolCallPart`. +- **`provider_metadata`** — an optional `dict[str, dict]` namespaced by provider name. Carries opaque data (signatures, encrypted blobs) that must be echoed back on future requests. + +Three additional fields exist for special cases: + +- **`server_executed: bool`** — set `True` for server-side tool calls (for example, Anthropic web search) and their results. This means the model ran the tool internally as part of responding to the prompt. +- **`tool_name`** — set on `tool_result` events to identify which tool this result came from. +- **`part_index: int | None`** — defaults to `None`, which means "let the framework decide which Part this event belongs to." Pass an explicit integer only when you need to override the default grouping (see [below](#part-index-overrides)). + +### How events group into Parts + +When you leave `part_index` as `None` (the default), the framework groups events using these rules: + +- **Consecutive same-family events concatenate.** Two `text` events in a row become one `TextPart`. Two `reasoning` events in a row become one `ReasoningPart`. A family transition (text → reasoning, or reasoning → text) starts a new Part. +- **Tool calls group by `tool_call_id`.** A `tool_call_name` and any number of `tool_call_args` events sharing a `tool_call_id` combine into one `ToolCallPart` — even if they're interleaved with other events (parallel tool calls). +- **`tool_result` is always its own Part**, paired to the originating call by `tool_call_id`. + +| Stream | Resulting Parts | +|-------------------------------------------|----------------------------------------------------------| +| `text` × N | one `TextPart` | +| `reasoning` × N, then `text` × N | `ReasoningPart`, `TextPart` | +| `text`, `tool_call_name`+`args`, `text` | `TextPart`, `ToolCallPart`, `TextPart` | +| Parallel tool calls (interleaved by id) | one `ToolCallPart` per distinct `tool_call_id` | +| `reasoning`, tool call, `reasoning` | `ReasoningPart`, `ToolCallPart`, `ReasoningPart` | + +(part-index-overrides)= +### Setting `part_index` explicitly + +In rare cases you'll want to override the default grouping: + +- **Forcing a single TextPart across non-adjacent text bursts.** If your provider interleaves text deltas with tool calls but you want all the text concatenated into one `TextPart`, pass `part_index=0` on every text event. (The default behavior produces separate `TextPart`s on each side of the tool calls — usually what you want, but not always.) +- **Tool-call args arriving before the id.** If your provider streams args before the `tool_call_id` is known, assign your own index per logical tool call and pass it on each event of that call. + +You can mix explicit indices with `None` in the same stream — the framework reserves your explicit values and decides the rest. + +(advanced-model-plugins-reasoning-tokens)= +### Reasoning tokens + +For streamed reasoning text: + +```python +yield StreamEvent(type="reasoning", chunk=text_chunk) +``` + +Reasoning events that appear before/after text events become distinct `ReasoningPart` and `TextPart` entries in `response.messages` automatically. If your provider emits two thinking blocks separated by a tool call, you'll get two `ReasoningPart`s. + +Plugins should respect `prompt.hide_reasoning`. This is set when the caller passes `hide_reasoning=True` to `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, or their async counterparts. It is also set by the CLI `-R/--hide-reasoning` option. + +`prompt.hide_reasoning` means "hide visible reasoning output", not "disable model reasoning". If your provider requires an explicit request for visible reasoning summaries, do not request those summaries when `prompt.hide_reasoning` is true: + +```python +kwargs = {} +if not prompt.hide_reasoning: + kwargs["reasoning"] = {"summary": "auto"} +``` + +If your provider emits reasoning blocks regardless of request parameters, keep yielding those reasoning events as usual: + +```python +if chunk.type == "thinking": + yield StreamEvent(type="reasoning", chunk=chunk.text) +``` + +LLM's display layers use `prompt.hide_reasoning` to avoid showing those events to the user, while still allowing the framework to persist `ReasoningPart` objects and provider metadata for logs, serialization, and future turns. + +### Tool calls -Here's how the OpenAI plugin does that: +Each tool call emits two event types sharing a `tool_call_id`: ```python -for prev_response in conversation.responses: - if prev_response.attachments: - attachment_message = [] - if prev_response.prompt.prompt: - attachment_message.append( - {"type": "text", "text": prev_response.prompt.prompt} +yield StreamEvent( + type="tool_call_name", + chunk=tool_name, + tool_call_id=tool_call_id, +) +# then, as the provider streams JSON args: +yield StreamEvent( + type="tool_call_args", + chunk=partial_json_fragment, + tool_call_id=tool_call_id, +) +``` + +The framework groups them by `tool_call_id` — so parallel tool calls (where args for tool A and tool B interleave on the wire) work without any per-call index tracking. Some providers (Gemini) emit the complete tool call in one chunk — it's OK to emit both events back-to-back with the full name and full JSON. + +For client-side tool calls — tools that LLM should execute locally in a chain — **also call `response.add_tool_call()`**. The chain-execution path (`response.tool_calls()` → `execute_tool_calls()`) reads from the explicitly-added list, not from the StreamEvent buffer. + +```python +response.add_tool_call( + llm.ToolCall( + tool_call_id=tool_id, + name=tool_name, + arguments=parsed_args, + ) +) +``` + +(advanced-model-plugins-execute-tool-call)= +### Provider-managed local tool calls + +Some provider SDKs orchestrate the tool loop themselves: they call a local callback with generated arguments, wait for that callback to return a result, and then continue the same model response. Plugins for those providers can delegate the actual invocation to LLM using `response.execute_tool_call()`: + +```python +tool_result = response.execute_tool_call( + llm.ToolCall( + tool_call_id=tool_id, + name=tool_name, + arguments=parsed_args, + ) +) +return tool_result.output +``` + +Async model plugins should use the awaitable equivalent method on `AsyncResponse`: + +```python +tool_result = await response.execute_tool_call( + llm.ToolCall( + tool_call_id=tool_id, + name=tool_name, + arguments=parsed_args, + ) +) +return tool_result.output +``` + +This uses LLM's normal tool executor, including toolbox preparation, error handling, `llm_tool_call` injection and the `before_call` and `after_call` lifecycle callbacks. If `tool_call_id` is omitted LLM generates one. + +Do **not** also call `response.add_tool_call()` for a provider-managed call. That would cause the normal chain loop to execute the same tool a second time after the provider has already received its result. + +### Server-side tool calls + +For tools the API executes internally, set `server_executed=True` on the events. Anthropic web search is an example: the API returns a `server_tool_use` block for the search request, followed by a `web_search_tool_result` block containing the result payload. + +```python +yield StreamEvent( + type="tool_call_name", + chunk="web_search", + tool_call_id=tool_id, + server_executed=True, +) +yield StreamEvent( + type="tool_call_args", + chunk=json.dumps(query_args), + tool_call_id=tool_id, + server_executed=True, +) +``` + +The tool *result* (for example, the search hits) is also emitted as an event: + +```python +yield StreamEvent( + type="tool_result", + chunk=human_readable_summary, + tool_call_id=tool_id, + server_executed=True, + tool_name="web_search", + provider_metadata={"myprovider": {"raw_content": full_payload}}, +) +``` + +For providers that don't stream server-tool-result contents (Anthropic's `web_search_tool_result` blocks only arrive in the final message), emit those results as a post-stream step. After the main iteration loop completes, inspect the final message and emit tool_result events for any server-side results. + +Do **not** call `response.add_tool_call()` for server-side tool calls. This method should only be used for tool calls that need to be executed locally by the framework. + +### Opaque provider metadata + +Some providers require you to echo back opaque fields on the next request for multi-turn continuity to work: + +- **Anthropic** — `signature` on each thinking block; `encrypted_content` inside web_search_tool_result items. +- **Google Gemini** — `thoughtSignature` on `functionCall` parts when thinking is active. +- **OpenAI Responses API** — `encrypted_content` on reasoning items in stateless mode. + +These values are attached to a `StreamEvent` via its `provider_metadata` field. The framework merges metadata across events that group into the same Part (last non-None wins per top-level key) and persists it on the finalized Part. + +Namespace under your provider's name so transcripts that mix providers don't collide: + +```python +# Anthropic signature arrives at the end of a thinking block. +yield StreamEvent( + type="reasoning", + chunk="", + provider_metadata={"anthropic": {"signature": sig}}, +) +``` + +```python +# Gemini attaches thoughtSignature to a functionCall part. +yield StreamEvent( + type="tool_call_name", + chunk=name, + tool_call_id=tc_id, + provider_metadata={"gemini": {"thoughtSignature": sig}}, +) +``` + +The framework round-trips the value verbatim via JSON, so use JSON-safe primitives (string, int, bool, dict, list) for provider metadata - use base64 encoding if you need to store binary data. + +### Non-streaming path + +When `stream=False` (or the provider returns a complete message at once), emit one event per content block. + +```python +else: + completion = client.messages.create(**kwargs) + response.response_json = completion.model_dump() + for block in completion.content: + if block.type == "thinking": + yield StreamEvent( + type="reasoning", + chunk=block.thinking, + provider_metadata={"anthropic": {"signature": block.signature}}, + ) + elif block.type == "text": + yield StreamEvent(type="text", chunk=block.text) + elif block.type == "tool_use": + yield StreamEvent( + type="tool_call_name", + chunk=block.name, + tool_call_id=block.id, + ) + yield StreamEvent( + type="tool_call_args", + chunk=json.dumps(block.input), + tool_call_id=block.id, ) - for attachment in prev_response.attachments: - attachment_message.append(_attachment(attachment)) - messages.append({"role": "user", "content": attachment_message}) +``` + +(advanced-model-plugins-json-replacements)= + +## Condensing logged payloads with json_replacements + +The raw provider payload your plugin assigns to `response.response_json` is {ref}`logged condensed `: content that the log database already stores elsewhere - the response text, reasoning blobs, tool definitions - is replaced with references instead of being written twice. + +Your model class can improve on this by declaring a `json_replacements` class attribute: a dictionary of payload fragments that you know recur in every response from your provider. Consult [the default OpenAI plugin](https://github.com/simonw/llm/blob/main/llm/default_plugins/openai_models.py) for an example of this pattern. + +```python +class MyModel(llm.KeyModel): + json_replacements = { + "tool_usage_0": { + "image_gen": {"input_tokens": 0, "output_tokens": 0}, + "web_search": {"num_requests": 0}, + }, + "response_env_0": { + "object": "response", + "status": "completed", + "store": False, + }, + } +``` + +Payload content matching an entry is stored as a reference to it. Dict entries match structurally and additionally serve as *merge bases*: a payload object that mostly matches an entry, such as a response envelope where only the id and usage vary per call, is stored as the base plus a patch of its differing keys. + +Rules to follow: + +- **Entries are append-only. Never remove or change an existing entry - only add new ones.** Stored payloads reference entries by key and resolve against your dictionary when they are read back, so editing an entry silently breaks every payload already logged against it. When a provider changes a boilerplate block, append a new entry with a new suffix and leave the old one in place. +- Reading these payloads requires your plugin to be installed: resolution looks your model up by id in the registry. If the model is unknown at read time, the payload is reported as unavailable rather than resolved incorrectly. +- There is no size threshold for dictionary entries - you are trusted to curate them - but an entry smaller than the roughly 20-byte reference that replaces it makes payloads larger, not smaller. + +## Consuming prompt.messages in build_messages + +`prompt.messages` is an `list[llm.Message]` that is always **the complete input chain for this turn** — whether the caller supplied it explicitly via `model.prompt(messages=[...])`, or it was synthesized from kwargs (`prompt=`, `system=`, `attachments=`, `tool_results=`), or it was pre-built by a `Conversation` or by `response.reply()`. + +**Do not also walk `conversation.responses`.** History is already baked into `prompt.messages`; walking the conversation would double-emit. + +A plugin's `build_messages` (or equivalent) iterates `prompt.messages` and dispatches per `Part` subtype: + +```python +from llm.parts import ( + TextPart, + ReasoningPart, + ToolCallPart, + ToolResultPart, + AttachmentPart, +) + +def build_messages(self, prompt, conversation): + messages = [] + for msg in prompt.messages: + if msg.role == "system": + # Some APIs put system on a separate kwarg (Anthropic, Gemini). + # OpenAI-style APIs emit it as a message; handle accordingly. + continue + self._append_message(messages, msg) + return messages + +def _append_message(self, out, msg): + # Map llm's role to the provider's (assistant→model for Gemini, + # tool→user for Anthropic/Gemini tool_result convention, etc.) + role = self._provider_role(msg.role) + parts = [] + for part in msg.parts: + if isinstance(part, TextPart): + parts.append({"type": "text", "text": part.text}) + elif isinstance(part, ReasoningPart): + # Skip redacted reasoning (no content to echo back). + if part.redacted or not part.text: + continue + block = {"type": "thinking", "thinking": part.text} + # Restore the signature from provider_metadata. + sig = (part.provider_metadata or {}).get("anthropic", {}).get("signature") + if sig: + block["signature"] = sig + parts.append(block) + elif isinstance(part, ToolCallPart): + parts.append({ + "type": "tool_use", + "id": part.tool_call_id, + "name": part.name, + "input": part.arguments, + }) + elif isinstance(part, ToolResultPart): + parts.append({ + "type": "tool_result", + "tool_use_id": part.tool_call_id, + "content": part.output, + }) + elif isinstance(part, AttachmentPart) and part.attachment: + parts.append(self._attachment_block(part.attachment)) + # Merge with the previous message if roles match (some providers + # require strict alternation between user and assistant). + if out and out[-1]["role"] == role: + out[-1]["content"].extend(parts) else: - messages.append( - {"role": "user", "content": prev_response.prompt.prompt} - ) - messages.append({"role": "assistant", "content": prev_response.text_or_raise()}) + out.append({"role": role, "content": parts}) +``` + +## Restoring opaque metadata on subsequent requests + +When a conversation continues, your `build_messages` walks prior-turn Parts via `prompt.messages`. Each Part's `provider_metadata` is a `dict[str, dict]` keyed by provider name — extract your namespace and fold the fields back into the outgoing request body: + +```python +if isinstance(part, ReasoningPart): + block = {"type": "thinking", "thinking": part.text} + pm = (part.provider_metadata or {}).get("anthropic", {}) + if "signature" in pm: + block["signature"] = pm["signature"] + parts.append(block) + +if isinstance(part, ToolCallPart): + fc_part = {"function_call": {"name": part.name, "args": part.arguments}} + pm = (part.provider_metadata or {}).get("gemini", {}) + if "thoughtSignature" in pm: + # Gemini expects thoughtSignature beside function_call, + # not nested inside it. + fc_part["thoughtSignature"] = pm["thoughtSignature"] + parts.append(fc_part) ``` -The `response.text_or_raise()` method used there will return the text from the response or raise a `ValueError` exception if the response is an `AsyncResponse` instance that has not yet been fully resolved. -This is a slightly weird hack to work around the common need to share logic for building up the `messages` list across both sync and async models. +If the key is missing (an older transcript that pre-dates your plugin's support), fall through — don't fail. Treat other providers' entries as opaque; don't parse them. (advanced-model-plugins-usage)= diff --git a/docs/plugins/directory.md b/docs/plugins/directory.md index f26193999..2dd8d806d 100644 --- a/docs/plugins/directory.md +++ b/docs/plugins/directory.md @@ -15,6 +15,7 @@ These plugins all help you run LLMs directly on your own computer: - **[llm-mlc](https://github.com/simonw/llm-mlc)** can run local models released by the [MLC project](https://mlc.ai/mlc-llm/), including models that can take advantage of the GPU on Apple Silicon M1/M2 devices. - **[llm-gpt4all](https://github.com/simonw/llm-gpt4all)** adds support for various models released by the [GPT4All](https://gpt4all.io/) project that are optimized to run locally on your own machine. These models include versions of Vicuna, Orca, Falcon and MPT - here's [a full list of models](https://observablehq.com/@simonw/gpt4all-models). - **[llm-mpt30b](https://github.com/simonw/llm-mpt30b)** adds support for the [MPT-30B](https://huggingface.co/mosaicml/mpt-30b) local model. +- **[llm-lmstudio](https://github.com/agustif/llm-lmstudio)** provides access to local models using [LM Studio](https://lmstudio.ai/), (plugin-directory-remote-apis)= ## Remote APIs diff --git a/docs/plugins/llm-markov/llm_markov.py b/docs/plugins/llm-markov/llm_markov.py index 3ba4d9d8f..4b5d86585 100644 --- a/docs/plugins/llm-markov/llm_markov.py +++ b/docs/plugins/llm-markov/llm_markov.py @@ -1,8 +1,9 @@ -import llm import random import time -from typing import Optional -from pydantic import field_validator, Field + +from pydantic import Field, field_validator + +import llm @llm.hookimpl @@ -35,10 +36,10 @@ class Markov(llm.Model): can_stream = True class Options(llm.Options): - length: Optional[int] = Field( + length: int | None = Field( description="Number of words to generate", default=None ) - delay: Optional[float] = Field( + delay: float | None = Field( description="Seconds to delay between each token", default=None ) diff --git a/docs/plugins/plugin-hooks.md b/docs/plugins/plugin-hooks.md index 062919019..4b8b2922a 100644 --- a/docs/plugins/plugin-hooks.md +++ b/docs/plugins/plugin-hooks.md @@ -28,7 +28,7 @@ def register_commands(cli): This new command will be added to `llm --help` and can be run using `llm hello-world`. (plugin-hooks-register-models)= -## register_models(register) +## register_models(register, model_aliases) This hook can be used to register one or more additional models. @@ -60,8 +60,15 @@ def register_models(register): ``` This demonstrates how to register a model with both sync and async versions, and how to specify an alias for that model. +The `model_aliases` parameter is a list of {class}`~llm.ModelWithAliases` objects representing all models registered so far by other plugins. Plugins that use `@llm.hookimpl(trylast=True)` can use this to inspect or modify models registered by other plugins. Both parameters are optional - plugins can accept just `register`, just `model_aliases`, or both. + The {ref}`model plugin tutorial ` describes how to use this hook in detail. Asynchronous models {ref}`are described here `. +```{eval-rst} +.. autoclass:: llm.ModelWithAliases + :exclude-members: matches +``` + (plugin-hooks-register-embedding-models)= ## register_embedding_models(register) @@ -221,14 +228,14 @@ The `llm.Template` class has the following constructor: .. autoclass:: llm.Template ``` -The loader function should raise a `ValueError` if the template cannot be found or loaded correctly, providing a clear error message. +The loader function should raise a `ValueError` if the template cannot be found or loaded correctly, providing a clear error message. If you need to translate an exception from another library, catch that specific exception and use `raise ValueError(...) from ex` to preserve its context. Note that `functions:` provided by templates using this plugin hook will not be made available, to avoid the risk of plugin hooks that load templates from remote sources introducing arbitrary code execution vulnerabilities. (plugin-hooks-register-fragment-loaders)= ## register_fragment_loaders(register) -Plugins can register new fragment loaders using the `register_template_loaders` hook. These can then be used with the `llm -f prefix:argument` syntax. +Plugins can register new fragment loaders using the `register_fragment_loaders` hook. These can then be used with the `llm -f prefix:argument` syntax. Fragment loader plugins differ from template loader plugins in that you can stack more than one fragment loader call together in the same prompt. @@ -268,11 +275,13 @@ def my_fragment_loader(argument: str) -> llm.Fragment: ) # Or for the case where you want to return multiple fragments and attachments: -def my_fragment_loader(argument: str) -> list[llm.Fragment]: +def my_fragment_loader( + argument: str, +) -> list[llm.Fragment | llm.Attachment]: "Docs go here." return [ - llm.Fragment("Fragment 1 content", "my-fragments:{argument}"), - llm.Fragment("Fragment 2 content", "my-fragments:{argument}"), + llm.Fragment("Fragment 1 content", f"my-fragments:{argument}"), + llm.Fragment("Fragment 2 content", f"my-fragments:{argument}"), llm.Attachment(path="/path/to/image.png"), ] ``` @@ -282,4 +291,4 @@ llm -f my-fragments:argument ``` If multiple fragments are returned they will be used as if the user passed multiple `-f X` arguments to the command. -Multiple fragments are particularly useful for things like plugins that return every file in a directory. If these were concatenated together by the plugin, a change to a single file would invalidate the de-duplicatino cache for that whole fragment. Giving each file its own fragment means we can avoid storing multiple copies of that full collection if only a single file has changed. +Multiple fragments are particularly useful for things like plugins that return every file in a directory. If these were concatenated together by the plugin, a change to a single file would invalidate the de-duplication cache for that whole fragment. Giving each file its own fragment means we can avoid storing multiple copies of that full collection if only a single file has changed. diff --git a/docs/plugins/tutorial-model-plugin.md b/docs/plugins/tutorial-model-plugin.md index 5de5e45c1..2c67cbcb0 100644 --- a/docs/plugins/tutorial-model-plugin.md +++ b/docs/plugins/tutorial-model-plugin.md @@ -348,18 +348,14 @@ The `delay` token will let us simulate a streaming language model, where tokens Options are defined using an inner class on the model, called `Options`. It should extend the `llm.Options` class. -First, add this import to the top of your `llm_markov.py` file: -```python -from typing import Optional -``` -Then add this `Options` class to your model: +Add this `Options` class to your model: ```python class Markov(Model): model_id = "markov" class Options(llm.Options): - length: Optional[int] = None - delay: Optional[float] = None + length: int | None = None + delay: float | None = None ``` Let's add extra validation rules to our options. Length must be at least 2. Duration must be between 0 and 10. @@ -376,11 +372,11 @@ We can now add Pydantic field validators for our two new rules, plus inline docu ```python class Options(llm.Options): - length: Optional[int] = Field( + length: int | None = Field( description="Number of words to generate", default=None ) - delay: Optional[float] = Field( + delay: float | None = Field( description="Seconds to delay between each token", default=None ) diff --git a/docs/python-api.md b/docs/python-api.md index f42789fb1..76e2cf37b 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -7,12 +7,12 @@ Understanding this API is also important for writing {ref}`plugins`. ## Basic prompt execution -To run a prompt against the `gpt-4o-mini` model, run this: +To run a prompt against the `gpt-5.6-luna` model, run this: ```python import llm -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") # key= is optional, you can configure the key in other ways response = model.prompt( "Five surprising names for a pet pelican", @@ -26,7 +26,7 @@ If you inspect the response before it has been evaluated it will look like this: -The `llm.get_model()` function accepts model IDs or aliases. You can also omit it to use the currently configured default model, which is `gpt-4o-mini` if you have not changed the default. +The `llm.get_model()` function accepts model IDs or aliases. You can also omit it to use the currently configured default model, which is `gpt-5.6-luna` if you have not changed the default. In this example the key is set by Python code. You can also provide the key using the `OPENAI_API_KEY` environment variable, or use the `llm keys set openai` command to store it in a `keys.json` file, see {ref}`api-keys`. @@ -68,7 +68,7 @@ This example shows two attachments - one from a file path and one from a URL: ```python import llm -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") response = model.prompt( "Describe these images", attachments=[ @@ -79,10 +79,15 @@ response = model.prompt( ``` Use `llm.Attachment(content=b"binary image content here")` to pass binary content directly. +```{eval-rst} +.. autoclass:: llm.Attachment + :members: resolve_type, content_bytes, base64_content +``` + You can check which attachment types (if any) a model supports using the `model.attachment_types` set: ```python -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") print(model.attachment_types) # {'image/gif', 'image/png', 'image/jpeg', 'image/webp'} @@ -111,12 +116,26 @@ response = model.prompt("Convert panda to upper", tools=[upper]) tool_calls = response.tool_calls() # [ToolCall(name='upper', arguments={'text': 'panda'}, tool_call_id='...')] ``` +Every tool call is guaranteed to have a unique `tool_call_id`. Most providers supply their own; for providers that do not, LLM synthesizes one of the form `tc_01...`, so you can always use the id to correlate a tool call with its result or to key external state against a specific invocation. You can call `response.execute_tool_calls()` to execute those calls and get back the results: ```python tool_results = response.execute_tool_calls() # [ToolResult(name='upper', output='PANDA', tool_call_id='...')] ``` -You can use the `model.chain()` to pass the results of tool calls back to the model automatically as subsequent prompts: +To get the model's follow-up reply, call `response.reply()` — when the previous response made tool calls, `reply()` automatically executes them and feeds the results back into the next turn: +```python +follow_up = response.reply() +print(follow_up.text()) +# The word "panda" converted to uppercase is "PANDA". +``` +You can also pass an additional user prompt: `response.reply("now translate it to French")`. To use custom or already-computed tool results (e.g. results you mutated, or synthetic ones for testing) pass them explicitly with `tool_results=` and the auto-execute step is skipped: +```python +follow_up = response.reply( + "now translate it", + tool_results=[llm.ToolResult(name="upper", output="PANDA", tool_call_id="...")], +) +``` +For an automatic loop that keeps going until the model stops requesting tools, use `model.chain()` — it passes tool call results back to the model automatically as subsequent prompts: ```python chain_response = model.chain( "Convert panda to upper", @@ -154,20 +173,20 @@ for response in chain.responses(): Pass a function to the `before_call=` parameter of `model.chain()` to have that called before every tool call is executed. You can raise `llm.CancelToolCall()` to cancel that tool call. -The method signature is `def before_call(tool: Optional[llm.Tool], tool_call: llm.ToolCall)` - that first `tool` argument can be `None` if the model requests a tool be executed that has not been provided in the `tools=` list. +The method signature is `def before_call(tool: llm.Tool | None, tool_call: llm.ToolCall)` - that first `tool` argument can be `None` if the model requests a tool be executed that has not been provided in the `tools=` list. Here's an example: ```python import llm -from typing import Optional def upper(text: str) -> str: "Convert text to uppercase." return text.upper() -def before_call(tool: Optional[llm.Tool], tool_call: llm.ToolCall): - print(f"About to call tool {tool.name} with arguments {tool_call.arguments}") - if tool.name == "upper" and "bad" in repr(tool_call.arguments): +def before_call(tool: llm.Tool | None, tool_call: llm.ToolCall): + tool_name = tool.name if tool is not None else tool_call.name + print(f"About to call tool {tool_name} with arguments {tool_call.arguments}") + if tool_name == "upper" and "bad" in repr(tool_call.arguments): raise llm.CancelToolCall("Not allowed to call upper on text containing 'bad'") model = llm.get_model("gpt-4.1-mini") @@ -193,6 +212,76 @@ response = model.chain( print(response.text()) ``` +(python-api-tools-llm-tool-call)= + +#### Accessing the tool call from inside a tool + +Tool implementations sometimes need to know about the `llm.ToolCall` that triggered them - most often the `tool_call_id` (always populated, see above), which can be used to key external state against that specific invocation. + +If your tool function accepts a parameter named `llm_tool_call` it will be passed the `llm.ToolCall` object for the current call: + +```python +import llm + +def lookup(name: str, llm_tool_call: llm.ToolCall) -> str: + "Look up a name." + return do_lookup(name, request_id=llm_tool_call.tool_call_id) +``` + +The `llm_tool_call` parameter name is reserved: it is excluded from the input schema that is exposed to the model and is populated automatically when the tool executes. The type annotation is optional. + +This works for both sync and async tool functions, and for methods on `llm.Toolbox` subclasses. The parameter must be declared explicitly - a `**kwargs` catch-all will not receive `llm_tool_call`. + +(python-api-tools-pause)= + +#### Pausing a chain from inside a tool + +Sometimes a tool cannot finish without outside input - human approval being the classic case. Raise `llm.PauseChain` inside a tool implementation to stop the chain cleanly: + +```python +import llm + +def delete_files(path: str) -> str: + if not approval_already_recorded(path): + record_approval_request(path) + raise llm.PauseChain("waiting for approval to delete " + path) + do_delete(path) + return "deleted" +``` + +Unlike other exceptions - which are converted into `"Error: ..."` tool results and sent back to the model - `PauseChain` propagates out of the chain to your code. No provider call is made with a placeholder result. Before re-raising, the framework populates two attributes: + +- `pause.tool_call` - the `llm.ToolCall` whose implementation paused +- `pause.tool_results` - results of sibling calls in the same batch that completed + +```python +try: + chain_response.text() +except llm.PauseChain as pause: + print("Paused on", pause.tool_call.name, pause.tool_call.tool_call_id) +``` + +The failure semantics are defined: concurrent (async) sibling tool calls always run to completion before the exception propagates - their `after_call` hooks fire and their results are preserved - while sequential (sync) execution stops at the paused call, leaving later calls unexecuted so they can safely run on resume. If several concurrent calls pause, the first by call order propagates. `after_call` does not fire for a paused call, and no `ToolResult` is recorded for it - which is what marks it as still pending. + +(python-api-tools-resume)= + +#### Resuming a chain with pending tool calls + +To resume after a pause (or a crash, or a server restart), re-run the chain with a `messages=` history that ends in the unresolved tool calls. When the last assistant message in the history contains tool calls that have no matching results, the chain executes them first - through the normal `before_call`/`after_call` machinery - and then sends the results to the model: + +```python +chain = model.chain( + messages=persisted_messages, # ends in assistant tool calls with no results + tools=[delete_files], + system=system_prompt, +) +chain.text() +``` + +Calls that already have results in trailing tool-role messages are skipped, so a batch where some calls completed before the pause only re-executes the unresolved ones. A re-executed tool may raise `PauseChain` again - multi-step approval flows work by repeating the cycle. If a user or assistant message follows the tool calls in the history, the conversation has moved on and nothing is re-executed. + +Matching uses `tool_call_id` (always populated for newly-created tool calls); id-less calls from older persisted histories match results by name. You can also execute an explicit list of calls directly with `response.execute_tool_calls(tool_calls_list=[...])`. + (python-api-tools-attachments)= #### Tools can return attachments @@ -217,6 +306,31 @@ def generate_image(prompt: str) -> llm.ToolOutput: ) ``` +```{eval-rst} +.. autoclass:: llm.ToolOutput +``` + +(python-api-server-side-tools)= + +#### Server-side tools + +Server-side tools are executed by the model provider during a response. Pass them in the same `tools=` list as function tools, but do not call `execute_tool_calls()` for them: their calls and results arrive in `response.messages()` with `server_executed=True` and are excluded from the local execution loop. + +For example, OpenAI Responses models support {ref}`Web Search ` and {ref}`Code Interpreter `: + +```python +import llm +from llm.default_plugins.openai_models import CodeInterpreter + +response = llm.get_model("gpt-5.6-luna").prompt( + "Use the python tool to calculate the first 20 Fibonacci numbers", + tools=[CodeInterpreter(memory_limit="4g")], +) +print(response.text()) +``` + +Provider plugins define these tools by subclassing {class}`llm.ServerSideTool`. Each model instance returns its supported classes from `supported_server_side_tools`; a model rejects classes it has not explicitly included rather than silently dropping them. See {ref}`advanced-model-plugins-server-side-tools` for the plugin API. + (python-api-toolbox)= #### Toolbox classes @@ -258,6 +372,12 @@ class Memory(llm.Toolbox): "Return a list of keys" return list(self._get_memory().keys()) ``` + +```{eval-rst} +.. autoclass:: llm.Toolbox + :members: tools, add_tool, prepare, prepare_async +``` + You can then use that from Python like this: ```python model = llm.get_model("gpt-4.1-mini") @@ -302,6 +422,8 @@ In asynchronous contexts the alternative method `await toolbox.prepare_async()` If you want to prepare the class in this way such that it can be used in both synchronous and asynchronous contexts, implement both `prepare()` and `prepare_async()` methods. +Toolbox classes that override `tools()`, `prepare()` or `prepare_async()` are treated as dynamic by the `llm tools` command. Since their full list of tools cannot be determined until an instance has been configured, `llm tools` displays their constructor signature and class docstring instead of a list of tools - so give your dynamic toolbox a descriptive docstring that shows how to configure it. Users can then pass a full specification such as `llm tools 'MCP("https://example.com/mcp")'` to instantiate the toolbox - running `prepare()` if it is implemented - and list the tools that instance provides. See {ref}`the tools usage documentation ` for details. + (python-api-schemas)= ### Schemas @@ -318,7 +440,7 @@ class Dog(BaseModel): name: str age: int -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") response = model.prompt("Describe a nice dog", schema=Dog) dog = json.loads(response.text()) print(dog) @@ -361,11 +483,13 @@ print(model.prompt( The {ref}`fragment system ` from the CLI tool can also be accessed from the Python API, by passing `fragments=` and/or `system_fragments=` lists of strings to the `prompt()` method: ```python +from pathlib import Path + response = model.prompt( "What do these documents say about dogs?", fragments=[ - open("dogs1.txt").read(), - open("dogs2.txt").read(), + Path("dogs1.txt").read_text(), + Path("dogs2.txt").read_text(), ], system_fragments=[ "You answer questions like Snoopy", @@ -383,11 +507,11 @@ Some model plugins may include features that take advantage of fragments, for ex ### Model options -For models that support options (view those with `llm models --options`) you can pass options as keyword arguments to the `.prompt()` method: +For models that support options (view those with `llm models --options`) pass them as a dictionary to the `options=` argument of the `.prompt()` method: ```python model = llm.get_model() -print(model.prompt("Names for otters", temperature=0.2)) +print(model.prompt("Names for otters", options={"temperature": 0.2})) ``` (python-api-models-api-keys)= @@ -397,7 +521,7 @@ print(model.prompt("Names for otters", temperature=0.2)) Models that accept API keys should take an additional `key=` parameter to their `model.prompt()` method: ```python -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") print(model.prompt("Names for beavers", key="sk-...")) ``` @@ -438,32 +562,30 @@ You can access this JSON data as a Python dictionary using the `response.json()` import llm from pprint import pprint -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") response = model.prompt("3 names for an otter") json_data = response.json() pprint(json_data) ``` -Here's that example output from GPT-4o mini: -```python -{'content': 'Sure! Here are three fun names for an otter:\n' - '\n' - '1. **Splash**\n' - '2. **Bubbles**\n' - '3. **Otto** \n' - '\n' - 'Feel free to mix and match or use these as inspiration!', - 'created': 1739291215, - 'finish_reason': 'stop', - 'id': 'chatcmpl-AznO31yxgBjZ4zrzBOwJvHEWgdTaf', - 'model': 'gpt-4o-mini-2024-07-18', - 'object': 'chat.completion.chunk', - 'usage': {'completion_tokens': 43, - 'completion_tokens_details': {'accepted_prediction_tokens': 0, - 'audio_tokens': 0, - 'reasoning_tokens': 0, - 'rejected_prediction_tokens': 0}, - 'prompt_tokens': 13, - 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}, +Here's an abbreviated example of that output from GPT-5.6 Luna: +```python +{'created_at': 1785451200.0, + 'id': 'resp_...', + 'model': 'gpt-5.6-luna', + 'object': 'response', + 'output': [ + # Reasoning and message items are abbreviated here + {'content': [{'annotations': [], + 'text': '1. Splash\n2. Bubbles\n3. Otto', + 'type': 'output_text'}], + 'id': 'msg_...', + 'role': 'assistant', + 'status': 'completed', + 'type': 'message'}], + 'status': 'completed', + 'usage': {'input_tokens': 13, + 'output_tokens': 43, + 'output_tokens_details': {'reasoning_tokens': 0}, 'total_tokens': 56}} ``` @@ -486,7 +608,9 @@ Usage(input=5, 'tokenCount': 2}], 'promptTokensDetails': [{'modality': 'TEXT', 'tokenCount': 5}]}) ``` -The `.input` and `.output` properties are integers representing the number of input and output tokens. The `.details` property may be a dictionary with additional custom values that vary by model. +```{eval-rst} +.. autoclass:: llm.Usage +``` (python-api-streaming-responses)= @@ -503,6 +627,124 @@ The `response.text()` method described earlier does this for you - it runs throu If a response has been evaluated, `response.text()` will continue to return the same string. +```{eval-rst} +.. autoclass:: llm.Response + :members: text, json, usage, tool_calls, on_done + :exclude-members: fake, from_row, log_to_db +``` + +(python-api-messages)= + +### Structured messages and streaming events + +Many LLMs return structure that goes beyond a plain text response. LLM represents these using **messages** that consist of **parts**. + +A conversation consists of turns, where each turn is an `llm.Message` with a `role` (`"user"`, `"assistant"`, `"system"`, or `"tool"`) and a list of `Part` objects — `TextPart`, `ReasoningPart`, `ToolCallPart`, `ToolResultPart`, or `AttachmentPart`. + +You can pass structured prompt inputs via `messages=[...]`, iterate over typed events as the model streams, and inspect the assembled message after the response completes. + +Here's how to prompt a model with a list of messages instead of a plain text prompt: + +```python +import llm +from llm import user, assistant, system + +model = llm.get_model("gpt-5.4-mini") + +response = model.prompt(messages=[ + system("You are a helpful pirate."), + user("What is the capital of France?"), + assistant("Paris, matey."), + user("And Germany?"), +]) +print(response.text()) +``` + +The `user()`, `assistant()`, and `system()` helpers accept strings (wrapped as `TextPart`) but can also accept `llm.Attachment` instances (wrapped as `AttachmentPart`) or more complex sequences of `Part` objects. + +Calling `model.prompt("hi", system="Be brief.")` is equivalent to `model.prompt(messages=[system("Be brief."), user("hi")])`. + +#### Streaming events as they arrive + +`response.stream_events()` yields typed events for every content block the model produces as they stream in. This is useful for interfaces that show the model response "live". + +```python +response = model.prompt("Explain quantum computing briefly.") +for event in response.stream_events(): + if event.type == "reasoning": + print(f"[thinking] {event.chunk}", end="", flush=True) + elif event.type == "text": + print(event.chunk, end="", flush=True) + elif event.type == "tool_call_name": + print(f"\n[calling tool: {event.chunk}]") + elif event.type == "tool_call_args": + print(event.chunk, end="", flush=True) +``` + +Event types are `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, and `"tool_result"`. Each event carries a `part_index` that groups events into the same logical Part (all events at the same `part_index` assemble into one Part after the stream completes). For async models, use `async for event in response.astream_events()`. + +Iterating against the response object itself (`for chunk in response`) yields only text strings — reasoning and tool-call events are filtered out. + +(python-api-messages-reasoning)= + +#### Hiding reasoning output + +Some model plugins can return visible reasoning text, exposed as `"reasoning"` events from `response.stream_events()` and assembled as `ReasoningPart` objects in `response.messages()`. + +Pass `hide_reasoning=True` to ask LLM and supported model plugins not to expose that visible reasoning output: + +```python +response = model.prompt( + "Explain quantum computing briefly.", + hide_reasoning=True, +) +print(response.text()) +``` + +This is the Python API equivalent of the CLI `-R/--hide-reasoning` option. It is available on `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, and their async counterparts. + +Note that this only requests that the underlying model does not return visible tokens. This request may not be supported by your provider, in which case this hint will not prevent visible reasoning tokens from being returned in the stream. + +#### Inspecting the finished response + +`response.messages()` returns the assembled list of `Message` objects produced by the model, excluding the messages from the original prompt. Calling it forces execution if the response hasn't been drained yet, so you don't need a separate `response.text()` first: + +```python +response = model.prompt("What's 2+2?") +for message in response.messages(): + for part in message.parts: + print(type(part).__name__, part.to_dict()) +``` + +On async models `messages()` is awaitable: `await response.messages()`. + +#### Persisting a conversation + +A `Response` can round-trip through a plain Python dictionary via `response.to_dict()` and `llm.Response.from_dict(...)`. The dict captures the model id, the input messages that were sent, the assistant output, and any options. The re-inflated object can be used to continue the conversation. + +Use `response.reply(...)` to continue from a rehydrated response: + +```python +import json +import llm + +model = llm.get_model("gpt-5.4-mini") +response = model.prompt("What's 2+2?") +print(response.text()) + +payload = json.dumps(response.to_dict()) +# ...save `payload` wherever you want... + +# Later — rehydrate and continue. +rebuilt = llm.Response.from_dict(json.loads(payload)) +followup = rebuilt.reply("Add 3 to that") +print(followup.text()) +``` + +`AttachmentPart` bytes are base64-encoded in the dict form, so multi-modal conversations round-trip via JSON too. + +Individual `Message` and `Part` objects also support `to_dict()` / `from_dict()` if you need to manipulate turns directly — for example, to edit, filter, or splice messages before passing them back via `model.prompt(messages=[...])`. + (python-api-async)= ## Async models @@ -529,6 +771,13 @@ async for chunk in model.prompt( ): print(chunk, end="", flush=True) ``` + +```{eval-rst} +.. autoclass:: llm.AsyncResponse + :members: text, json, usage, tool_calls, on_done + :exclude-members: fake, from_row, log_to_db +``` + This `await model.prompt()` method takes the same arguments as the synchronous `model.prompt()` method, for options and attachments and `key=` and suchlike. (python-api-async-tools)= @@ -582,6 +831,13 @@ async for chunk in model.chain( ): print(chunk, end="", flush=True) ``` +`response.reply()` is awaitable on async models — it `await`s `execute_tool_calls()` internally before building the next turn: +```python +response = model.prompt("Convert panda to upper", tools=[upper]) +await response.text() +follow_up = await response.reply() +print(await follow_up.text()) +``` The `before_call` and `after_call` hooks can be async functions when used with async models. (python-api-conversations)= @@ -685,7 +941,7 @@ Example usage: ```python import llm -model = llm.get_model("gpt-4o-mini") +model = llm.get_model("gpt-5.6-luna") response = model.prompt("a poem about a hippo") response.on_done(lambda response: print(response.usage())) print(response.text()) @@ -702,7 +958,7 @@ Or using an `asyncio` model, where you need to `await response.on_done(done)` to import asyncio, llm async def run(): - model = llm.get_async_model("gpt-4o-mini") + model = llm.get_async_model("gpt-5.6-luna") response = model.prompt("a short poem about a brick") async def done(response): print(await response.usage()) @@ -724,7 +980,7 @@ The `llm.set_alias()` function can be used to define a new alias: ```python import llm -llm.set_alias("mini", "gpt-4o-mini") +llm.set_alias("mini", "gpt-5.6-luna") ``` The second argument can be a model identifier or another alias, in which case that alias will be resolved. @@ -749,12 +1005,12 @@ This sets the default model to the given model ID or alias. Any changes to defau ```python import llm -llm.set_default_model("claude-3.5-sonnet") +llm.set_default_model("claude-5-sonnet") ``` ### get_default_model() -This returns the currently configured default model, or `gpt-4o-mini` if no default has been set. +This returns the currently configured default model, or `gpt-5.6-luna` if no default has been set. ```python import llm diff --git a/docs/schemas.md b/docs/schemas.md index 219e3d7a7..175bdb847 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -32,7 +32,7 @@ I got back Ziggy: ``` The response matched my schema, with `name` and `one_sentence_bio` string columns and an integer for `age`. -We're using the default LLM model here - `gpt-4o-mini`. Add `-m model` to use another model - for example use `-m o3-mini` to have O3 mini invent some dogs. +We're using the default LLM model here - `gpt-5.6-luna`. Add `-m model` to use another model - for example use `-m gpt-5.6-sol` to have GPT-5.6 Sol invent some dogs. For a list of available models that support schemas, run this command: ```bash @@ -596,4 +596,4 @@ Output: ``` If a row already has a property called `"conversation_id"` or `"response_id"` additional underscores will be appended to the ID key until it no longer overlaps with the existing keys. -The `--id-gt $ID` and `--id-gte $ID` options can be useful for ignoring logged schema data prior to a certain point, see {ref}`logging-filter-id` for details. \ No newline at end of file +The `--id-gt $ID` and `--id-gte $ID` options can be useful for ignoring logged schema data prior to a certain point, see {ref}`logging-filter-id` for details. diff --git a/docs/setup.md b/docs/setup.md index 72801ba68..2460a1014 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -167,7 +167,7 @@ You can configure LLM in a number of different ways. (setup-default-model)= ### Setting a custom default model -The model used when calling `llm` without the `-m/--model` option defaults to `gpt-4o-mini` - the fastest and least expensive OpenAI model. +The model used when calling `llm` without the `-m/--model` option defaults to `gpt-5.6-luna` - a fast and less expensive OpenAI model. You can use the `llm models default` command to set a different default model. For GPT-4o (slower and more expensive, but more capable) run this: @@ -205,4 +205,4 @@ Or turn it back on again with: ``` llm logs on ``` -Run `llm logs status` to see the current states of the setting. \ No newline at end of file +Run `llm logs status` to see the current states of the setting. diff --git a/docs/templates.md b/docs/templates.md index 3757af4b7..0a2cc93a1 100644 --- a/docs/templates.md +++ b/docs/templates.md @@ -335,7 +335,7 @@ extract: true ### Setting a default model for a template -Templates executed using `llm -t template-name` will execute using the default model that the user has configured for the tool - or `gpt-3.5-turbo` if they have not configured their own default. +Templates executed using `llm -t template-name` will execute using the default model that the user has configured for the tool - or `gpt-5.6-luna` if they have not configured their own default. You can specify a new default model for a template using the `model:` key in the associated YAML. Here's a template called `roast`: @@ -384,4 +384,4 @@ The `-sL` flags to `curl` are used to follow redirects and suppress progress met This command will fetch the content of the LLM index page and feed it to the template defined by [summarize.yaml](https://github.com/simonw/llm-templates/blob/main/summarize.yaml) in the [simonw/llm-templates](https://github.com/simonw/llm-templates) GitHub repository. -If two template loader plugins attempt to register the same prefix one of them will have `_1` added to the end of their prefix. Use `llm templates loaders` to check if this has occurred. \ No newline at end of file +If two template loader plugins attempt to register the same prefix one of them will have `_1` added to the end of their prefix. Use `llm templates loaders` to check if this has occurred. diff --git a/docs/tools.md b/docs/tools.md index a67600745..bc37078b8 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -64,11 +64,11 @@ Further tools can be installed using plugins, or you can use the `llm --function ## LLM's implementation of tools -In LLM every tool is a defined as a Python function. The function can take any number of arguments and can return a string or an object that can be converted to a string. +In LLM every tool is defined as a Python function. The function can take any number of arguments and can return a string or an object that can be converted to a string. Tool functions should include a docstring that describes what the function does. This docstring will become the description that is passed to the model. -Tools can also be defined as {ref}`toolbox classes `, a subclass of `llm.Toolbox` that allows multiple related tools to be bundled together. Toolbox classes can be be configured when they are instantiated, and can also maintain state in between multiple tool calls. +Tools can also be defined as {ref}`toolbox classes `, a subclass of `llm.Toolbox` that allows multiple related tools to be bundled together. Toolbox classes can be configured when they are instantiated, and can also maintain state in between multiple tool calls. The Python API can accept functions directly. The command-line interface has two ways for tools to be defined: via plugins that implement the {ref}`register_tools() plugin hook `, or directly on the command-line using the `--functions` argument to specify a block of Python code defining one or more functions - or a path to a Python file containing the same. @@ -97,4 +97,6 @@ Consult the {ref}`register_tools() plugin hook ` do If your plugin needs access to API secrets I recommend storing those using `llm keys set api-name` and then reading them using the {ref}`plugin-utilities-get-key` utility function. This avoids secrets being logged to the database as part of tool calls. +If your tool implementation needs to know which tool call invoked it - for example to key state against the unique `tool_call_id` - add a parameter named `llm_tool_call` to your function. It will be passed the `llm.ToolCall` object for the current invocation, and is hidden from the schema the model sees. See {ref}`python-api-tools-llm-tool-call` for details. + diff --git a/docs/usage.md b/docs/usage.md index 5298e85e2..f4271cddf 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -6,7 +6,7 @@ The command to run a prompt is `llm prompt 'your prompt'`. This is the default c (usage-executing-prompts)= ## Executing a prompt -These examples use the default OpenAI `gpt-4o-mini` model, which requires you to first {ref}`set an OpenAI API key `. +These examples use the default OpenAI `gpt-5.6-luna` model, which requires you to first {ref}`set an OpenAI API key `. You can {ref}`install LLM plugins ` to use models from other providers, including openly licensed models you can run directly on your own computer. @@ -18,22 +18,21 @@ To disable streaming and only return the response once it has completed: ```bash llm 'Ten names for cheesecakes' --no-stream ``` -To switch from ChatGPT 4o-mini (the default) to GPT-4o: +To switch from the default model to GPT-5.6 Sol: ```bash -llm 'Ten names for cheesecakes' -m gpt-4o +llm 'Ten names for cheesecakes' -m gpt-5.6-sol ``` -You can use `-m 4o` as an even shorter shortcut. Pass `--model ` to use a different model. Run `llm models` to see a list of available models. Or if you know the name is too long to type, use `-q` once or more to provide search terms - the model with the shortest model ID that matches all of those terms (as a lowercase substring) will be used: ```bash -llm 'Ten names for cheesecakes' -q 4o -q mini +llm 'Ten names for cheesecakes' -q gpt -q sol ``` To change the default model for the current session, set the `LLM_MODEL` environment variable: ```bash -export LLM_MODEL=gpt-4.1-mini -llm 'Ten names for cheesecakes' # Uses gpt-4.1-mini +export LLM_MODEL=gpt-4.1 +llm 'Ten names for cheesecakes' # Uses gpt-4.1 ``` You can send a prompt directly to standard input like this: @@ -59,14 +58,14 @@ Some models support options. You can pass these using `-o/--option name value` - llm 'Ten names for cheesecakes' -o temperature 1.5 ``` -Use the `llm models --options` command to see which options are supported by each model. +Use the `llm models --options` command to see which options are supported by each model, or `llm -m gpt-5.5 --options` to show the options for a specific selected model. You can also {ref}`configure default options ` for a model using the `llm models options` commands. (usage-attachments)= ### Attachments -Some models are multi-modal, which means they can accept input in more than just text. GPT-4o and GPT-4o mini can accept images, and models such as Google Gemini 1.5 can accept audio and video as well. +Some models are multi-modal, which means they can accept input in more than just text. GPT-4o and GPT-4o mini can accept images, and models such as Google's `gemini-flash-latest` can accept audio and video as well. LLM calls these **attachments**. You can pass attachments using the `-a` option like this: @@ -172,7 +171,15 @@ Run this command to see a list of available tools from plugins: ```bash llm tools ``` -If you run a prompt that uses tools from plugins (as opposed to tools provided using the `--functions` option) continuing that conversation using `llm -c` will reuse the tools from the first prompt. Running `llm chat -c` will start a chat that continues using those same tools. For example: +Server-side tools are model-specific. Pass `-m/--model` to include the tools supported by a particular model: + +```bash +llm tools -m gpt-5.6-luna +``` + +These are displayed in a separate `Server-side tools` section with their constructor signatures and documentation. Add `--json` to return them in a `server_side_tools` array whose entries have `"server_side": true`. + +If you run a prompt that uses tools from plugins or model-specific server-side tools, continuing that conversation using `llm -c` will reuse the tools from the first prompt. Configured constructor arguments such as `CodeInterpreter(memory_limit="4g")` are retained. Running `llm chat -c` will start a chat that continues using those same tools. For example: ``` llm -T simple_eval "12345 * 12345" --td @@ -208,7 +215,43 @@ Toolboxes always start with a capital letter. They can be configured by passing - Single JSON value: `ToolboxName("hello")` or `ToolboxName([1,2,3])` - Key-value pairs: `ToolboxName(name="test", count=5, items=[1,2])` - treated the same as `{"name": "test", "count": 5, "items": [1, 2]}`, all values must be valid JSON -Toolboxes are not currently supported with the `llm -c` option, but they work well with `llm chat`. Try chatting with the Datasette content database like this: +The `llm tools` command lists toolboxes along with the tools they provide. Some toolboxes cannot know their list of tools until they have been configured - [llm-mcp-client](https://github.com/simonw/llm-mcp-client) for example fetches its tools from whichever MCP server it is pointed at. These dynamic toolboxes are listed with their constructor signature and documentation instead: + +``` +MCP(server, mode='auto', prefix='') (plugin: mcp_client) + + Expose the tools from an MCP server as LLM tools. + + Usage: + + MCP("https://example.com/mcp") + MCP("https://example.com/mcp", mode="legacy", prefix="demo_") +``` + +Pass a full toolbox specification to `llm tools` to see the tools provided by a configured instance: + +```bash +llm tools 'MCP("https://datasette.simonwillison.net/-/mcp")' +``` +``` +MCP("https://datasette.simonwillison.net/-/mcp"): + + list_databases(**kwargs) + + List the databases available in this Datasette instance. + + get_database_schema(**kwargs) + + Return the complete SQL schema for a database. + + execute_sql(**kwargs) + + Execute one read-only SQL statement and return its tabular results. +``` + +Continuing a conversation with `llm -c` or `llm chat -c` reconstructs any toolboxes from the configuration recorded in the logs, so you don't need to repeat the `-T` option. Each instance is rebuilt fresh, which means any in-memory state a toolbox accumulated during the earlier prompt does not carry over. + +Toolboxes work well with `llm chat`. Try chatting with the Datasette content database like this: ```bash llm chat -T 'Datasette("https://datasette.io/content")' --td @@ -239,6 +282,49 @@ Use `--xl/--extract-last` to return the last fenced code block instead of the fi The entire response including explanatory text is still logged to the database, and can be viewed using `llm logs -c`. +(usage-json-output)= +### JSON output + +Add `--json` to get back a JSON array describing the prompt and the response, in the same format as {ref}`llm logs --json `: + +```bash +llm 'Five names for a pet pelican' --json +``` +```json +[ + { + "id": "01jm8ec74wxsdatyn5pq1fp0s5", + "model": "gpt-5.6-luna", + "resolved_model": null, + "prompt": "Five names for a pet pelican", + "system": null, + "prompt_json": null, + "options_json": {}, + "response": "1. Captain Beaky\n...", + "reasoning": null, + "response_json": null, + "conversation_id": "01jm8ec74taftdgj2t4zra9z0j", + "duration_ms": 1560, + "datetime_utc": "2025-02-16T22:34:30.374882+00:00", + "input_tokens": 8, + "output_tokens": 62, + "token_details": null, + "conversation_name": "Five names for a pet pelican", + "conversation_model": "gpt-5.6-luna", + "schema_json": null, + "prompt_fragments": [], + "system_fragments": [], + "tools": [], + "tool_calls": [], + "tool_results": [], + "attachments": [] + } +] +``` +The array will contain more than one object if the prompt triggered {ref}`tool calls `, since each round-trip with the model is logged as a separate response. + +This works even if logging is turned off or you use `-n/--no-log` - in that case the JSON is assembled without writing anything to your logs database. + (usage-schemas)= ### Schemas @@ -267,7 +353,7 @@ llm --schema '{ } } } -}' -m gpt-4o-mini 'invent two dogs' +}' -m gpt-5.6-sol 'invent two dogs' ``` Or use LLM's custom {ref}`concise schema syntax ` like this: @@ -408,26 +494,26 @@ llm chat -c For models that support them, you can pass options using `-o/--option`: ```bash -llm chat -m gpt-4 -o temperature 0.5 +llm chat -m gpt-4.1 -o temperature 0.5 ``` You can pass a system prompt to be used for your chat conversation: ```bash -llm chat -m gpt-4 -s 'You are a sentient cheesecake' +llm chat -m gpt-5.6-luna -s 'You are a sentient cheesecake' ``` You can also pass {ref}`a template ` - useful for creating chat personas that you wish to return to. -Here's how to create a template for your GPT-4 powered cheesecake: +Here's how to create a template for your GPT-5.6 Luna powered cheesecake: ```bash -llm --system 'You are a sentient cheesecake' -m gpt-4 --save cheesecake +llm --system 'You are a sentient cheesecake' -m gpt-5.6-luna --save cheesecake ``` Now you can start a new chat with your cheesecake any time you like using this: ```bash llm chat -t cheesecake ``` ``` -Chatting with gpt-4 +Chatting with gpt-5.6-luna Type 'exit' or 'quit' to exit Type '!multi' to enter multiple lines, then '!end' to finish Type '!edit' to open your default editor and modify the prompt @@ -449,7 +535,7 @@ To do that, type `!multi` to start a multi-line input. Type or paste your text, If your pasted text might itself contain a `!end` line, you can set a custom delimiter using `!multi abc` followed by `!end abc` at the end: ``` -Chatting with gpt-4 +Chatting with gpt-5.6-luna Type 'exit' or 'quit' to exit Type '!multi' to enter multiple lines, then '!end' to finish Type '!edit' to open your default editor and modify the prompt. @@ -469,7 +555,7 @@ urllib.error.URLError: `: ```bash -llm models options show gpt-4o +llm models options show gpt-4.1 ``` To clear a default option, use the `llm models options clear` command: ```bash -llm models options clear gpt-4o temperature +llm models options clear gpt-4.1 temperature ``` Or clear all default options for a model like this: ```bash -llm models options clear gpt-4o +llm models options clear gpt-4.1 ``` Default model options are respected by both the `llm prompt` and the `llm chat` commands. They will not be applied when you use LLM as a {ref}`Python library `. diff --git a/llm/__init__.py b/llm/__init__.py index 09ee01844..de994edb1 100644 --- a/llm/__init__.py +++ b/llm/__init__.py @@ -1,8 +1,19 @@ -from .hookspecs import hookimpl +import inspect +import json +import os +import pathlib +import struct +from collections.abc import Callable +from typing import Any + +import click + +from .embeddings import Collection from .errors import ( ModelError, NeedsKeyException, ) +from .hookspecs import hookimpl from .models import ( AsyncConversation, AsyncKeyModel, @@ -17,25 +28,27 @@ Model, ModelWithAliases, Options, + PauseChain, Prompt, Response, + ServerSideTool, Tool, Toolbox, ToolCall, ToolOutput, ToolResult, + Usage, ) -from .utils import schema_dsl, Fragment -from .embeddings import Collection +from .parts import ( + Message, + assistant, + system, + tool_message, + user, +) +from .plugins import load_plugins, pm from .templates import Template -from .plugins import pm, load_plugins -import click -from typing import Any, Dict, List, Optional, Callable, Type, Union -import inspect -import json -import os -import pathlib -import struct +from .utils import Fragment, schema_dsl __all__ = [ "AsyncConversation", @@ -47,27 +60,35 @@ "Collection", "Conversation", "Fragment", - "get_async_model", - "get_key", - "get_model", - "hookimpl", "KeyModel", + "Message", "Model", "ModelError", "NeedsKeyException", "Options", + "PauseChain", "Prompt", "Response", + "ServerSideTool", "Template", "Tool", - "Toolbox", "ToolCall", "ToolOutput", "ToolResult", - "user_dir", + "Toolbox", + "Usage", + "assistant", + "get_async_model", + "get_key", + "get_model", + "hookimpl", "schema_dsl", + "system", + "tool_message", + "user", + "user_dir", ] -DEFAULT_MODEL = "gpt-4o-mini" +DEFAULT_MODEL = "gpt-5.6-luna" def get_plugins(all=False): @@ -90,12 +111,12 @@ def get_plugins(all=False): return plugins -def get_models_with_aliases() -> List["ModelWithAliases"]: +def get_models_with_aliases() -> list["ModelWithAliases"]: model_aliases = [] # Include aliases from aliases.json aliases_path = user_dir() / "aliases.json" - extra_model_aliases: Dict[str, list] = {} + extra_model_aliases: dict[str, list] = {} if aliases_path.exists(): configured_aliases = json.loads(aliases_path.read_text()) for alias, model_id in configured_aliases.items(): @@ -108,12 +129,12 @@ def register(model, async_model=None, aliases=None): model_aliases.append(ModelWithAliases(model, async_model, alias_list)) load_plugins() - pm.hook.register_models(register=register) + pm.hook.register_models(register=register, model_aliases=model_aliases) return model_aliases -def _get_loaders(hook_method) -> Dict[str, Callable]: +def _get_loaders(hook_method) -> dict[str, Callable]: load_plugins() loaders = {} @@ -129,32 +150,32 @@ def register(prefix, loader): return loaders -def get_template_loaders() -> Dict[str, Callable[[str], Template]]: +def get_template_loaders() -> dict[str, Callable[[str], Template]]: """Get template loaders registered by plugins.""" return _get_loaders(pm.hook.register_template_loaders) -def get_fragment_loaders() -> Dict[ +def get_fragment_loaders() -> dict[ str, - Callable[[str], Union[Fragment, Attachment, List[Union[Fragment, Attachment]]]], + Callable[[str], Fragment | Attachment | list[Fragment | Attachment]], ]: """Get fragment loaders registered by plugins.""" return _get_loaders(pm.hook.register_fragment_loaders) -def get_tools() -> Dict[str, Union[Tool, Type[Toolbox]]]: +def get_tools() -> dict[str, Tool | type[Toolbox]]: """Return all tools (llm.Tool and llm.Toolbox) registered by plugins.""" load_plugins() - tools: Dict[str, Union[Tool, Type[Toolbox]]] = {} + tools: dict[str, Tool | type[Toolbox]] = {} # Variable to track current plugin name current_plugin_name = None def register( - tool_or_function: Union[Tool, Type[Toolbox], Callable[..., Any]], - name: Optional[str] = None, + tool_or_function: Tool | type[Toolbox] | Callable[..., Any], + name: str | None = None, ) -> None: - tool: Union[Tool, Type[Toolbox], None] = None + tool: Tool | type[Toolbox] | None = None # If it's a Toolbox class, set the plugin field on it if inspect.isclass(tool_or_function): @@ -165,9 +186,7 @@ def register( tool.name = name or tool.__name__ else: raise TypeError( - "Toolbox classes must inherit from llm.Toolbox, {} does not.".format( - tool_or_function.__name__ - ) + f"Toolbox classes must inherit from llm.Toolbox, {tool_or_function.__name__} does not." ) # If it's already a Tool instance, use it directly @@ -215,12 +234,12 @@ def register( return tools -def get_embedding_models_with_aliases() -> List["EmbeddingModelWithAliases"]: +def get_embedding_models_with_aliases() -> list["EmbeddingModelWithAliases"]: model_aliases = [] # Include aliases from aliases.json aliases_path = user_dir() / "aliases.json" - extra_model_aliases: Dict[str, list] = {} + extra_model_aliases: dict[str, list] = {} if aliases_path.exists(): configured_aliases = json.loads(aliases_path.read_text()) for alias, model_id in configured_aliases.items(): @@ -257,7 +276,7 @@ def get_embedding_model(name): raise UnknownModelError("Unknown model: " + str(name)) -def get_embedding_model_aliases() -> Dict[str, EmbeddingModel]: +def get_embedding_model_aliases() -> dict[str, EmbeddingModel]: model_aliases = {} for model_with_aliases in get_embedding_models_with_aliases(): for alias in model_with_aliases.aliases: @@ -266,7 +285,7 @@ def get_embedding_model_aliases() -> Dict[str, EmbeddingModel]: return model_aliases -def get_async_model_aliases() -> Dict[str, AsyncModel]: +def get_async_model_aliases() -> dict[str, AsyncModel]: async_model_aliases = {} for model_with_aliases in get_models_with_aliases(): if model_with_aliases.async_model: @@ -278,7 +297,7 @@ def get_async_model_aliases() -> Dict[str, AsyncModel]: return async_model_aliases -def get_model_aliases() -> Dict[str, Model]: +def get_model_aliases() -> dict[str, Model]: model_aliases = {} for model_with_aliases in get_models_with_aliases(): if model_with_aliases.model: @@ -292,19 +311,19 @@ class UnknownModelError(KeyError): pass -def get_models() -> List[Model]: +def get_models() -> list[Model]: "Get all registered models" models_with_aliases = get_models_with_aliases() return [mwa.model for mwa in models_with_aliases if mwa.model] -def get_async_models() -> List[AsyncModel]: +def get_async_models() -> list[AsyncModel]: "Get all registered async models" models_with_aliases = get_models_with_aliases() return [mwa.async_model for mwa in models_with_aliases if mwa.async_model] -def get_async_model(name: Optional[str] = None) -> AsyncModel: +def get_async_model(name: str | None = None) -> AsyncModel: "Get an async model by name or alias" aliases = get_async_model_aliases() name = name or get_default_model() @@ -323,7 +342,7 @@ def get_async_model(name: Optional[str] = None) -> AsyncModel: raise UnknownModelError("Unknown model: " + name) -def get_model(name: Optional[str] = None, _skip_async: bool = False) -> Model: +def get_model(name: str | None = None, _skip_async: bool = False) -> Model: "Get a model by name or alias" aliases = get_model_aliases() name = name or get_default_model() @@ -345,14 +364,14 @@ def get_model(name: Optional[str] = None, _skip_async: bool = False) -> Model: def get_key( - explicit_key: Optional[str] = None, - key_alias: Optional[str] = None, - env_var: Optional[str] = None, + explicit_key: str | None = None, + key_alias: str | None = None, + env_var: str | None = None, *, - alias: Optional[str] = None, - env: Optional[str] = None, - input: Optional[str] = None, -) -> Optional[str]: + alias: str | None = None, + env: str | None = None, + input: str | None = None, +) -> str | None: """ Return an API key based on a hierarchy of potential sources. You should use the keyword arguments, the positional arguments are here purely for backwards-compatibility with older code. @@ -443,7 +462,7 @@ def remove_alias(alias): except json.decoder.JSONDecodeError: raise KeyError("aliases.json file is not valid JSON") if alias not in current: - raise KeyError("No such alias: {}".format(alias)) + raise KeyError(f"No such alias: {alias}") del current[alias] path.write_text(json.dumps(current, indent=4) + "\n") diff --git a/llm/cli.py b/llm/cli.py index fc6fb41b1..a9968160d 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1,50 +1,77 @@ import asyncio -import click -from click_default_group import DefaultGroup -from dataclasses import asdict -from importlib.metadata import version +import base64 +import inspect import io import json import os +import pathlib +import re +import readline +import shutil +import sqlite3 +import sys +import textwrap +import warnings +from collections.abc import Iterable +from dataclasses import asdict +from importlib.metadata import version +from runpy import run_module +from typing import Any, cast + +import click +import httpx +import pydantic +import sqlite_utils +import yaml +from click_default_group import DefaultGroup +from sqlite_utils.utils import Format, rows_from_file + from llm import ( - Attachment, AsyncConversation, AsyncKeyModel, AsyncResponse, + Attachment, CancelToolCall, Collection, Conversation, Fragment, + KeyModel, Response, + ServerSideTool, Template, Tool, Toolbox, UnknownModelError, - KeyModel, encode, get_async_model, - get_default_model, get_default_embedding_model, - get_embedding_models_with_aliases, - get_embedding_model_aliases, + get_default_model, get_embedding_model, - get_plugins, - get_tools, + get_embedding_model_aliases, + get_embedding_models_with_aliases, get_fragment_loaders, - get_template_loaders, get_model, get_model_aliases, get_models_with_aliases, - user_dir, + get_plugins, + get_template_loaders, + get_tools, + remove_alias, set_alias, - set_default_model, set_default_embedding_model, - remove_alias, + set_default_model, + user_dir, ) -from llm.models import _BaseConversation, ChainResponse +from llm.models import ChainResponse, _BaseChainResponse, _BaseConversation +from .logs import ( + LogStore, + legacy_log_row_extras, + log_row_extras, + merged_log_rows, +) from .migrations import migrate -from .plugins import pm, load_plugins +from .plugins import load_plugins, pm from .utils import ( ensure_fragment, extract_fenced_code_block, @@ -63,22 +90,6 @@ token_usage_string, truncate_string, ) -import base64 -import httpx -import inspect -import pathlib -import pydantic -import re -import readline -from runpy import run_module -import shutil -import sqlite_utils -from sqlite_utils.utils import rows_from_file, Format -import sys -import textwrap -from typing import cast, Dict, Optional, Iterable, List, Union, Tuple, Type, Any -import warnings -import yaml warnings.simplefilter("ignore", ResourceWarning) @@ -89,6 +100,125 @@ class FragmentNotFound(Exception): pass +def display_stream_events(events, *, show_reasoning=True): + """Consume a sync iterator of StreamEvents and write them. + + Text events go to stdout. Reasoning events go to stderr in dim style. + A newline is written to stderr at each reasoning→text transition so + the assistant text starts on a fresh visual line. + """ + was_reasoning = False + for event in events: + if event.type == "text": + if was_reasoning and show_reasoning: + click.echo("", err=True) + was_reasoning = False + click.echo(event.chunk, nl=False) + elif event.type == "reasoning" and show_reasoning: + was_reasoning = True + click.echo(click.style(event.chunk, dim=True), nl=False, err=True) + + +async def display_async_stream_events(events, *, show_reasoning=True): + """Async counterpart of display_stream_events.""" + was_reasoning = False + async for event in events: + if event.type == "text": + if was_reasoning and show_reasoning: + click.echo("", err=True) + was_reasoning = False + click.echo(event.chunk, nl=False) + elif event.type == "reasoning" and show_reasoning: + was_reasoning = True + click.echo(click.style(event.chunk, dim=True), nl=False, err=True) + + +def _run_chat( + model_label, + prompt_callback, + *, + db=None, + initial_fragments=None, + initial_attachments=None, + transform_prompt=None, + after_response=None, + show_reasoning=True, +): + """Run the terminal chat loop shared by managed and transient models.""" + click.echo(f"Chatting with {model_label}") + click.echo("Type 'exit' or 'quit' to exit") + click.echo("Type '!multi' to enter multiple lines, then '!end' to finish") + click.echo("Type '!edit' to open your default editor and modify the prompt") + if db is not None: + click.echo( + "Type '!fragment [ ...]' to insert one or more fragments" + ) + + argument_fragments = list(initial_fragments or []) + argument_attachments = list(initial_attachments or []) + in_multi = False + accumulated = [] + accumulated_fragments = [] + accumulated_attachments = [] + end_token = "!end" + + while True: + prompt = click.prompt("", prompt_suffix="> " if not in_multi else "") + fragments = [] + attachments = [] + if argument_fragments: + fragments += argument_fragments + # Fragments from command options are added to the first message only. + argument_fragments = [] + if argument_attachments: + attachments = argument_attachments + argument_attachments = [] + if prompt.strip().startswith("!multi"): + in_multi = True + bits = prompt.strip().split() + if len(bits) > 1: + end_token = "!end {}".format(" ".join(bits[1:])) + continue + if prompt.strip() == "!edit": + edited_prompt = click.edit() + if edited_prompt is None: + click.echo("Editor closed without saving.", err=True) + continue + prompt = edited_prompt.strip() + if db is not None and prompt.strip().startswith("!fragment "): + prompt, fragments, attachments = process_fragments_in_chat(db, prompt) + + if in_multi: + if prompt.strip() == end_token: + prompt = "\n".join(accumulated) + fragments = accumulated_fragments + attachments = accumulated_attachments + in_multi = False + accumulated = [] + accumulated_fragments = [] + accumulated_attachments = [] + else: + if prompt: + accumulated.append(prompt) + accumulated_fragments += fragments + accumulated_attachments += attachments + continue + + if prompt.strip() in ("exit", "quit"): + break + if transform_prompt is not None: + prompt = transform_prompt(prompt) + + response = prompt_callback(prompt, fragments, attachments) + display_stream_events( + response.stream_events(), + show_reasoning=show_reasoning, + ) + if after_response is not None: + after_response(response) + print() + + def validate_fragment_alias(ctx, param, value): if not re.match(r"^[a-zA-Z0-9_-]+$", value): raise click.BadParameter("Fragment alias must be alphanumeric") @@ -97,12 +227,12 @@ def validate_fragment_alias(ctx, param, value): def resolve_fragments( db: sqlite_utils.Database, fragments: Iterable[str], allow_attachments: bool = False -) -> List[Union[Fragment, Attachment]]: +) -> list[Fragment | Attachment]: """ Resolve fragment strings into a mixed of llm.Fragment() and llm.Attachment() objects. """ - def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: + def _load_by_alias(fragment: str) -> tuple[str | None, str | None]: rows = list( db.query( """ @@ -119,9 +249,9 @@ def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: return None, None # The fragment strings could be URLs or paths or plugin references - resolved: List[Union[Fragment, Attachment]] = [] + resolved: list[Fragment | Attachment] = [] for fragment in fragments: - if fragment.startswith("http://") or fragment.startswith("https://"): + if fragment.startswith(("http://", "https://")): llm_version = version("llm") headers = {"User-Agent": f"llm/{llm_version} (https://llm.datasette.io/)"} client = httpx.Client( @@ -132,11 +262,11 @@ def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: resolved.append(Fragment(response.text, fragment)) elif fragment == "-": resolved.append(Fragment(sys.stdin.read(), "-")) - elif has_plugin_prefix(fragment): + elif has_plugin_prefix(fragment) and not pathlib.Path(fragment).exists(): prefix, rest = fragment.split(":", 1) loaders = get_fragment_loaders() if prefix not in loaders: - raise FragmentNotFound("Unknown fragment prefix: {}".format(prefix)) + raise FragmentNotFound(f"Unknown fragment prefix: {prefix}") loader = loaders[prefix] try: result = loader(rest) @@ -146,15 +276,11 @@ def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: isinstance(r, Attachment) for r in result ): raise FragmentNotFound( - "Fragment loader {} returned a disallowed attachment".format( - prefix - ) + f"Fragment loader {prefix} returned a disallowed attachment" ) resolved.extend(result) - except Exception as ex: - raise FragmentNotFound( - "Could not load fragment {}: {}".format(fragment, ex) - ) + except Exception as ex: # noqa: BLE001 + raise FragmentNotFound(f"Could not load fragment {fragment}: {ex}") else: # Try from the DB content, source = _load_by_alias(fragment) @@ -206,8 +332,6 @@ def process_fragments_in_chat( class AttachmentError(Exception): """Exception raised for errors in attachment resolution.""" - pass - def resolve_attachment(value): """ @@ -277,13 +401,64 @@ def resolve_attachment_with_type(value: str, mimetype: str) -> Attachment: return attachment -def attachment_types_callback(ctx, param, values) -> List[Attachment]: +def attachment_types_callback(ctx, param, values) -> list[Attachment]: collected = [] for value, mimetype in values: collected.append(resolve_attachment_with_type(value, mimetype)) return collected +def _apply_template(template, prompt, params, system): + """Apply a loaded template to a prompt and system prompt.""" + try: + uses_input = "input" in template.vars() + input_ = prompt if uses_input else "" + template_prompt, template_system = template.evaluate(input_, params) + except Template.MissingVariables as ex: + raise click.ClickException(str(ex)) + if template_system and not system: + system = template_system + if template_prompt: + if prompt and not uses_input: + prompt = f"{template_prompt}\n{prompt}" + else: + prompt = template_prompt + return prompt, system + + +def _merge_template_options(template, options): + """Add template options unless the same option was provided explicitly.""" + merged_options = list(options) + specified_options = dict(merged_options) + for option_name, option_value in (template.options or {}).items(): + if option_name not in specified_options: + merged_options.append((option_name, option_value)) + return merged_options + + +def _merge_template_attachments(template, attachments, attachment_types): + """Resolve and prepend attachments declared by a loaded template.""" + if template.attachments: + attachments = [ + resolve_attachment(value) for value in template.attachments + ] + list(attachments) + if template.attachment_types: + attachment_types = [ + resolve_attachment_with_type(item.value, item.type) + for item in template.attachment_types + ] + list(attachment_types) + return attachments, attachment_types + + +def _merge_template_tools(template, tools, python_tools): + """Prepend trusted tool definitions declared by a loaded template.""" + if template.tools: + tools = [*template.tools, *tools] + if template.functions and template._functions_is_trusted: + python_tools = [template.functions, *python_tools] + return tools, python_tools + + def json_validator(object_name): def validator(ctx, param, value): if value is None: @@ -308,6 +483,54 @@ def schema_option(fn): return fn +def tool_options(fn): + """Add the shared CLI options for selecting and executing tools.""" + decorators = ( + click.option( + "tools", + "-T", + "--tool", + multiple=True, + help="Name of a tool to make available to the model", + ), + click.option( + "python_tools", + "--functions", + multiple=True, + help="Python code block or file path defining functions to register as tools", + ), + click.option( + "tools_debug", + "--td", + "--tools-debug", + is_flag=True, + help="Show full details of tool executions", + envvar="LLM_TOOLS_DEBUG", + ), + click.option( + "tools_approve", + "--ta", + "--tools-approve", + is_flag=True, + help="Manually approve every tool execution", + ), + click.option( + "chain_limit", + "--cl", + "--chain-limit", + type=int, + default=5, + help=( + "How many chained tool responses to allow, " + "default 5, set 0 for unlimited" + ), + ), + ) + for decorator in reversed(decorators): + fn = decorator(fn) + return fn + + @click.group( cls=DefaultGroup, default="prompt", @@ -376,42 +599,7 @@ def cli(): callback=attachment_types_callback, help="\b\nAttachment with explicit mimetype,\n--at image.jpg image/jpeg", ) -@click.option( - "tools", - "-T", - "--tool", - multiple=True, - help="Name of a tool to make available to the model", -) -@click.option( - "python_tools", - "--functions", - help="Python code block or file path defining functions to register as tools", - multiple=True, -) -@click.option( - "tools_debug", - "--td", - "--tools-debug", - is_flag=True, - help="Show full details of tool executions", - envvar="LLM_TOOLS_DEBUG", -) -@click.option( - "tools_approve", - "--ta", - "--tools-approve", - is_flag=True, - help="Manually approve every tool execution", -) -@click.option( - "chain_limit", - "--cl", - "--chain-limit", - type=int, - default=5, - help="How many chained tool responses to allow, default 5, set 0 for unlimited", -) +@tool_options @click.option( "options", "-o", @@ -420,6 +608,12 @@ def cli(): multiple=True, help="key/value options for the model", ) +@click.option( + "show_model_options", + "--options", + is_flag=True, + help="Show options for the selected model", +) @schema_option @click.option( "--schema-multi", @@ -450,6 +644,7 @@ def cli(): @click.option("--no-stream", is_flag=True, help="Do not stream output") @click.option("-n", "--no-log", is_flag=True, help="Don't log to database") @click.option("--log", is_flag=True, help="Log prompt and response to the database") +@click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") @click.option( "_continue", "-c", @@ -476,6 +671,12 @@ def cli(): is_flag=True, help="Extract last fenced code block", ) +@click.option( + "json_output", + "--json", + is_flag=True, + help="Output the response as JSON, same format as llm logs --json", +) def prompt( prompt, system, @@ -490,6 +691,7 @@ def prompt( tools_approve, chain_limit, options, + show_model_options, schema_input, schema_multi, fragments, @@ -499,6 +701,7 @@ def prompt( no_stream, no_log, log, + hide_reasoning, _continue, conversation_id, key, @@ -507,6 +710,7 @@ def prompt( usage, extract, extract_last, + json_output, ): """ Execute a prompt @@ -517,7 +721,7 @@ def prompt( \b llm 'Capital of France?' - llm 'Capital of France?' -m gpt-4o + llm 'Capital of France?' -m gpt-5.5 llm 'Capital of France?' -s 'answer in Spanish' Multi-modal models can be called with attachments like this: @@ -538,11 +742,6 @@ def prompt( if log and no_log: raise click.ClickException("--log and --no-log are mutually exclusive") - log_path = pathlib.Path(database) if database else logs_db_path() - (log_path.parent).mkdir(parents=True, exist_ok=True) - db = sqlite_utils.Database(log_path) - migrate(db) - if queries and not model_id: # Use -q options to find model with shortest model_id matches = [] @@ -555,6 +754,23 @@ def prompt( ) model_id = min(matches, key=len) + if show_model_options and not (conversation_id or _continue or template): + model_id = model_id or get_default_model() + try: + if async_: + get_async_model(model_id) + else: + get_model(model_id) + except UnknownModelError as ex: + raise click.ClickException(ex) + click.echo(render_model_with_options(model_id, async_=async_)) + return + + log_path = pathlib.Path(database) if database else logs_db_path() + (log_path.parent).mkdir(parents=True, exist_ok=True) + db = sqlite_utils.Database(log_path) + migrate(db) + if schema_multi: schema_input = schema_multi @@ -613,7 +829,7 @@ def read_prompt(): try: to_save["model"] = model_aliases[model_id].model_id except KeyError: - raise click.ClickException("'{}' is not a known model".format(model_id)) + raise click.ClickException(f"'{model_id}' is not a known model") prompt = read_prompt() if prompt: to_save["prompt"] = prompt @@ -677,8 +893,9 @@ def read_prompt(): template_obj = load_template(template) except LoadTemplateError as ex: raise click.ClickException(str(ex)) - extract = template_obj.extract - extract_last = template_obj.extract_last + if not (extract or extract_last): + extract = template_obj.extract + extract_last = template_obj.extract_last # Combine with template fragments/system_fragments if template_obj.fragments: fragments = [*template_obj.fragments, *fragments] @@ -686,46 +903,18 @@ def read_prompt(): system_fragments = [*template_obj.system_fragments, *system_fragments] if template_obj.schema_object: schema = template_obj.schema_object - if template_obj.tools: - tools = [*template_obj.tools, *tools] - if template_obj.functions and template_obj._functions_is_trusted: - python_tools = [template_obj.functions, *python_tools] - input_ = "" + tools, python_tools = _merge_template_tools(template_obj, tools, python_tools) if template_obj.options: - # Make options mutable (they start as a tuple) - options = list(options) - # Load any options, provided they were not set using -o already - specified_options = dict(options) - for option_name, option_value in template_obj.options.items(): - if option_name not in specified_options: - options.append((option_name, option_value)) + options = _merge_template_options(template_obj, options) if "input" in template_obj.vars(): - input_ = read_prompt() - try: - template_prompt, template_system = template_obj.evaluate(input_, params) - if template_prompt: - # Combine with user prompt - if prompt and "input" not in template_obj.vars(): - prompt = template_prompt + "\n" + prompt - else: - prompt = template_prompt - if template_system and not system: - system = template_system - except Template.MissingVariables as ex: - raise click.ClickException(str(ex)) + prompt = read_prompt() + prompt, system = _apply_template(template_obj, prompt, params, system) if model_id is None and template_obj.model: model_id = template_obj.model - # Merge in any attachments - if template_obj.attachments: - attachments = [ - resolve_attachment(a) for a in template_obj.attachments - ] + list(attachments) - if template_obj.attachment_types: - attachment_types = [ - resolve_attachment_with_type(at.value, at.type) - for at in template_obj.attachment_types - ] + list(attachment_types) - if extract or extract_last: + attachments, attachment_types = _merge_template_attachments( + template_obj, attachments, attachment_types + ) + if extract or extract_last or json_output: no_stream = True conversation = None @@ -757,7 +946,15 @@ def read_prompt(): except UnknownModelError as ex: raise click.ClickException(ex) - if conversation is None and (tools or python_tools): + if show_model_options: + click.echo(render_model_with_options(model_id, async_=async_)) + return + + if conversation is None: + # Always work through a conversation, even for a one-off prompt. + # The legacy logger invents one anyway and throws the id away; + # creating it here means both writers agree on which conversation + # (and so which thread) this response belongs to. conversation = model.conversation() if conversation: @@ -769,11 +966,11 @@ def read_prompt(): if options: # Validate with pydantic try: - validated_options = dict( - (key, value) + validated_options = { + key: value for key, value in model.Options(**dict(options)) if value is not None - ) + } except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) @@ -819,21 +1016,20 @@ def read_prompt(): if conversation: prompt_method = conversation.prompt - tool_implementations = _gather_tools(tools, python_tools) - - if tool_implementations: + tool_kwargs = _tool_chain_kwargs( + tools, python_tools, tools_debug, tools_approve, chain_limit, model=model + ) + if tool_kwargs: prompt_method = conversation.chain kwargs["options"] = validated_options - kwargs["chain_limit"] = chain_limit - if tools_debug: - kwargs["after_call"] = _debug_tool_call - if tools_approve: - kwargs["before_call"] = _approve_tool_call - kwargs["tools"] = tool_implementations + kwargs.update(tool_kwargs) else: # Merge in options for the .prompt() methods kwargs.update(validated_options) + if hide_reasoning: + kwargs["hide_reasoning"] = True + try: if async_: @@ -848,10 +1044,11 @@ async def inner(): system_fragments=resolved_system_fragments, **kwargs, ) - async for chunk in response: - print(chunk, end="") - sys.stdout.flush() - print("") + await display_async_stream_events( + response.astream_events(), + show_reasoning=not hide_reasoning, + ) + print() else: response = prompt_method( prompt, @@ -867,7 +1064,8 @@ async def inner(): text = ( extract_fenced_code_block(text, last=extract_last) or text ) - print(text) + if not json_output: + print(text) return response response = asyncio.run(inner()) @@ -882,15 +1080,17 @@ async def inner(): **kwargs, ) if should_stream: - for chunk in response: - print(chunk, end="") - sys.stdout.flush() - print("") + display_stream_events( + response.stream_events(), + show_reasoning=not hide_reasoning, + ) + print() else: text = response.text() if extract or extract_last: text = extract_fenced_code_block(text, last=extract_last) or text - print(text) + if not json_output: + print(text) # List of exceptions that should never be raised in pytest: except (ValueError, NotImplementedError) as ex: raise click.ClickException(str(ex)) @@ -911,7 +1111,7 @@ async def inner(): # Show token usage to stderr in yellow click.echo( click.style( - "Token usage: {}".format(response_object.token_usage()), + f"Token usage: {response_object.token_usage()}", fg="yellow", bold=True, ), @@ -919,12 +1119,27 @@ async def inner(): ) # Log responses to the database + log_db = None if (logs_on() or log) and not no_log: + log_db = db + elif json_output: + # --json needs logged rows, so use a temporary in-memory database + log_db = sqlite_utils.Database(memory=True) + migrate(log_db) + + if log_db is not None: # Could be Response, AsyncResponse, ChainResponse, AsyncChainResponse if isinstance(response, AsyncResponse): response = asyncio.run(response.to_sync_response()) # At this point ALL forms should have a log_to_db() method that works: - response.log_to_db(db) + response.log_to_db(log_db) + + if json_output: + if isinstance(response, _BaseChainResponse): + response_ids = [response_.id for response_ in response._responses] + else: + response_ids = [response.id] + click.echo(logs_json_for_response_ids(log_db, response_ids)) @cli.command() @@ -981,43 +1196,9 @@ async def inner(): help="Path to log database", ) @click.option("--no-stream", is_flag=True, help="Do not stream output") +@click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") @click.option("--key", help="API key to use") -@click.option( - "tools", - "-T", - "--tool", - multiple=True, - help="Name of a tool to make available to the model", -) -@click.option( - "python_tools", - "--functions", - help="Python code block or file path defining functions to register as tools", - multiple=True, -) -@click.option( - "tools_debug", - "--td", - "--tools-debug", - is_flag=True, - help="Show full details of tool executions", - envvar="LLM_TOOLS_DEBUG", -) -@click.option( - "tools_approve", - "--ta", - "--tools-approve", - is_flag=True, - help="Manually approve every tool execution", -) -@click.option( - "chain_limit", - "--cl", - "--chain-limit", - type=int, - default=5, - help="How many chained tool responses to allow, default 5, set 0 for unlimited", -) +@tool_options def chat( system, model_id, @@ -1029,6 +1210,7 @@ def chat( param, options, no_stream, + hide_reasoning, key, database, tools, @@ -1072,10 +1254,7 @@ def chat( raise click.ClickException(str(ex)) if model_id is None and template_obj.model: model_id = template_obj.model - if template_obj.tools: - tools = [*template_obj.tools, *tools] - if template_obj.functions and template_obj._functions_is_trusted: - python_tools = [template_obj.functions, *python_tools] + tools, python_tools = _merge_template_tools(template_obj, tools, python_tools) # Figure out which model we are using if model_id is None: @@ -1088,7 +1267,7 @@ def chat( try: model = get_model(model_id) except KeyError: - raise click.ClickException("'{}' is not a known model".format(model_id)) + raise click.ClickException(f"'{model_id}' is not a known model") if conversation is None: # Start a fresh conversation for this chat @@ -1097,20 +1276,15 @@ def chat( # Ensure it can see the API key conversation.model = model - if tools_debug: - conversation.after_call = _debug_tool_call - if tools_approve: - conversation.before_call = _approve_tool_call - # Validate options validated_options = get_model_options(model.model_id) if options: try: - validated_options = dict( - (key, value) + validated_options = { + key: value for key, value in model.Options(**dict(options)) if value is not None - ) + } except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) @@ -1118,11 +1292,16 @@ def chat( if validated_options: kwargs["options"] = validated_options - tool_functions = _gather_tools(tools, python_tools) - - if tool_functions: - kwargs["chain_limit"] = chain_limit - kwargs["tools"] = tool_functions + kwargs.update( + _tool_chain_kwargs( + tools, + python_tools, + tools_debug, + tools_approve, + chain_limit, + model=model, + ) + ) should_stream = model.can_stream and not no_stream if not should_stream: @@ -1130,6 +1309,8 @@ def chat( if key and isinstance(model, KeyModel): kwargs["key"] = key + if hide_reasoning: + kwargs["hide_reasoning"] = True try: fragments_and_attachments = resolve_fragments( @@ -1144,83 +1325,19 @@ def chat( attachment for attachment in fragments_and_attachments if isinstance(attachment, Attachment) - ] - argument_system_fragments = resolve_fragments(db, system_fragments) - except FragmentNotFound as ex: - raise click.ClickException(str(ex)) - - click.echo("Chatting with {}".format(model.model_id)) - click.echo("Type 'exit' or 'quit' to exit") - click.echo("Type '!multi' to enter multiple lines, then '!end' to finish") - click.echo("Type '!edit' to open your default editor and modify the prompt") - click.echo( - "Type '!fragment [ ...]' to insert one or more fragments" - ) - in_multi = False - - accumulated = [] - accumulated_fragments = [] - accumulated_attachments = [] - end_token = "!end" - while True: - prompt = click.prompt("", prompt_suffix="> " if not in_multi else "") - fragments = [] - attachments = [] - if argument_fragments: - fragments += argument_fragments - # fragments from --fragments will get added to the first message only - argument_fragments = [] - if argument_attachments: - attachments = argument_attachments - argument_attachments = [] - if prompt.strip().startswith("!multi"): - in_multi = True - bits = prompt.strip().split() - if len(bits) > 1: - end_token = "!end {}".format(" ".join(bits[1:])) - continue - if prompt.strip() == "!edit": - edited_prompt = click.edit() - if edited_prompt is None: - click.echo("Editor closed without saving.", err=True) - continue - prompt = edited_prompt.strip() - if prompt.strip().startswith("!fragment "): - prompt, fragments, attachments = process_fragments_in_chat(db, prompt) - - if in_multi: - if prompt.strip() == end_token: - prompt = "\n".join(accumulated) - fragments = accumulated_fragments - attachments = accumulated_attachments - in_multi = False - accumulated = [] - accumulated_fragments = [] - accumulated_attachments = [] - else: - if prompt: - accumulated.append(prompt) - accumulated_fragments += fragments - accumulated_attachments += attachments - continue + ] + argument_system_fragments = resolve_fragments(db, system_fragments) + except FragmentNotFound as ex: + raise click.ClickException(str(ex)) + + def transform_chat_prompt(prompt): + nonlocal system if template_obj: - try: - # Mirror prompt() logic: only pass input if template uses it - uses_input = "input" in template_obj.vars() - input_ = prompt if uses_input else "" - template_prompt, template_system = template_obj.evaluate(input_, params) - except Template.MissingVariables as ex: - raise click.ClickException(str(ex)) - if template_system and not system: - system = template_system - if template_prompt: - if prompt and not uses_input: - prompt = f"{template_prompt}\n{prompt}" - else: - prompt = template_prompt - if prompt.strip() in ("exit", "quit"): - break + prompt, system = _apply_template(template_obj, prompt, params, system) + return prompt + def execute_chat_prompt(prompt, fragments, attachments): + nonlocal system, argument_system_fragments response = conversation.chain( prompt, fragments=fragments, @@ -1233,24 +1350,39 @@ def chat( # System prompt and system fragments only sent for the first message system = None argument_system_fragments = [] - for chunk in response: - print(chunk, end="") - sys.stdout.flush() - response.log_to_db(db) - print("") + return response + + _run_chat( + model.model_id, + execute_chat_prompt, + db=db, + initial_fragments=argument_fragments, + initial_attachments=argument_attachments, + transform_prompt=transform_chat_prompt, + after_response=lambda response: response.log_to_db(db), + show_reasoning=not hide_reasoning, + ) def load_conversation( - conversation_id: Optional[str], + conversation_id: str | None, async_=False, database=None, -) -> Optional[_BaseConversation]: +) -> _BaseConversation | None: log_path = pathlib.Path(database) if database else logs_db_path() db = sqlite_utils.Database(log_path) migrate(db) if conversation_id is None: - # Return the most recent conversation, or None if there are none - matches = list(db["conversations"].rows_where(order_by="id desc", limit=1)) + # Most recent conversation from either generation of tables - + # thread ids are conversation ids, so the union dedupes rows + # from the dual-write era. + matches = list(db.query(""" + select id from ( + select id from threads + union + select id from conversations + ) order by id desc limit 1 + """)) if matches: conversation_id = matches[0]["id"] else: @@ -1258,17 +1390,104 @@ def load_conversation( try: row = cast(sqlite_utils.db.Table, db["conversations"]).get(conversation_id) except sqlite_utils.db.NotFoundError: - raise click.ClickException( - "No conversation found with id={}".format(conversation_id) + # No legacy record - reconstruct the equivalent from the thread + # and its most recent turn's model. + try: + thread_row = cast(sqlite_utils.db.Table, db["threads"]).get(conversation_id) + except sqlite_utils.db.NotFoundError: + raise click.ClickException( + f"No conversation found with id={conversation_id}" + ) + model_match = next( + db.query( + "select model from turns where thread_id = ? order by id desc limit 1", + [conversation_id], + ), + None, ) + if model_match is None: + raise click.ClickException( + f"No conversation found with id={conversation_id}" + ) + row = { + "id": conversation_id, + "name": thread_row["name"], + "model": model_match["model"], + } # Inflate that conversation conversation_class = AsyncConversation if async_ else Conversation response_class = AsyncResponse if async_ else Response conversation = conversation_class.from_row(row) for response in db["responses"].rows_where( - "conversation_id = ?", [conversation_id] + "conversation_id = ?", [conversation_id], order_by="id" + ): + response_obj = response_class.from_row(db, response) + if conversation.responses: + previous_response = conversation.responses[-1] + # SQLite rows store each response's legacy current-turn inputs + # (prompt text, attachments, tool_results), not the full + # prompt.messages chain. Rebuild that chain here so follow-up + # prompts via `llm -c` satisfy the Prompt.messages invariant. + response_obj.prompt._explicit_messages = ( + list(previous_response.prompt.messages) + + list(previous_response._messages_now()) + + list(response_obj.prompt.messages) + ) + conversation.responses.append(response_obj) + + # If this conversation has a thread in the content-addressed tables, + # take the history from there. That chain is the exact message list + # that was sent and returned, so reasoning signatures and provider + # metadata survive - unlike the rebuild above, which can only work + # from the flattened legacy columns. + try: + conversation.loaded_messages = LogStore(db).thread_messages(conversation_id) + except KeyError: + pass + + # Plugin and server-side tools recorded against the first turn, for + # the same reuse-on-continue behaviour the rebuilt responses provide. + # Configured instances are collapsed into a single spec string like + # Datasette({"url": "..."}) - the same format -T accepts - so the + # instance can be reconstructed with its configuration. + loaded_tools = [] + seen_instance_ids = set() + supported_server_side_tool_names = { + tool_class.__name__ + for tool_class in conversation.model.supported_server_side_tools + } + for tool_row in db.query( + """ + select tools.name, tools.plugin, turn_tools.instance_id, + tool_instances.name as instance_name, + tool_instances.arguments as instance_arguments + from tools + join turn_tools on turn_tools.tool_id = tools.id + left join tool_instances on tool_instances.id = turn_tools.instance_id + where turn_tools.turn_id = ( + select id from turns where thread_id = ? order by id limit 1 + ) + """, + [conversation_id], ): - conversation.responses.append(response_class.from_row(db, response)) + if ( + tool_row["plugin"] is None + and tool_row["instance_name"] not in supported_server_side_tool_names + ): + continue + if tool_row["instance_id"] is None: + loaded_tools.append(tool_row["name"]) + elif tool_row["instance_id"] not in seen_instance_ids: + seen_instance_ids.add(tool_row["instance_id"]) + arguments = tool_row["instance_arguments"] + if arguments and arguments != "{}": + loaded_tools.append( + "{}({})".format(tool_row["instance_name"], arguments) + ) + else: + loaded_tools.append(tool_row["instance_name"]) + conversation.loaded_tools = loaded_tools + return conversation @@ -1318,7 +1537,7 @@ def keys_get(name): try: click.echo(keys[name]) except KeyError: - raise click.ClickException("No key found with name '{}'".format(name)) + raise click.ClickException(f"No key found with name '{name}'") @keys.command(name="set") @@ -1368,7 +1587,7 @@ def logs_status(): "Show current status of database logging" path = logs_db_path() if not path.exists(): - click.echo("No log database found at {}".format(path)) + click.echo(f"No log database found at {path}") return if logs_on(): click.echo("Logging is ON for all prompts".format()) @@ -1376,12 +1595,15 @@ def logs_status(): click.echo("Logging is OFF".format()) db = sqlite_utils.Database(path) migrate(db) - click.echo("Found log database at {}".format(path)) - click.echo("Number of conversations logged:\t{}".format(db["conversations"].count)) - click.echo("Number of responses logged:\t{}".format(db["responses"].count)) - click.echo( - "Database file size: \t\t{}".format(_human_readable_size(path.stat().st_size)) - ) + click.echo(f"Found log database at {path}") + click.echo("Number of threads logged:\t{}".format(db["threads"].count)) + click.echo("Number of turns logged:\t\t{}".format(db["turns"].count)) + legacy_conversations = db["conversations"].count + legacy_responses = db["responses"].count + if legacy_conversations or legacy_responses: + click.echo(f"Number of legacy conversations:\t{legacy_conversations}") + click.echo(f"Number of legacy responses:\t{legacy_responses}") + click.echo(f"Database file size: \t\t{_human_readable_size(path.stat().st_size)}") @logs.command(name="backup") @@ -1393,11 +1615,9 @@ def backup(path): db = sqlite_utils.Database(logs_path) try: db.execute("vacuum into ?", [str(path)]) - except Exception as ex: + except Exception as ex: # noqa: BLE001 raise click.ClickException(str(ex)) - click.echo( - "Backed up {} to {}".format(_human_readable_size(path.stat().st_size), path) - ) + click.echo(f"Backed up {_human_readable_size(path.stat().st_size)} to {path}") @logs.command(name="on") @@ -1415,60 +1635,97 @@ def logs_turn_off(): path.touch() -LOGS_COLUMNS = """ responses.id, - responses.model, - responses.resolved_model, - responses.prompt, - responses.system, - responses.prompt_json, - responses.options_json, - responses.response, - responses.response_json, - responses.conversation_id, - responses.duration_ms, - responses.datetime_utc, - responses.input_tokens, - responses.output_tokens, - responses.token_details, - conversations.name as conversation_name, - conversations.model as conversation_model, - schemas.content as schema_json""" - -LOGS_SQL = """ -select -{columns} -from - responses -left join schemas on responses.schema_id = schemas.id -left join conversations on responses.conversation_id = conversations.id{extra_where} -order by {order_by}{limit} -""" -LOGS_SQL_SEARCH = """ -select -{columns} -from - responses -left join schemas on responses.schema_id = schemas.id -left join conversations on responses.conversation_id = conversations.id -join responses_fts on responses_fts.rowid = responses.rowid -where responses_fts match :query{extra_where} -order by {order_by}{limit} -""" - -ATTACHMENTS_SQL = """ -select - response_id, - attachments.id, - attachments.type, - attachments.path, - attachments.url, - length(attachments.content) as content_length -from attachments -join prompt_attachments - on attachments.id = prompt_attachments.attachment_id -where prompt_attachments.response_id in ({}) -order by prompt_attachments."order" -""" +def annotate_log_rows(db, rows, expand=False, truncate=False): + """ + Modify log rows from the merged reader in place: attach fragments + and tool information, decode (or, if truncate is on, remove) their + JSON columns and strip the reader's internal keys. + + Returns a dict mapping row id to its attachments, for + log_rows_as_json and the rendered output. + """ + store = LogStore(db) + # New rows carry their extras in the row's parts; legacy rows + # batch-fetch from the legacy tables. + legacy_extras = legacy_log_row_extras( + db, [row["id"] for row in rows if row.get("_legacy")] + ) + extras_by_id = { + row["id"]: ( + legacy_extras[row["id"]] + if row.get("_legacy") + else log_row_extras(store, row) + ) + for row in rows + } + for row in rows: + for internal in ( + "_input_parts", + "_output_parts", + "_parent_message_hash", + "_input_message_hashes", + "_tip_message_hash", + "_legacy", + "_search_rank", + ): + row.pop(internal, None) + extras = extras_by_id[row["id"]] + if truncate: + row["prompt"] = truncate_string(row["prompt"] or "") + row["response"] = truncate_string(row["response"] or "") + # Add prompt and system fragments + for key in ("prompt_fragments", "system_fragments"): + row[key] = [ + { + "hash": fragment["hash"], + "content": ( + fragment["content"] + if expand + else truncate_string(fragment["content"]) + ), + "aliases": json.loads(fragment["aliases"]), + } + for fragment in extras[key] + ] + # Either decode or remove all JSON keys + keys = list(row.keys()) + for key in keys: + if key.endswith("_json") and row[key] is not None: + if truncate: + del row[key] + else: + row[key] = json.loads(row[key]) + row.update( + { + "tools": extras["tools"], + "tool_calls": extras["tool_calls"], + "tool_results": extras["tool_results"], + } + ) + return {id: extras["attachments"] for id, extras in extras_by_id.items()} + + +def log_rows_as_json(rows, attachments_by_id): + "Serialize annotated log rows to the JSON used by 'llm logs --json'" + for row in rows: + row["attachments"] = [ + {k: v for k, v in attachment.items() if k != "response_id"} + for attachment in attachments_by_id.get(row["id"], []) + ] + return json.dumps(list(rows), indent=2) + + +def logs_json_for_response_ids(db, ids): + """ + Return the JSON that 'llm logs --json' would output for these response IDs, + in chronological order + """ + if not ids: + return "[]" + rows = merged_log_rows(LogStore(db), ids=list(ids)) + # Newest first out of the reader, chronological out here + rows.reverse() + return log_rows_as_json(rows, annotate_log_rows(db, rows)) @logs.command(name="list") @@ -1606,7 +1863,7 @@ def logs_list( path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): - raise click.ClickException("No log database found at {}".format(path)) + raise click.ClickException(f"No log database found at {path}") db = sqlite_utils.Database(path) migrate(db) @@ -1624,18 +1881,22 @@ def logs_list( if flag[1] ] ) - raise click.ClickException("Cannot use --short and {} together".format(invalid)) + raise click.ClickException(f"Cannot use --short and {invalid} together") if response and not current_conversation and not conversation_id: current_conversation = True if current_conversation: try: - conversation_id = next( - db.query( - "select conversation_id from responses order by id desc limit 1" - ) - )["conversation_id"] + # Thread ids are conversation ids and both id spaces are + # ULIDs, so the most recent of either world wins. + conversation_id = next(db.query(""" + select conversation_id from ( + select thread_id as conversation_id, id from turns + union all + select conversation_id, id from responses + ) order by id desc limit 1 + """))["conversation_id"] except StopIteration: # No conversations yet raise click.ClickException("No conversations found") @@ -1656,165 +1917,41 @@ def logs_list( # Maybe they uninstalled a model, use the -m option as-is model_id = model - sql = LOGS_SQL - order_by = "responses.id desc" - if query: - sql = LOGS_SQL_SEARCH - if not latest: - order_by = "responses_fts.rank desc" - - limit = "" - if count is not None and count > 0: - limit = " limit {}".format(count) - - sql_format = { - "limit": limit, - "columns": LOGS_COLUMNS, - "extra_where": "", - "order_by": order_by, - } - where_bits = [] - sql_params = { - "model": model_id, - "query": query, - "conversation_id": conversation_id, - "id_gt": id_gt, - "id_gte": id_gte, - } - if model_id: - where_bits.append("responses.model = :model") - if conversation_id: - where_bits.append("responses.conversation_id = :conversation_id") - if id_gt: - where_bits.append("responses.id > :id_gt") - if id_gte: - where_bits.append("responses.id >= :id_gte") - if fragments: - # Resolve the fragments to their hashes - fragment_hashes = [ - fragment.id() for fragment in resolve_fragments(db, fragments) - ] - exists_clauses = [] - - for i, fragment_hash in enumerate(fragment_hashes): - exists_clause = f""" - exists ( - select 1 from prompt_fragments - where prompt_fragments.response_id = responses.id - and prompt_fragments.fragment_id in ( - select fragments.id from fragments - where hash = :f{i} - ) - union - select 1 from system_fragments - where system_fragments.response_id = responses.id - and system_fragments.fragment_id in ( - select fragments.id from fragments - where hash = :f{i} - ) - ) - """ - exists_clauses.append(exists_clause) - sql_params["f{}".format(i)] = fragment_hash - - where_bits.append(" and ".join(exists_clauses)) - - if any_tools: - # Any response that involved at least one tool result - where_bits.append( - """ - exists ( - select 1 - from tool_results - where - tool_results.response_id = responses.id - ) - """ - ) - if tools: - tools_by_name = get_tools() - # Filter responses by tools (must have ALL of the named tools, including plugin) - tool_clauses = [] - for i, tool_name in enumerate(tools): - try: - plugin_name = tools_by_name[tool_name].plugin - except KeyError: - raise click.ClickException(f"Unknown tool: {tool_name}") - - tool_clauses.append( - f""" - exists ( - select 1 - from tool_results - join tools on tools.id = tool_results.tool_id - where tool_results.response_id = responses.id - and tools.name = :tool{i} - and tools.plugin = :plugin{i} - ) - """ - ) - sql_params[f"tool{i}"] = tool_name - sql_params[f"plugin{i}"] = plugin_name - - # AND means “must have all” — use OR instead if you want “any of” - where_bits.append(" and ".join(tool_clauses)) + fragment_hashes = [fragment.id() for fragment in resolve_fragments(db, fragments)] - schema_id = None - if schema: - schema_id = make_schema_id(schema)[0] - where_bits.append("responses.schema_id = :schema_id") - sql_params["schema_id"] = schema_id + schema_id = make_schema_id(schema)[0] if schema else None - if where_bits: - where_ = " and " if query else " where " - sql_format["extra_where"] = where_ + " and ".join(where_bits) - - final_sql = sql.format(**sql_format) - rows = list(db.query(final_sql, sql_params)) + store = LogStore(db) + try: + rows = merged_log_rows( + store, + count=count if count and count > 0 else None, + model_id=model_id, + thread_id=conversation_id, + fragment_hashes=fragment_hashes, + tool_names=tools, + any_tools=any_tools, + schema_id=schema_id, + id_gt=id_gt, + id_gte=id_gte, + query=query, + latest=latest, + ) + except sqlite3.OperationalError as ex: + if query: + # Almost certainly FTS5 syntax - unbalanced quotes, stray + # operators and the like + raise click.ClickException( + f"Invalid search query: {ex} - see the FTS5 query syntax " + "documentation at https://sqlite.org/fts5.html#full_text_query_syntax" + ) + raise - # Reverse the order - we do this because we 'order by id desc limit 3' to get the - # 3 most recent results, but we still want to display them in chronological order - # ... except for searches where we don't do this + # Newest first out of the query, but read chronologically - except + # for search results, which are already most-relevant first. if not query and not data: rows.reverse() - # Fetch any attachments - ids = [row["id"] for row in rows] - attachments = list(db.query(ATTACHMENTS_SQL.format(",".join("?" * len(ids))), ids)) - attachments_by_id = {} - for attachment in attachments: - attachments_by_id.setdefault(attachment["response_id"], []).append(attachment) - - FRAGMENTS_SQL = """ - select - {table}.response_id, - fragments.hash, - fragments.id as fragment_id, - fragments.content, - ( - select json_group_array(fragment_aliases.alias) - from fragment_aliases - where fragment_aliases.fragment_id = fragments.id - ) as aliases - from {table} - join fragments on {table}.fragment_id = fragments.id - where {table}.response_id in ({placeholders}) - order by {table}."order" - """ - - # Fetch any prompt or system prompt fragments - prompt_fragments_by_id = {} - system_fragments_by_id = {} - for table, dictionary in ( - ("prompt_fragments", prompt_fragments_by_id), - ("system_fragments", system_fragments_by_id), - ): - for fragment in db.query( - FRAGMENTS_SQL.format(placeholders=",".join("?" * len(ids)), table=table), - ids, - ): - dictionary.setdefault(fragment["response_id"], []).append(fragment) - if data or data_array or data_key or data_ids: # Special case for --data to output valid JSON to_output = [] @@ -1822,16 +1959,14 @@ def logs_list( response = row["response"] or "" try: decoded = json.loads(response) - new_items = [] if ( isinstance(decoded, dict) and (data_key in decoded) and all(isinstance(item, dict) for item in decoded[data_key]) ): - for item in decoded[data_key]: - new_items.append(item) + new_items = list(decoded[data_key]) else: - new_items.append(decoded) + new_items = [decoded] if data_ids: for item in new_items: item[find_unused_key(item, "response_id")] = row["id"] @@ -1843,137 +1978,64 @@ def logs_list( click.echo(line) return - # Tool usage information - TOOLS_SQL = """ - SELECT responses.id, - -- Tools related to this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', t.id, - 'hash', t.hash, - 'name', t.name, - 'description', t.description, - 'input_schema', json(t.input_schema) - )) - FROM tools t - JOIN tool_responses tr ON t.id = tr.tool_id - WHERE tr.response_id = responses.id - ), - '[]' - ) AS tools, - -- Tool calls for this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', tc.id, - 'tool_id', tc.tool_id, - 'name', tc.name, - 'arguments', json(tc.arguments), - 'tool_call_id', tc.tool_call_id - )) - FROM tool_calls tc - WHERE tc.response_id = responses.id - ), - '[]' - ) AS tool_calls, - -- Tool results for this response - COALESCE( - (SELECT json_group_array(json_object( - 'id', tr.id, - 'tool_id', tr.tool_id, - 'name', tr.name, - 'output', tr.output, - 'tool_call_id', tr.tool_call_id, - 'exception', tr.exception, - 'attachments', COALESCE( - (SELECT json_group_array(json_object( - 'id', a.id, - 'type', a.type, - 'path', a.path, - 'url', a.url, - 'content', a.content - )) - FROM tool_results_attachments tra - JOIN attachments a ON tra.attachment_id = a.id - WHERE tra.tool_result_id = tr.id - ), - '[]' - ) - )) - FROM tool_results tr - WHERE tr.response_id = responses.id - ), - '[]' - ) AS tool_results - FROM responses - where id in ({placeholders}) - """ - tool_info_by_id = { - row["id"]: { - "tools": json.loads(row["tools"]), - "tool_calls": json.loads(row["tool_calls"]), - "tool_results": json.loads(row["tool_results"]), - } - for row in db.query( - TOOLS_SQL.format(placeholders=",".join("?" * len(ids))), ids - ) - } - - for row in rows: - if truncate: - row["prompt"] = truncate_string(row["prompt"] or "") - row["response"] = truncate_string(row["response"] or "") - # Add prompt and system fragments - for key in ("prompt_fragments", "system_fragments"): - row[key] = [ - { - "hash": fragment["hash"], - "content": ( - fragment["content"] - if expand - else truncate_string(fragment["content"]) - ), - "aliases": json.loads(fragment["aliases"]), - } - for fragment in ( - prompt_fragments_by_id.get(row["id"], []) - if key == "prompt_fragments" - else system_fragments_by_id.get(row["id"], []) - ) - ] - # Either decode or remove all JSON keys - keys = list(row.keys()) - for key in keys: - if key.endswith("_json") and row[key] is not None: - if truncate: - del row[key] - else: - row[key] = json.loads(row[key]) - row.update(tool_info_by_id[row["id"]]) + attachments_by_id = annotate_log_rows(db, rows, expand=expand, truncate=truncate) output = None if json_output: # Output as JSON if requested - for row in rows: - row["attachments"] = [ - {k: v for k, v in attachment.items() if k != "response_id"} - for attachment in attachments_by_id.get(row["id"], []) - ] - output = json.dumps(list(rows), indent=2) + output = log_rows_as_json(rows, attachments_by_id) elif extract or extract_last: # Extract and return first code block for row in rows: output = extract_fenced_code_block(row["response"], last=extract_last) if output is not None: break - elif response: + elif response and rows: # Just output the last response - if rows: - output = rows[-1]["response"] + output = rows[-1]["response"] if output is not None: click.echo(output) else: # Output neatly formatted human-readable logs + def _fenced_block(value): + # Fenced code block, indented to nest inside a list item + num_backticks = 3 + while "`" * num_backticks in value: + num_backticks += 1 + fence = "`" * num_backticks + return textwrap.indent(f"{fence}\n{value}\n{fence}", " ") + + def _inline_code(value): + num_backticks = 1 + while "`" * num_backticks in value: + num_backticks += 1 + delimiter = "`" * num_backticks + if value.startswith("`") or value.endswith("`"): + return f"{delimiter} {value} {delimiter}" + return f"{delimiter}{value}{delimiter}" + + def _format_tool_call_arguments(arguments): + if not isinstance(arguments, dict) or not arguments: + return f" Arguments: {_inline_code(json.dumps(arguments))}" + lines = [] + for key, value in arguments.items(): + if isinstance(value, str): + lines.append(f" {key}:") + lines.append(_fenced_block(value)) + else: + lines.append(f" {key}: {_inline_code(json.dumps(value))}") + return "\n".join(lines) + + def _token_usage_markdown(input_tokens, output_tokens, token_details): + usage = token_usage_string(input_tokens, output_tokens, None) + if token_details: + details = _inline_code(json.dumps(token_details)) + if usage: + return f"{usage}, {details}" + return details + return usage + def _display_fragments(fragments, title): if not fragments: return @@ -1995,6 +2057,7 @@ def _display_fragments(fragments, title): current_system = None should_show_conversation = True + seen_tool_hashes = set() for row in rows: if short: system = truncate_string( @@ -2088,9 +2151,9 @@ def _display_fragments(fragments, title): options = json.loads(options) if options: options_text = "\n".join( - "- {}: {}".format(key, value) for key, value in options.items() + f"- {key}: {value}" for key, value in options.items() ) - click.echo("\n## Options\n\n{}".format(options_text)) + click.echo(f"\n## Options\n\n{options_text}") if row["system"] != current_system: if row["system"] is not None: click.echo("\n## System\n\n{}".format(row["system"])) @@ -2105,15 +2168,44 @@ def _display_fragments(fragments, title): # Show tool calls and results if row["tools"]: click.echo("\n### Tools\n") - for tool in row["tools"]: - click.echo( - "- **{}**: `{}`
\n {}
\n Arguments: {}".format( + + def echo_tool(tool, indent=""): + if tool["hash"] in seen_tool_hashes: + block = "- **{}**: `{}`".format(tool["name"], tool["hash"][:7]) + else: + seen_tool_hashes.add(tool["hash"]) + block = "- **{}**: `{}` \n{} \n Arguments: `{}`".format( tool["name"], tool["hash"], - tool["description"], - json.dumps(tool["input_schema"]["properties"]), + textwrap.indent( + (tool["description"] or "").rstrip(), " " + ), + json.dumps(tool["input_schema"].get("properties", {})), + ) + click.echo(textwrap.indent(block, indent)) + + # Tools provided by the same configured toolbox instance + # nest beneath one instance line rather than repeating it + plain_tools = [] + by_instance: dict = {} + for tool in row["tools"]: + instance = tool.get("instance") + if instance: + key = (instance["name"], instance["arguments"]) + by_instance.setdefault(key, []).append(tool) + else: + plain_tools.append(tool) + for tool in plain_tools: + echo_tool(tool) + for (name, arguments), instance_tools in by_instance.items(): + click.echo( + "- `{}({})`:".format( + name, + arguments if arguments and arguments != "{}" else "", ) ) + for tool in instance_tools: + echo_tool(tool, " ") if row["tool_results"]: click.echo("\n### Tool results\n") for tool_result in row["tool_results"]: @@ -2128,14 +2220,14 @@ def _display_fragments(fragments, title): desc += attachment["url"] elif attachment.get("content"): desc += f"<{attachment['content_length']:,} bytes>" - attachments += "\n - {}".format(desc) + attachments += f"\n - {desc}" click.echo( - "- **{}**: `{}`
\n{}{}{}".format( + "- **{}**: `{}` \n{}{}{}".format( tool_result["name"], tool_result["tool_call_id"], - textwrap.indent(tool_result["output"], " "), + _fenced_block(tool_result["output"]), ( - "
\n **Error**: {}\n".format( + " \n **Error**: {}\n".format( tool_result["exception"] ) if tool_result["exception"] @@ -2173,31 +2265,33 @@ def _display_fragments(fragments, title): if row["schema_json"]: try: parsed = json.loads(response) - response = "```json\n{}\n```".format(json.dumps(parsed, indent=2)) + response = f"```json\n{json.dumps(parsed, indent=2)}\n```" except ValueError: pass + if row.get("reasoning"): + click.echo("\n## Reasoning\n\n{}".format(row["reasoning"].rstrip())) click.echo("\n## Response\n") if row["tool_calls"]: click.echo("### Tool calls\n") for tool_call in row["tool_calls"]: click.echo( - "- **{}**: `{}`
\n Arguments: {}".format( + "- **{}**: `{}` \n{}".format( tool_call["name"], tool_call["tool_call_id"], - json.dumps(tool_call["arguments"]), + _format_tool_call_arguments(tool_call["arguments"]), ) ) click.echo("") if response: - click.echo("{}\n".format(response)) + click.echo(f"{response}\n") if usage: - token_usage = token_usage_string( + token_usage = _token_usage_markdown( row["input_tokens"], row["output_tokens"], json.loads(row["token_details"]) if row["token_details"] else None, ) if token_usage: - click.echo("## Token usage\n\n{}\n".format(token_usage)) + click.echo(f"## Token usage\n\n{token_usage}\n") @cli.group( @@ -2217,6 +2311,93 @@ def models(): } +def model_matches_id_or_alias(model_with_aliases, model_ids): + ids_and_aliases = set( + [model_with_aliases.model.model_id] + model_with_aliases.aliases + ) + return ids_and_aliases.intersection(model_ids) + + +def render_model_with_aliases( + model_with_aliases, + *, + options=False, + async_=False, + models_that_have_shown_options=None, +): + extra_info = [] + if model_with_aliases.aliases: + extra_info.append("aliases: {}".format(", ".join(model_with_aliases.aliases))) + model = model_with_aliases.model if not async_ else model_with_aliases.async_model + output = str(model) + if extra_info: + output += " ({})".format(", ".join(extra_info)) + if options and model.Options.model_json_schema()["properties"]: + output += "\n Options:" + for name, field in model.Options.model_json_schema()["properties"].items(): + any_of = field.get("anyOf") + if any_of is None: + any_of = [{"type": field.get("type", "str")}] + types = ", ".join( + [ + _type_lookup.get(item.get("type"), item.get("type", "str")) + for item in any_of + if item.get("type") != "null" + ] + ) + bits = ["\n ", name, ": ", types] + description = field.get("description", "") + if ( + description + and models_that_have_shown_options is not None + and model.__class__ not in models_that_have_shown_options + ): + wrapped = textwrap.wrap(description, 70) + bits.append("\n ") + bits.extend("\n ".join(wrapped)) + output += "".join(bits) + if models_that_have_shown_options is not None: + models_that_have_shown_options.add(model.__class__) + if options and model.attachment_types: + attachment_types = ", ".join(sorted(model.attachment_types)) + wrapper = textwrap.TextWrapper( + width=min(max(shutil.get_terminal_size().columns, 30), 70), + initial_indent=" ", + subsequent_indent=" ", + ) + output += f"\n Attachment types:\n{wrapper.fill(attachment_types)}" + features = ( + [] + + (["streaming"] if model.can_stream else []) + + (["schemas"] if model.supports_schema else []) + + (["tools"] if model.supports_tools else []) + + (["async"] if model_with_aliases.async_model else []) + ) + if options and features: + output += "\n Features:\n{}".format( + "\n".join(f" - {feature}" for feature in features) + ) + if options and hasattr(model, "needs_key") and model.needs_key: + output += "\n Keys:" + if hasattr(model, "needs_key") and model.needs_key: + output += f"\n key: {model.needs_key}" + if hasattr(model, "key_env_var") and model.key_env_var: + output += f"\n env_var: {model.key_env_var}" + return output + + +def render_model_with_options(model_id, *, async_=False): + for model_with_aliases in get_models_with_aliases(): + if model_matches_id_or_alias(model_with_aliases, [model_id]): + return render_model_with_aliases( + model_with_aliases, + options=True, + async_=async_, + models_that_have_shown_options=set(), + ) + raise click.ClickException(f"'{model_id}' is not a known model") + + @models.command(name="list") @click.option( "--options", is_flag=True, help="Show options for each model, if available" @@ -2224,6 +2405,7 @@ def models(): @click.option("async_", "--async", is_flag=True, help="List async models") @click.option("--schemas", is_flag=True, help="List models that support schemas") @click.option("--tools", is_flag=True, help="List models that support tools") +@click.option("json_", "--json", is_flag=True, help="Output as JSON") @click.option( "-q", "--query", @@ -2231,86 +2413,57 @@ def models(): help="Search for models matching these strings", ) @click.option("model_ids", "-m", "--model", help="Specific model IDs", multiple=True) -def models_list(options, async_, schemas, tools, query, model_ids): +def models_list(options, async_, schemas, tools, json_, query, model_ids): "List available models" models_that_have_shown_options = set() + json_models = [] for model_with_aliases in get_models_with_aliases(): if async_ and not model_with_aliases.async_model: continue - if query: - # Only show models where every provided query string matches - if not all(model_with_aliases.matches(q) for q in query): - continue - if model_ids: - ids_and_aliases = set( - [model_with_aliases.model.model_id] + model_with_aliases.aliases - ) - if not ids_and_aliases.intersection(model_ids): - continue + # Only show models where every provided query string matches + if query and not all(model_with_aliases.matches(q) for q in query): + continue + if model_ids and not model_matches_id_or_alias(model_with_aliases, model_ids): + continue if schemas and not model_with_aliases.model.supports_schema: continue if tools and not model_with_aliases.model.supports_tools: continue - extra_info = [] - if model_with_aliases.aliases: - extra_info.append( - "aliases: {}".format(", ".join(model_with_aliases.aliases)) + if json_: + model = ( + model_with_aliases.async_model if async_ else model_with_aliases.model ) - model = ( - model_with_aliases.model if not async_ else model_with_aliases.async_model - ) - output = str(model) - if extra_info: - output += " ({})".format(", ".join(extra_info)) - if options and model.Options.model_json_schema()["properties"]: - output += "\n Options:" - for name, field in model.Options.model_json_schema()["properties"].items(): - any_of = field.get("anyOf") - if any_of is None: - any_of = [{"type": field.get("type", "str")}] - types = ", ".join( - [ - _type_lookup.get(item.get("type"), item.get("type", "str")) - for item in any_of - if item.get("type") != "null" - ] - ) - bits = ["\n ", name, ": ", types] - description = field.get("description", "") - if description and ( - model.__class__ not in models_that_have_shown_options - ): - wrapped = textwrap.wrap(description, 70) - bits.append("\n ") - bits.extend("\n ".join(wrapped)) - output += "".join(bits) - models_that_have_shown_options.add(model.__class__) - if options and model.attachment_types: - attachment_types = ", ".join(sorted(model.attachment_types)) - wrapper = textwrap.TextWrapper( - width=min(max(shutil.get_terminal_size().columns, 30), 70), - initial_indent=" ", - subsequent_indent=" ", + model_json = { + "model_id": model.model_id, + "aliases": model_with_aliases.aliases, + "can_stream": model.can_stream, + "supports_schema": model.supports_schema, + "supports_tools": model.supports_tools, + "supports_async": model_with_aliases.async_model is not None, + "attachment_types": sorted(model.attachment_types), + "server_side_tools": [ + { + "name": tool_class.__name__, + "plugin": getattr(tool_class, "plugin", None), + } + for tool_class in model.supported_server_side_tools + ], + } + if options: + model_json["options"] = model.Options.model_json_schema()["properties"] + json_models.append(model_json) + continue + click.echo( + render_model_with_aliases( + model_with_aliases, + options=options, + async_=async_, + models_that_have_shown_options=models_that_have_shown_options, ) - output += "\n Attachment types:\n{}".format(wrapper.fill(attachment_types)) - features = ( - [] - + (["streaming"] if model.can_stream else []) - + (["schemas"] if model.supports_schema else []) - + (["tools"] if model.supports_tools else []) - + (["async"] if model_with_aliases.async_model else []) ) - if options and features: - output += "\n Features:\n{}".format( - "\n".join(" - {}".format(feature) for feature in features) - ) - if options and hasattr(model, "needs_key") and model.needs_key: - output += "\n Keys:" - if hasattr(model, "needs_key") and model.needs_key: - output += "\n key: {}".format(model.needs_key) - if hasattr(model, "key_env_var") and model.key_env_var: - output += "\n env_var: {}".format(model.key_env_var) - click.echo(output) + if json_: + click.echo(json.dumps(json_models, indent=2)) + return if not query and not options and not schemas and not model_ids: click.echo(f"Default: {get_default_model()}") @@ -2327,7 +2480,7 @@ def models_default(model): model = get_model(model) set_default_model(model.model_id) except KeyError: - raise click.ClickException("Unknown model: {}".format(model)) + raise click.ClickException(f"Unknown model: {model}") @cli.group( @@ -2380,7 +2533,7 @@ def templates_show(name): raise click.ClickException(f"Template '{name}' not found or invalid") click.echo( yaml.dump( - dict((k, v) for k, v in template.model_dump().items() if v is not None), + {k: v for k, v in template.model_dump().items() if v is not None}, indent=4, default_flow_style=False, ) @@ -2460,7 +2613,7 @@ def schemas_list(path, database, queries, full, json_, nl): path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): - raise click.ClickException("No log database found at {}".format(path)) + raise click.ClickException(f"No log database found at {path}") db = sqlite_utils.Database(path) migrate(db) @@ -2469,9 +2622,9 @@ def schemas_list(path, database, queries, full, json_, nl): if queries: where_bits = ["schemas.content like ?" for _ in queries] where_sql += " where {}".format(" and ".join(where_bits)) - params.extend("%{}%".format(q) for q in queries) + params.extend(f"%{q}%" for q in queries) - sql = """ + sql = f""" select schemas.id, schemas.content, @@ -2480,11 +2633,9 @@ def schemas_list(path, database, queries, full, json_, nl): from schemas join responses on responses.schema_id = schemas.id - {} group by responses.schema_id + {where_sql} group by responses.schema_id order by recently_used - """.format( - where_sql - ) + """ rows = db.query(sql, params) if json_ or nl: @@ -2538,7 +2689,7 @@ def schemas_show(schema_id, path, database): path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): - raise click.ClickException("No log database found at {}".format(path)) + raise click.ClickException(f"No log database found at {path}") db = sqlite_utils.Database(path) migrate(db) @@ -2575,18 +2726,51 @@ def tools(): @tools.command(name="list") @click.argument("tool_defs", nargs=-1) @click.option("json_", "--json", is_flag=True, help="Output as JSON") +@click.option("model_id", "-m", "--model", help="List tools supported by this model") @click.option( "python_tools", "--functions", help="Python code block or file path defining functions to register as tools", multiple=True, ) -def tools_list(tool_defs, json_, python_tools): - "List available tools that have been provided by plugins" +def tools_list(tool_defs, json_, model_id, python_tools): + "List available tools, optionally including tools supported by a model" + + model = None + if model_id: + try: + model = get_model(model_id) + except UnknownModelError as ex: + raise click.ClickException(str(ex)) - def introspect_tools(toolbox_class): + server_side_tools = [] + if model is not None: + for tool_class in model.supported_server_side_tools: + try: + signature = str(inspect.signature(tool_class)) + except (ValueError, TypeError): + signature = "(...)" + server_side_tools.append( + { + "name": tool_class.__name__, + "description": inspect.getdoc(tool_class), + "signature": signature, + "server_side": True, + } + ) + + def introspect_tools(toolbox): + # Instances report their tools(), which may be generated dynamically. + # Classes can only report tools for their introspectable methods. + if isinstance(toolbox, Toolbox): + if not toolbox._prepared: + toolbox.prepare() + toolbox._prepared = True + tool_iter = toolbox.tools() + else: + tool_iter = toolbox.method_tools() methods = [] - for tool in toolbox_class.method_tools(): + for tool in tool_iter: methods.append( { "name": tool.name, @@ -2597,13 +2781,19 @@ def introspect_tools(toolbox_class): ) return methods + toolbox_specs: dict[int, str] = {} if tool_defs: tools = {} - for tool in _gather_tools(tool_defs, python_tools): + gathered = _gather_tools(tool_defs, python_tools) + # _gather_tools returns --functions tools first, then one per spec + specs = [None] * (len(gathered) - len(tool_defs)) + list(tool_defs) + for spec, tool in zip(specs, gathered): if hasattr(tool, "name"): tools[tool.name] = tool else: tools[tool.__class__.__name__] = tool + if spec is not None and isinstance(tool, Toolbox): + toolbox_specs[id(tool)] = spec else: tools = get_tools() if python_tools: @@ -2614,7 +2804,7 @@ def introspect_tools(toolbox_class): output_tools = [] output_toolboxes = [] tool_objects = [] - toolbox_objects = [] + toolbox_infos = [] for name, tool in sorted(tools.items()): if isinstance(tool, Tool): tool_objects.append(tool) @@ -2627,27 +2817,34 @@ def introspect_tools(toolbox_class): } ) else: - toolbox_objects.append(tool) + toolbox_class = tool if isinstance(tool, type) else tool.__class__ + # Overriding tools() or prepare() means the toolbox generates + # tools at runtime + is_dynamic = any( + getattr(toolbox_class, method) is not getattr(Toolbox, method) + for method in ("tools", "prepare", "prepare_async") + ) + introspected = introspect_tools(tool) + toolbox_infos.append((name, tool, toolbox_class, is_dynamic, introspected)) output_toolboxes.append( { "name": name, + "dynamic": is_dynamic, "tools": [ { - "name": tool["name"], - "description": tool["description"], - "arguments": tool["arguments"], + "name": tool_info["name"], + "description": tool_info["description"], + "arguments": tool_info["arguments"], } - for tool in introspect_tools(tool) + for tool_info in introspected ], } ) if json_: - click.echo( - json.dumps( - {"tools": output_tools, "toolboxes": output_toolboxes}, - indent=2, - ) - ) + output = {"tools": output_tools, "toolboxes": output_toolboxes} + if model is not None: + output["server_side_tools"] = server_side_tools + click.echo(json.dumps(output, indent=2)) else: for tool in tool_objects: sig = "()" @@ -2657,27 +2854,54 @@ def introspect_tools(toolbox_class): "{}{}{}\n".format( tool.name, sig, - " (plugin: {})".format(tool.plugin) if tool.plugin else "", + f" (plugin: {tool.plugin})" if tool.plugin else "", ) ) if tool.description: click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") - for toolbox in toolbox_objects: - click.echo(toolbox.name + ":\n") - for tool in toolbox.method_tools(): - sig = ( - str(inspect.signature(tool.implementation)) - .replace("(self, ", "(") - .replace("(self)", "()") - ) + for name, toolbox, toolbox_class, is_dynamic, introspected in toolbox_infos: + if is_dynamic and isinstance(toolbox, type): + # A dynamic toolbox class has no tools until it is + # instantiated - show its constructor and docstring instead + try: + constructor_sig = str(inspect.signature(toolbox_class)) + except (ValueError, TypeError): + constructor_sig = "(...)" + plugin = getattr(toolbox_class, "plugin", None) click.echo( - " {}{}\n".format( - tool.name, - sig, + "{}{}{}\n".format( + name, + constructor_sig, + f" (plugin: {plugin})" if plugin else "", ) ) - if tool.description: - click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") + doc = toolbox_class.__doc__ + if doc: + click.echo(textwrap.indent(inspect.cleandoc(doc), " ") + "\n") + else: + click.echo(toolbox_specs.get(id(toolbox), name) + ":\n") + for tool_info in introspected: + sig = "()" + if tool_info["implementation"]: + sig = ( + str(inspect.signature(tool_info["implementation"])) + .replace("(self, ", "(") + .replace("(self)", "()") + ) + click.echo(f" {tool_info['name']}{sig}\n") + if tool_info["description"]: + click.echo( + textwrap.indent(tool_info["description"].strip(), " ") + "\n" + ) + if model is not None and server_side_tools: + click.echo( + f"Server-side tools for {model.model_id} " + "(executed by the provider):\n" + ) + for tool_info in server_side_tools: + click.echo(f"{tool_info['name']}{tool_info['signature']}\n") + if tool_info["description"]: + click.echo(textwrap.indent(tool_info["description"], " ") + "\n") @cli.group( @@ -2731,13 +2955,13 @@ def aliases_set(alias, model_id, query): Example usage: \b - llm aliases set mini gpt-4o-mini + llm aliases set luna gpt-5.6-luna Alternatively you can omit the model ID and specify one or more -q options. The first model matching all of those query strings will be used. \b - llm aliases set mini -q 4o -q mini + llm aliases set luna -q gpt -q luna """ if not model_id: if not query: @@ -2815,25 +3039,21 @@ def fragments_list(queries, aliases, json_): db = sqlite_utils.Database(logs_db_path()) migrate(db) params = {} - param_count = 0 where_bits = [] if aliases: where_bits.append("fragment_aliases.alias is not null") - for q in queries: - param_count += 1 + for param_count, q in enumerate(queries, start=1): p = f"p{param_count}" params[p] = q - where_bits.append( - f""" + where_bits.append(f""" (fragments.hash = :{p} or fragment_aliases.alias = :{p} or fragments.source like '%' || :{p} || '%' or fragments.content like '%' || :{p} || '%') - """ - ) + """) where = "\n and\n ".join(where_bits) if where: where = " where " + where - sql = """ + sql = f""" select fragments.hash, json_group_array(fragment_aliases.alias) filter ( @@ -2851,9 +3071,7 @@ def fragments_list(queries, aliases, json_): group by fragments.id, fragments.hash, fragments.content, fragments.datetime_utc, fragments.source order by fragments.datetime_utc - """.format( - where=where - ) + """ results = list(db.query(sql, params)) for result in results: result["aliases"] = json.loads(result["aliases"]) @@ -2898,9 +3116,9 @@ def fragments_set(alias, fragment): on conflict(alias) do update set fragment_id = excluded.fragment_id; """ - with db.conn: + with db.atomic(): fragment_id = ensure_fragment(db, resolved) - db.conn.execute(alias_sql, {"alias": alias, "fragment_id": fragment_id}) + db.execute(alias_sql, {"alias": alias, "fragment_id": fragment_id}) @fragments.command(name="show") @@ -2934,10 +3152,7 @@ def fragments_remove(alias): """ db = sqlite_utils.Database(logs_db_path()) migrate(db) - with db.conn: - db.conn.execute( - "delete from fragment_aliases where alias = :alias", {"alias": alias} - ) + db.execute("delete from fragment_aliases where alias = :alias", {"alias": alias}) @fragments.command(name="loaders") @@ -3262,11 +3477,8 @@ def embed_multi( if not input_path and not sql and not files: raise click.UsageError("Either --sql or input path or --files is required") - if files: - if input_path or sql or format: - raise click.UsageError( - "Cannot use --files with --sql, input path or --format" - ) + if files and (input_path or sql or format): + raise click.UsageError("Cannot use --files with --sql, input path or --format") if database: db = sqlite_utils.Database(database) @@ -3276,11 +3488,12 @@ def embed_multi( for alias, attach_path in attach: db.attach(alias, attach_path) + model_id = model or get_default_embedding_model() try: collection_obj = Collection( - collection, db=db, model_id=model or get_default_embedding_model() + collection, db=db, model_id=model_id, create=model_id is not None ) - except ValueError: + except (Collection.DoesNotExist, UnknownModelError): raise click.ClickException( "You need to specify an embedding model (no default model is set)" ) @@ -3318,7 +3531,7 @@ def iterate_files(): if content is None: # Log to stderr click.echo( - "Could not decode text in file {}".format(path), + f"Could not decode text in file {path}", err=True, ) else: @@ -3328,7 +3541,7 @@ def iterate_files(): rows = iterate_files() elif sql: rows = db.query(sql) - count_sql = "select count(*) as c from ({})".format(sql) + count_sql = f"select count(*) as c from ({sql})" expected_length = next(db.query(count_sql))["c"] else: @@ -3343,11 +3556,15 @@ def load_rows(fp): for _ in load_rows(fp): expected_length += 1 - rows = load_rows( - open(input_path, "rb") - if input_path != "-" - else io.BufferedReader(sys.stdin.buffer) - ) + if input_path != "-": + + def rows_from_input(): + with open(input_path, "rb") as fp: + yield from load_rows(fp) + + rows = rows_from_input() + else: + rows = load_rows(io.BufferedReader(sys.stdin.buffer)) except json.JSONDecodeError as ex: raise click.ClickException(str(ex)) @@ -3355,11 +3572,11 @@ def load_rows(fp): rows, label="Embedding", show_percent=True, length=expected_length ) as rows: - def tuples() -> Iterable[Tuple[str, Union[bytes, str]]]: + def tuples() -> Iterable[tuple[str, bytes | str]]: for row in rows: values = list(row.values()) id: str = prefix + str(values[0]) - content: Optional[Union[bytes, str]] = None + content: bytes | str | None = None if binary: content = cast(bytes, values[1]) else: @@ -3478,9 +3695,8 @@ def embed_models_list(query): "List available embedding models" output = [] for model_with_aliases in get_embedding_models_with_aliases(): - if query: - if not all(model_with_aliases.matches(q) for q in query): - continue + if query and not all(model_with_aliases.matches(q) for q in query): + continue s = str(model_with_aliases.model) if model_with_aliases.aliases: s += " (aliases: {})".format(", ".join(model_with_aliases.aliases)) @@ -3510,7 +3726,7 @@ def embed_models_default(model, remove_default): model = get_embedding_model(model) set_default_embedding_model(model.model_id) except KeyError: - raise click.ClickException("Unknown embedding model: {}".format(model)) + raise click.ClickException(f"Unknown embedding model: {model}") @cli.group( @@ -3542,9 +3758,8 @@ def embed_db_collections(database, json_): database = database or (user_dir() / "embeddings.db") db = sqlite_utils.Database(str(database)) if not db["collections"].exists(): - raise click.ClickException("No collections table found in {}".format(database)) - rows = db.query( - """ + raise click.ClickException(f"No collections table found in {database}") + rows = db.query(""" select collections.name, collections.model, @@ -3554,8 +3769,7 @@ def embed_db_collections(database, json_): on collections.id = embeddings.collection_id group by collections.name, collections.model - """ - ) + """) if json_: click.echo(json.dumps(list(rows), indent=4)) else: @@ -3634,7 +3848,7 @@ def options_show(model): Example usage: \b - llm models options show gpt-4o + llm models options show gpt-4.1 """ import llm @@ -3666,7 +3880,7 @@ def options_set(model, key, value): Example usage: \b - llm models options set gpt-4o temperature 0.5 + llm models options set gpt-4.1 temperature 0.5 """ import llm @@ -3701,9 +3915,9 @@ def options_clear(model, key): Example usage: \b - llm models options clear gpt-4o + llm models options clear gpt-4.1 # Or for a single option - llm models options clear gpt-4o temperature + llm models options clear gpt-4.1 temperature """ import llm @@ -3785,7 +3999,7 @@ def _human_readable_size(size_bytes): size_bytes /= 1024.0 i += 1 - return "{:.2f}{}".format(size_bytes, size_name[i]) + return f"{size_bytes:.2f}{size_name[i]}" def logs_on(): @@ -3895,7 +4109,7 @@ def _parse_yaml_template(name, content): try: loaded = yaml.safe_load(content) except yaml.YAMLError as ex: - raise LoadTemplateError("Invalid YAML: {}".format(str(ex))) + raise LoadTemplateError(f"Invalid YAML: {ex!s}") if isinstance(loaded, str): return Template(name=name, prompt=loaded) loaded["name"] = name @@ -3909,12 +4123,12 @@ def _parse_yaml_template(name, content): def load_template(name: str) -> Template: "Load template, or raise LoadTemplateError(msg)" - if name.startswith("https://") or name.startswith("http://"): + if name.startswith(("https://", "http://")): response = httpx.get(name) try: response.raise_for_status() except httpx.HTTPStatusError as ex: - raise LoadTemplateError("Could not load template {}: {}".format(name, ex)) + raise LoadTemplateError(f"Could not load template {name}: {ex}") return _parse_yaml_template(name, response.text) potential_path = pathlib.Path(name) @@ -3923,12 +4137,12 @@ def load_template(name: str) -> Template: prefix, rest = name.split(":", 1) loaders = get_template_loaders() if prefix not in loaders: - raise LoadTemplateError("Unknown template prefix: {}".format(prefix)) + raise LoadTemplateError(f"Unknown template prefix: {prefix}") loader = loaders[prefix] try: return loader(rest) - except Exception as ex: - raise LoadTemplateError("Could not load template {}: {}".format(name, ex)) + except Exception as ex: # noqa: BLE001 + raise LoadTemplateError(f"Could not load template {name}: {ex}") # Try local file if potential_path.exists(): @@ -3945,7 +4159,7 @@ def load_template(name: str) -> Template: return template_obj -def _tools_from_code(code_or_path: str) -> List[Tool]: +def _tools_from_code(code_or_path: str) -> list[Tool]: """ Treat all Python functions in the code as tools """ @@ -3953,13 +4167,13 @@ def _tools_from_code(code_or_path: str) -> List[Tool]: try: code_or_path = pathlib.Path(code_or_path).read_text() except FileNotFoundError: - raise click.ClickException("File not found: {}".format(code_or_path)) - namespace: Dict[str, Any] = {} + raise click.ClickException(f"File not found: {code_or_path}") + namespace: dict[str, Any] = {} tools = [] try: - exec(code_or_path, namespace) + exec(code_or_path, namespace) # noqa: S102 except SyntaxError as ex: - raise click.ClickException("Error in --functions definition: {}".format(ex)) + raise click.ClickException(f"Error in --functions definition: {ex}") # Register all callables in the locals dict: for name, value in namespace.items(): if callable(value) and not name.startswith("_"): @@ -3970,7 +4184,7 @@ def _tools_from_code(code_or_path: str) -> List[Tool]: def _debug_tool_call(_, tool_call, tool_result): click.echo( click.style( - "\nTool call: {}({})".format(tool_call.name, tool_call.arguments), + f"\nTool call: {tool_call.name}({tool_call.arguments})", fg="yellow", bold=True, ), @@ -3981,7 +4195,7 @@ def _debug_tool_call(_, tool_call, tool_result): if tool_result.attachments: attachments += "\nAttachments:\n" for attachment in tool_result.attachments: - attachments += f" {repr(attachment)}\n" + attachments += f" {attachment!r}\n" try: output = json.dumps(json.loads(tool_result.output), indent=2) @@ -3999,7 +4213,7 @@ def _debug_tool_call(_, tool_call, tool_result): if tool_result.exception: click.echo( click.style( - " Exception: {}".format(tool_result.exception), + f" Exception: {tool_result.exception}", fg="red", bold=True, ), @@ -4010,7 +4224,7 @@ def _debug_tool_call(_, tool_call, tool_result): def _approve_tool_call(_, tool_call): click.echo( click.style( - "Tool call: {}({})".format(tool_call.name, tool_call.arguments), + f"Tool call: {tool_call.name}({tool_call.arguments})", fg="yellow", bold=True, ), @@ -4021,41 +4235,73 @@ def _approve_tool_call(_, tool_call): def _gather_tools( - tool_specs: List[str], python_tools: List[str] -) -> List[Union[Tool, Type[Toolbox]]]: - tools: List[Union[Tool, Type[Toolbox]]] = [] + tool_specs: list[str], python_tools: list[str], model=None +) -> list[Tool | Toolbox | ServerSideTool]: + tools: list[Tool | Toolbox | ServerSideTool] = [] if python_tools: for code_or_path in python_tools: tools.extend(_tools_from_code(code_or_path)) registered_tools = get_tools() - registered_classes = dict( - (key, value) - for key, value in registered_tools.items() - if inspect.isclass(value) - ) + server_side_tool_classes = { + tool_class.__name__: tool_class + for tool_class in ( + model.supported_server_side_tools if model is not None else () + ) + } + available_tools = {**registered_tools, **server_side_tool_classes} + registered_classes = { + key: value for key, value in available_tools.items() if inspect.isclass(value) + } bad_tools = [ - tool for tool in tool_specs if tool.split("(")[0] not in registered_tools + tool + for tool in tool_specs + if tool.split("(", 1)[0].strip() not in available_tools ] if bad_tools: raise click.ClickException( "Tool(s) {} not found. Available tools: {}".format( - ", ".join(bad_tools), ", ".join(registered_tools.keys()) + ", ".join(bad_tools), ", ".join(available_tools.keys()) ) ) for tool_spec in tool_specs: if not tool_spec[0].isupper(): # It's a function - tools.append(registered_tools[tool_spec]) + tools.append(available_tools[tool_spec]) else: # It's a class tools.append(instantiate_from_spec(registered_classes, tool_spec)) return tools +def _tool_chain_kwargs( + tool_specs, python_tools, tools_debug, tools_approve, chain_limit, model=None +): + """Build Conversation.chain() keyword arguments for CLI-selected tools.""" + tool_implementations = _gather_tools(tool_specs, python_tools, model=model) + if not tool_implementations: + return {} + kwargs = { + "tools": tool_implementations, + "chain_limit": chain_limit, + } + if tools_debug: + kwargs["after_call"] = _debug_tool_call + if tools_approve: + kwargs["before_call"] = _approve_tool_call + return kwargs + + def _get_conversation_tools(conversation, tools): - if conversation and not tools and conversation.responses: + if not conversation or tools: + return None + if conversation.responses: # Copy plugin tools from first response in conversation initial_tools = conversation.responses[0].prompt.tools if initial_tools: # Only tools from plugins: return [tool.name for tool in initial_tools if tool.plugin] + elif conversation.loaded_tools: + # Conversation loaded from the message store - tool names and + # toolbox specs were read from turn_tools instead of rebuilt + # responses. + return list(conversation.loaded_tools) diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index ccca240cf..94cec5938 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -1,3 +1,19 @@ +import datetime +import json +import os +import sys +from collections.abc import AsyncGenerator, Iterable, Iterator +from enum import Enum +from typing import Any, ClassVar, Literal + +import click +import httpx +import openai +import sqlite_utils +import yaml +from pydantic import Field, ValidationError, create_model, field_validator + +import llm from llm import ( AsyncConversation, AsyncKeyModel, @@ -9,160 +25,169 @@ Response, hookimpl, ) -import llm +from llm.models import _partition_tools +from llm.parts import StreamEvent from llm.utils import ( dicts_to_table_string, - remove_dict_none_values, logging_client, + remove_dict_none_values, simplify_usage_dict, ) -import click -import datetime -from enum import Enum -import httpx -import openai -import os - -from pydantic import field_validator, Field - -from typing import AsyncGenerator, cast, List, Iterable, Iterator, Optional, Union -import json -import yaml @hookimpl def register_models(register): # GPT-4o register( - Chat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), - AsyncChat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), + Chat( + "gpt-4o", + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + "gpt-4o", + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), aliases=("4o",), ) register( - Chat("chatgpt-4o-latest", vision=True), - AsyncChat("chatgpt-4o-latest", vision=True), - aliases=("chatgpt-4o",), - ) - register( - Chat("gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True), + Chat( + "gpt-4o-mini", + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), AsyncChat( - "gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True + "gpt-4o-mini", + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, ), aliases=("4o-mini",), ) - for audio_model_id in ( - "gpt-4o-audio-preview", - "gpt-4o-audio-preview-2024-12-17", - "gpt-4o-audio-preview-2024-10-01", - "gpt-4o-mini-audio-preview", - "gpt-4o-mini-audio-preview-2024-12-17", - ): - register( - Chat(audio_model_id, audio=True), - AsyncChat(audio_model_id, audio=True), - ) # GPT-4.1 for model_id in ("gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano"): register( - Chat(model_id, vision=True, supports_schema=True, supports_tools=True), - AsyncChat(model_id, vision=True, supports_schema=True, supports_tools=True), + Chat( + model_id, + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + AsyncChat( + model_id, + vision=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), aliases=(model_id.replace("gpt-", ""),), ) # 3.5 and 4 register( - Chat("gpt-3.5-turbo"), AsyncChat("gpt-3.5-turbo"), aliases=("3.5", "chatgpt") + Chat("gpt-3.5-turbo", service_tier=True), + AsyncChat("gpt-3.5-turbo", service_tier=True), + aliases=("3.5", "chatgpt"), ) register( - Chat("gpt-3.5-turbo-16k"), - AsyncChat("gpt-3.5-turbo-16k"), + Chat("gpt-3.5-turbo-16k", service_tier=True), + AsyncChat("gpt-3.5-turbo-16k", service_tier=True), aliases=("chatgpt-16k", "3.5-16k"), ) - register(Chat("gpt-4"), AsyncChat("gpt-4"), aliases=("4", "gpt4")) - register(Chat("gpt-4-32k"), AsyncChat("gpt-4-32k"), aliases=("4-32k",)) - # GPT-4 Turbo models - register(Chat("gpt-4-1106-preview"), AsyncChat("gpt-4-1106-preview")) - register(Chat("gpt-4-0125-preview"), AsyncChat("gpt-4-0125-preview")) - register(Chat("gpt-4-turbo-2024-04-09"), AsyncChat("gpt-4-turbo-2024-04-09")) register( - Chat("gpt-4-turbo"), - AsyncChat("gpt-4-turbo"), - aliases=("gpt-4-turbo-preview", "4-turbo", "4t"), + Chat("gpt-4", service_tier=True), + AsyncChat("gpt-4", service_tier=True), + aliases=("4", "gpt4"), ) - # GPT-4.5 + # GPT-4 Turbo models register( - Chat( - "gpt-4.5-preview-2025-02-27", - vision=True, - supports_schema=True, - supports_tools=True, - ), - AsyncChat( - "gpt-4.5-preview-2025-02-27", - vision=True, - supports_schema=True, - supports_tools=True, - ), + Chat("gpt-4-turbo-2024-04-09", service_tier=True), + AsyncChat("gpt-4-turbo-2024-04-09", service_tier=True), ) register( - Chat("gpt-4.5-preview", vision=True, supports_schema=True, supports_tools=True), - AsyncChat( - "gpt-4.5-preview", vision=True, supports_schema=True, supports_tools=True - ), - aliases=("gpt-4.5",), + Chat("gpt-4-turbo", service_tier=True), + AsyncChat("gpt-4-turbo", service_tier=True), + aliases=("gpt-4-turbo-preview", "4-turbo", "4t"), ) # o1 for model_id in ("o1", "o1-2024-12-17"): register( - Chat( + Responses( model_id, vision=True, can_stream=False, reasoning=True, + service_tier=True, supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( model_id, vision=True, can_stream=False, reasoning=True, + service_tier=True, supports_schema=True, supports_tools=True, ), ) register( - Chat("o1-preview", allows_system_prompt=False), - AsyncChat("o1-preview", allows_system_prompt=False), - ) - register( - Chat("o1-mini", allows_system_prompt=False), - AsyncChat("o1-mini", allows_system_prompt=False), - ) - register( - Chat("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), - AsyncChat("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), + Responses( + "o3-mini", + reasoning=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + AsyncResponses( + "o3-mini", + reasoning=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), ) register( - Chat( - "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True + Responses( + "o3", + vision=True, + reasoning=True, + service_tier=True, + supports_schema=True, + supports_tools=True, ), - AsyncChat( - "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True + AsyncResponses( + "o3", + vision=True, + reasoning=True, + service_tier=True, + supports_schema=True, + supports_tools=True, ), ) register( - Chat( + Responses( "o4-mini", vision=True, reasoning=True, + service_tier=True, supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( "o4-mini", vision=True, reasoning=True, + service_tier=True, supports_schema=True, supports_tools=True, ), @@ -177,61 +202,154 @@ def register_models(register): "gpt-5-nano-2025-08-07", ): register( - Chat( + Responses( model_id, vision=True, reasoning=True, + verbosity=True, + service_tier=True, supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( model_id, vision=True, reasoning=True, + verbosity=True, + service_tier=True, supports_schema=True, supports_tools=True, ), ) # GPT-5.1 + register( + Responses( + "gpt-5.1", + vision=True, + reasoning=True, + verbosity=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + AsyncResponses( + "gpt-5.1", + vision=True, + reasoning=True, + verbosity=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + ) + # GPT-5.2 + for model_id in ("gpt-5.2", "gpt-5.2-chat-latest"): + register( + Responses( + model_id, + vision=True, + reasoning=True, + verbosity=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + AsyncResponses( + model_id, + vision=True, + reasoning=True, + verbosity=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + ) + # "gpt-5.2-pro" is Responses API only + + # GPT-5.4 for model_id in ( - "gpt-5.1", - "gpt-5.1-chat-latest", + "gpt-5.4", + "gpt-5.4-2026-03-05", + "gpt-5.4-mini", + "gpt-5.4-mini-2026-03-17", + "gpt-5.4-nano", + "gpt-5.4-nano-2026-03-17", ): register( - Chat( + Responses( model_id, vision=True, reasoning=True, + verbosity=True, + image_detail_original=True, + service_tier=True, supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( model_id, vision=True, reasoning=True, + verbosity=True, + image_detail_original=True, + service_tier=True, supports_schema=True, supports_tools=True, ), ) - # GPT-5.2 - for model_id in ("gpt-5.2", "gpt-5.2-chat-latest"): + # GPT-5.5 — routes through the Responses API by default; pass + # ``-o chat_completions 1`` to fall back to /v1/chat/completions. + for model_id in ( + "gpt-5.5", + "gpt-5.5-2026-04-23", + ): register( - Chat( + Responses( model_id, vision=True, reasoning=True, + verbosity=True, + image_detail_original=True, + service_tier=True, supports_schema=True, supports_tools=True, ), - AsyncChat( + AsyncResponses( model_id, vision=True, reasoning=True, + verbosity=True, + image_detail_original=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + ) + + # GPT-5.6 + for model_id in ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"): + register( + Responses( + model_id, + vision=True, + reasoning=True, + verbosity=True, + image_detail_original=True, + service_tier=True, + supports_schema=True, + supports_tools=True, + ), + AsyncResponses( + model_id, + vision=True, + reasoning=True, + verbosity=True, + image_detail_original=True, + service_tier=True, supports_schema=True, supports_tools=True, ), ) - # "gpt-5.2-pro" is Responses API only # The -instruct completion model register( @@ -266,12 +384,19 @@ def register_models(register): kwargs["vision"] = True if extra_model.get("audio") is True: kwargs["audio"] = True + if extra_model.get("service_tier") is True: + kwargs["service_tier"] = True if extra_model.get("completion"): klass = Completion + async_klass = None + elif extra_model.get("responses"): + klass = Responses + async_klass = AsyncResponses else: klass = Chat - chat_model = klass( - model_id, + async_klass = AsyncChat + model_kwargs = dict( + model_id=model_id, model_name=model_name, api_base=api_base, api_type=api_type, @@ -281,12 +406,19 @@ def register_models(register): reasoning=reasoning, **kwargs, ) + chat_model = klass(**model_kwargs) + async_model = async_klass(**model_kwargs) if async_klass else None if api_base: chat_model.needs_key = None + if async_model: + async_model.needs_key = None if extra_model.get("api_key_name"): chat_model.needs_key = extra_model["api_key_name"] + if async_model: + async_model.needs_key = extra_model["api_key_name"] register( chat_model, + async_model, aliases=aliases, ) @@ -339,7 +471,7 @@ def __init__(self, model_id, openai_model_id, dimensions=None): self.openai_model_id = openai_model_id self.dimensions = dimensions - def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float]]: + def embed_batch(self, items: Iterable[str | bytes]) -> Iterator[list[float]]: kwargs = { "input": items, "model": self.openai_model_id, @@ -353,9 +485,351 @@ def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float @hookimpl def register_commands(cli): + from llm.cli import ( + AttachmentType, + attachment_types_callback, + schema_option, + tool_options, + ) + @cli.group(name="openai") def openai_(): - "Commands for working directly with the OpenAI API" + "Commands for working with OpenAI and OpenAI-compatible APIs" + + @openai_.command() + @click.argument("url") + @click.argument("prompt", required=False) + @click.option( + "model_id", + "-m", + "--model", + help="Model ID (required unless --models or provided by template)", + ) + @click.option("-s", "--system", help="System prompt to use") + @click.option("-t", "--template", help="Template to use") + @click.option( + "param", + "-p", + "--param", + multiple=True, + type=(str, str), + help="Parameters for template", + ) + @click.option( + "options", + "-o", + "--option", + type=(str, str), + multiple=True, + help="key/value options for the model", + ) + @schema_option + @click.option( + "--schema-multi", + help="JSON schema to use for multiple results", + ) + @click.option( + "attachments", + "-a", + "--attachment", + type=AttachmentType(), + multiple=True, + help="Attachment path or URL or -", + ) + @click.option( + "attachment_types", + "--at", + "--attachment-type", + type=(str, str), + multiple=True, + callback=attachment_types_callback, + help="\b\nAttachment with explicit mimetype,\n--at image.jpg image/jpeg", + ) + @tool_options + @click.option("--key", help="API key or stored key alias to send") + @click.option( + "headers", + "-H", + "--header", + type=(str, str), + multiple=True, + help="Additional HTTP header", + ) + @click.option( + "use_responses", + "--responses", + is_flag=True, + help="Use the Responses API instead of Chat Completions", + ) + @click.option( + "force_chat", + "--chat", + is_flag=True, + help="Start an interactive chat", + ) + @click.option( + "list_models", + "--models", + is_flag=True, + help="List model IDs from the endpoint and exit", + ) + @click.option("--no-stream", is_flag=True, help="Do not stream output") + @click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") + def endpoint( + url, + prompt, + model_id, + system, + template, + param, + options, + schema_input, + schema_multi, + attachments, + attachment_types, + tools, + python_tools, + tools_debug, + tools_approve, + chain_limit, + key, + headers, + use_responses, + force_chat, + list_models, + no_stream, + hide_reasoning, + ): + """ + Run against an OpenAI-compatible endpoint without logging. + + PROMPT or stdin is executed once. If neither is provided, wait for + input on stdin. Use --chat to start an interactive chat. Templates run + once by default; use --chat to apply one interactively. Use --models + to list the available model IDs without running a prompt. + """ + from llm.cli import ( + AttachmentError, + LoadTemplateError, + _apply_template, + _merge_template_attachments, + _merge_template_options, + _merge_template_tools, + _run_chat, + _tool_chain_kwargs, + display_stream_events, + load_template, + logs_db_path, + migrate, + multi_schema, + render_errors, + resolve_schema_input, + ) + + if list_models and prompt is not None: + raise click.ClickException("--models cannot be used with a prompt") + if list_models and template: + raise click.ClickException("--models cannot be used with --template") + if list_models and (tools or python_tools): + raise click.ClickException("--models cannot be used with tools") + if list_models and (schema_input or schema_multi): + raise click.ClickException("--models cannot be used with schemas") + if force_chat and prompt is not None: + raise click.ClickException("--chat cannot be used with a prompt") + + if schema_multi: + schema_input = schema_multi + schema = None + if schema_input: + # Never create logs.db for this unlogged command. An existing + # database can resolve stored schema IDs; all other schema input + # is resolved using a temporary in-memory database. + log_path = logs_db_path() + if log_path.exists(): + schema_db = sqlite_utils.Database(log_path) + else: + schema_db = sqlite_utils.Database(memory=True) + migrate(schema_db) + schema = resolve_schema_input(schema_db, schema_input, load_template) + if schema_multi: + schema = multi_schema(schema) + + template_obj = None + params = dict(param) + if template: + try: + template_obj = load_template(template) + attachments, attachment_types = _merge_template_attachments( + template_obj, attachments, attachment_types + ) + except (AttachmentError, LoadTemplateError) as ex: + raise click.ClickException(str(ex)) + if not model_id and template_obj.model: + model_id = template_obj.model + if template_obj.schema_object and not schema: + schema = template_obj.schema_object + if template_obj.options: + options = _merge_template_options(template_obj, options) + tools, python_tools = _merge_template_tools( + template_obj, tools, python_tools + ) + + if not list_models and not model_id: + raise click.ClickException( + "--model is required unless --models or a template model is used" + ) + + model_class = Responses if use_responses else Chat + model_kwargs = { + "model_id": model_id or "", + "model_name": model_id or "", + "api_base": url, + "headers": dict(headers), + "vision": True, + "audio": not use_responses, + # Optimistically expose capabilities that have no effect until + # the user explicitly exercises them. + "reasoning": True, + "verbosity": True, + "image_detail_original": True, + "supports_schema": True, + "supports_tools": True, + } + if use_responses: + model_kwargs["reasoning_summary"] = False + model = model_class(**model_kwargs) + + # A configured api_base never receives the user's default OpenAI key. + # Match that safety property here: only send credentials when --key + # was explicitly provided for this invocation. + if not key: + model.needs_key = None + + try: + validated_options = { + option_name: option_value + for option_name, option_value in model.Options(**dict(options)) + if option_value is not None + } + except ValidationError as ex: + raise click.ClickException(render_errors(ex.errors())) + + prompt_kwargs = { + "options": validated_options, + "schema": schema, + "stream": not no_stream, + "hide_reasoning": hide_reasoning, + } + if key: + prompt_kwargs["key"] = key + + tool_kwargs = _tool_chain_kwargs( + tools, + python_tools, + tools_debug, + tools_approve, + chain_limit, + model=model, + ) + resolved_attachments = [*attachments, *attachment_types] + try: + if list_models: + available_models = model.get_client(key).models.list() + error = getattr(available_models, "error", None) + if error: + if isinstance(error, dict): + error = error.get("message") or json.dumps(error) + raise click.ClickException(str(error)) + for available_model in available_models: + click.echo(available_model.id) + return + + if force_chat: + conversation = model.conversation() + + def transform_chat_prompt(chat_prompt): + nonlocal system + if template_obj: + chat_prompt, system = _apply_template( + template_obj, chat_prompt, params, system + ) + return chat_prompt + + def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): + nonlocal system + prompt_method = ( + conversation.chain if tool_kwargs else conversation.prompt + ) + response = prompt_method( + chat_prompt, + system=system, + attachments=turn_attachments, + **prompt_kwargs, + **tool_kwargs, + ) + system = None + return response + + _run_chat( + f"{model_id} at {url}", + execute_chat_prompt, + initial_attachments=resolved_attachments, + transform_prompt=transform_chat_prompt, + show_reasoning=not hide_reasoning, + ) + return + + if not sys.stdin.isatty(): + stdin_prompt = sys.stdin.read() + if stdin_prompt: + prompt = " ".join( + part for part in (stdin_prompt, prompt) if part is not None + ) + elif ( + prompt is None + and not resolved_attachments + and not schema + and (template_obj is None or "input" in template_obj.vars()) + ): + # Match `llm prompt`: wait for stdin until EOF instead of + # implicitly starting an interactive chat. + prompt = sys.stdin.read() + if template_obj: + prompt, system = _apply_template(template_obj, prompt, params, system) + if prompt is None and not (resolved_attachments or schema): + raise click.ClickException( + "A prompt is required when stdin is not interactive" + ) + if tool_kwargs: + response = model.conversation().chain( + prompt, + system=system, + attachments=resolved_attachments, + **prompt_kwargs, + **tool_kwargs, + ) + else: + response = model.prompt( + prompt, + system=system, + attachments=resolved_attachments, + **prompt_kwargs, + ) + display_stream_events( + response.stream_events(), + show_reasoning=not hide_reasoning, + ) + click.echo() + except (click.Abort, click.ClickException): + raise + except (ValueError, NotImplementedError) as ex: + raise click.ClickException(str(ex)) + except Exception as ex: + if getattr(sys, "_called_from_test", False) or os.environ.get( + "LLM_RAISE_ERRORS" + ): + raise + raise click.ClickException(str(ex)) @openai_.command() @click.option("json_", "--json", is_flag=True, help="Output as JSON") @@ -390,12 +864,12 @@ def models(json_, key): "created": created_str, } ) - done = dicts_to_table_string("id owned_by created".split(), to_print) + done = dicts_to_table_string(["id", "owned_by", "created"], to_print) print("\n".join(done)) class SharedOptions(llm.Options): - temperature: Optional[float] = Field( + temperature: float | None = Field( description=( "What sampling temperature to use, between 0 and 2. Higher values like " "0.8 will make the output more random, while lower values like 0.2 will " @@ -405,10 +879,10 @@ class SharedOptions(llm.Options): le=2, default=None, ) - max_tokens: Optional[int] = Field( + max_tokens: int | None = Field( description="Maximum number of tokens to generate.", default=None ) - top_p: Optional[float] = Field( + top_p: float | None = Field( description=( "An alternative to sampling with temperature, called nucleus sampling, " "where the model considers the results of the tokens with top_p " @@ -420,7 +894,7 @@ class SharedOptions(llm.Options): le=1, default=None, ) - frequency_penalty: Optional[float] = Field( + frequency_penalty: float | None = Field( description=( "Number between -2.0 and 2.0. Positive values penalize new tokens based " "on their existing frequency in the text so far, decreasing the model's " @@ -430,7 +904,7 @@ class SharedOptions(llm.Options): le=2, default=None, ) - presence_penalty: Optional[float] = Field( + presence_penalty: float | None = Field( description=( "Number between -2.0 and 2.0. Positive values penalize new tokens based " "on whether they appear in the text so far, increasing the model's " @@ -440,18 +914,18 @@ class SharedOptions(llm.Options): le=2, default=None, ) - stop: Optional[str] = Field( + stop: str | None = Field( description=("A string where the API will stop generating further tokens."), default=None, ) - logit_bias: Optional[Union[dict, str]] = Field( + logit_bias: dict | str | None = Field( description=( "Modify the likelihood of specified tokens appearing in the completion. " 'Pass a JSON string like \'{"1712":-100, "892":-100, "1489":-100}\'' ), default=None, ) - seed: Optional[int] = Field( + seed: int | None = Field( description="Integer seed to attempt to sample deterministically", default=None, ) @@ -483,28 +957,145 @@ def validate_logit_bias(cls, logit_bias): class ReasoningEffortEnum(str, Enum): + none = "none" minimal = "minimal" low = "low" medium = "medium" high = "high" + xhigh = "xhigh" + max = "max" -class OptionsForReasoning(SharedOptions): - json_object: Optional[bool] = Field( - description="Output a valid JSON object {...}. Prompt must mention JSON.", - default=None, +class ReasoningSummaryEnum(str, Enum): + auto = "auto" + concise = "concise" + detailed = "detailed" + + +class VerbosityEnum(str, Enum): + low = "low" + medium = "medium" + high = "high" + + +class ImageDetailEnum(str, Enum): + low = "low" + high = "high" + auto = "auto" + + +class ImageDetailWithOriginalEnum(str, Enum): + low = "low" + high = "high" + original = "original" + auto = "auto" + + +def enum_values_sentence(enum_class): + values = [item.value for item in enum_class] + if len(values) == 1: + return values[0] + return "{}, and {}".format(", ".join(values[:-1]), values[-1]) + + +def build_options_class( + *, + reasoning=False, + reasoning_summary=False, + verbosity=False, + image_detail_original=False, + chat_completions=False, + service_tier=False, +): + fields = { + "json_object": ( + bool | None, + Field( + description="Output a valid JSON object {...}. Prompt must mention JSON.", + default=None, + ), + ) + } + if chat_completions: + fields["chat_completions"] = ( + bool | None, + Field( + description=( + "Force the use of the older /v1/chat/completions endpoint " + "instead of /v1/responses. Most callers should leave this " + "off; set to true to fall back to the Chat Completions code " + "path for compatibility." + ), + default=None, + ), + ) + image_detail_enum = ( + ImageDetailWithOriginalEnum if image_detail_original else ImageDetailEnum ) - reasoning_effort: Optional[ReasoningEffortEnum] = Field( - description=( - "Constraints effort on reasoning for reasoning models. Currently supported " - "values are low, medium, and high. Reducing reasoning effort can result in " - "faster responses and fewer tokens used on reasoning in a response." + image_detail_values = enum_values_sentence(image_detail_enum) + fields["image_detail"] = ( + image_detail_enum | None, + Field( + description=( + "Controls the detail level for image attachments. Supported values are " + f"{image_detail_values}." + ), + default=None, ), - default=None, ) + if reasoning: + fields["reasoning_effort"] = ( + ReasoningEffortEnum | None, + Field( + description=( + "Constraints effort on reasoning for reasoning models. Currently " + "supported values are low, medium, and high. Reducing reasoning " + "effort can result in faster responses and fewer tokens used on " + "reasoning in a response." + ), + default=None, + ), + ) + if reasoning_summary: + reasoning_summary_values = enum_values_sentence(ReasoningSummaryEnum) + fields["reasoning_summary"] = ( + ReasoningSummaryEnum | None, + Field( + description=( + "Requests a summary of the model's reasoning. Supported values " + f"are {reasoning_summary_values}." + ), + default=None, + ), + ) + if verbosity: + fields["verbosity"] = ( + VerbosityEnum | None, + Field( + description=( + "Controls how verbose the model's response should be. Supported " + "values are low, medium, and high." + ), + default=None, + ), + ) + if service_tier: + fields["service_tier"] = ( + str | None, + Field( + description=( + "The processing tier to use for this request - for example " + "'fast' for Fast mode (faster responses at a higher price) " + "or 'flex' for slower, cheaper processing on models that " + "support those tiers." + ), + default=None, + ), + ) + return create_model("Options", __base__=SharedOptions, **fields) -def _attachment(attachment): +def _attachment(attachment, image_detail=None): url = attachment.url base64_content = "" if not url or attachment.resolve_type().startswith("audio/"): @@ -521,7 +1112,10 @@ def _attachment(attachment): }, } if attachment.resolve_type().startswith("image/"): - return {"type": "image_url", "image_url": {"url": url}} + image_url = {"url": url} + if image_detail: + image_url["detail"] = image_detail + return {"type": "image_url", "image_url": image_url} else: format_ = "wav" if attachment.resolve_type() == "audio/wav" else "mp3" return { @@ -534,6 +1128,21 @@ def _attachment(attachment): class _Shared: + # NEVER remove or change an existing entry - only ever append new + # ones. + json_replacements: ClassVar[dict] = { + "completion_tokens_details_0": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details_0": { + "audio_tokens": 0, + "cached_tokens": 0, + }, + } + def __init__( self, model_id, @@ -548,6 +1157,9 @@ def __init__( vision=False, audio=False, reasoning=False, + verbosity=False, + image_detail_original=False, + service_tier=False, supports_schema=False, supports_tools=False, allows_system_prompt=True, @@ -568,8 +1180,13 @@ def __init__( self.attachment_types = set() - if reasoning: - self.Options = OptionsForReasoning + if reasoning or verbosity or image_detail_original or service_tier: + self.Options = build_options_class( + reasoning=reasoning, + verbosity=verbosity, + image_detail_original=image_detail_original, + service_tier=service_tier, + ) if vision: self.attachment_types.update( @@ -591,87 +1208,104 @@ def __init__( ) def __str__(self) -> str: - return "OpenAI Chat: {}".format(self.model_id) - - def build_messages(self, prompt, conversation): - messages = [] - current_system = None - if conversation is not None: - for prev_response in conversation.responses: - if ( - prev_response.prompt.system - and prev_response.prompt.system != current_system - ): - messages.append( - {"role": "system", "content": prev_response.prompt.system} - ) - current_system = prev_response.prompt.system - if prev_response.attachments: - attachment_message = [] - if prev_response.prompt.prompt: - attachment_message.append( - {"type": "text", "text": prev_response.prompt.prompt} - ) - for attachment in prev_response.attachments: - attachment_message.append(_attachment(attachment)) - messages.append({"role": "user", "content": attachment_message}) - elif prev_response.prompt.prompt: - messages.append( - {"role": "user", "content": prev_response.prompt.prompt} - ) - for tool_result in prev_response.prompt.tool_results: - messages.append( - { - "role": "tool", - "tool_call_id": tool_result.tool_call_id, - "content": tool_result.output, - } - ) - prev_text = prev_response.text_or_raise() - if prev_text: - messages.append({"role": "assistant", "content": prev_text}) - tool_calls = prev_response.tool_calls_or_raise() - if tool_calls: - messages.append( - { - "role": "assistant", - "tool_calls": [ - { - "type": "function", - "id": tool_call.tool_call_id, - "function": { - "name": tool_call.name, - "arguments": json.dumps(tool_call.arguments), - }, - } - for tool_call in tool_calls - ], - } - ) - if prompt.system and prompt.system != current_system: - messages.append({"role": "system", "content": prompt.system}) - for tool_result in prompt.tool_results: - messages.append( - { - "role": "tool", - "tool_call_id": tool_result.tool_call_id, - "content": tool_result.output, - } - ) - if not prompt.attachments: - if prompt.prompt: - messages.append({"role": "user", "content": prompt.prompt or ""}) - else: - attachment_message = [] - if prompt.prompt: - attachment_message.append({"type": "text", "text": prompt.prompt}) - for attachment in prompt.attachments: - attachment_message.append(_attachment(attachment)) - messages.append({"role": "user", "content": attachment_message}) - return messages + return f"OpenAI Chat: {self.model_id}" + + def _append_llm_message(self, out, message, current_system, image_detail=None): + """Translate one llm.Message into one (or more) OpenAI message + dicts and append them to ``out``. + + Returns the (possibly updated) current_system value so the caller + can avoid re-emitting an unchanged system prompt. + """ + from llm.parts import ( + AttachmentPart, + TextPart, + ToolCallPart, + ToolResultPart, + ) - def set_usage(self, response, usage): - if not usage: + text_bits = [] + attachment_items = [] + tool_calls = [] + tool_results = [] + + for part in message.parts: + if isinstance(part, TextPart): + text_bits.append(part.text) + elif isinstance(part, AttachmentPart) and part.attachment: + attachment_items.append( + _attachment(part.attachment, image_detail=image_detail) + ) + elif isinstance(part, ToolCallPart): + tool_calls.append( + { + "type": "function", + "id": part.tool_call_id, + "function": { + "name": part.name, + "arguments": json.dumps(part.arguments), + }, + } + ) + elif isinstance(part, ToolResultPart): + tool_results.append( + { + "role": "tool", + "tool_call_id": part.tool_call_id, + "content": part.output, + } + ) + + # Role "tool" emits one OpenAI "tool" message per ToolResultPart. + if message.role == "tool": + out.extend(tool_results) + return current_system + + # System dedup: skip if this text is already the active system prompt. + if message.role == "system": + text = "".join(text_bits) + if text == current_system: + return current_system + current_system = text + + if attachment_items: + content = [] + if text_bits: + content.append({"type": "text", "text": "".join(text_bits)}) + content.extend(attachment_items) + entry = {"role": message.role, "content": content} + else: + entry = { + "role": message.role, + "content": "".join(text_bits) if text_bits else None, + } + + if tool_calls: + entry["tool_calls"] = tool_calls + # OpenAI expects content=null when only tool_calls are present. + if not text_bits: + entry["content"] = None + elif entry["content"] is None and message.role != "assistant": + # For user/system, an empty message is pointless — drop it. + return current_system + + out.append(entry) + return current_system + + def build_messages(self, prompt, conversation, image_detail=None): + """Translate prompt.messages into OpenAI's wire format.""" + messages: list[dict[str, Any]] = [] + if image_detail is not None: + image_detail = image_detail.value + current_system: str | None = None + for msg in prompt.messages: + current_system = self._append_llm_message( + messages, msg, current_system, image_detail=image_detail + ) + return messages + + def set_usage(self, response, usage): + if not usage: return input_tokens = usage.pop("prompt_tokens") output_tokens = usage.pop("completion_tokens") @@ -708,6 +1342,11 @@ def get_client(self, key, *, async_=False): def build_kwargs(self, prompt, stream): kwargs = dict(not_nulls(prompt.options)) json_object = kwargs.pop("json_object", None) + kwargs.pop("image_detail", None) + kwargs.pop("chat_completions", None) + # Responses models reuse their Options object when explicitly routed + # through the Chat Completions compatibility path. + kwargs.pop("reasoning_summary", None) if "max_tokens" not in kwargs and self.default_max_tokens is not None: kwargs["max_tokens"] = self.default_max_tokens if json_object: @@ -739,23 +1378,23 @@ class Chat(_Shared, KeyModel): key_env_var = "OPENAI_API_KEY" default_max_tokens = None - class Options(SharedOptions): - json_object: Optional[bool] = Field( - description="Output a valid JSON object {...}. Prompt must mention JSON.", - default=None, - ) + Options = build_options_class() def execute( self, prompt: Prompt, stream: bool, response: Response, - conversation: Optional[Conversation] = None, - key: Optional[str] = None, - ) -> Iterator[str]: + conversation: Conversation | None = None, + key: str | None = None, + ) -> Iterator[str | StreamEvent]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") - messages = self.build_messages(prompt, conversation) + messages = self.build_messages( + prompt, + conversation, + image_detail=getattr(prompt.options, "image_detail", None), + ) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key) usage = None @@ -776,29 +1415,40 @@ def execute( for tool_call in chunk.choices[0].delta.tool_calls or []: if tool_call.function.arguments is None: tool_call.function.arguments = "" - index = tool_call.index - if index not in tool_calls: - tool_calls[index] = tool_call + idx = tool_call.index + if idx not in tool_calls: + tool_calls[idx] = tool_call + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + tool_call_id=tool_call.id, + ) else: tool_calls[ - index + idx ].function.arguments += tool_call.function.arguments + if tool_call.function.arguments: + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments, + tool_call_id=tool_calls[idx].id, + ) try: content = chunk.choices[0].delta.content except IndexError: content = None - if content is not None: - yield content + if content: + # Empty strings are noise (OpenAI's first chunk + # with role=assistant has content=""). + yield StreamEvent(type="text", chunk=content) response.response_json = remove_dict_none_values(combine_chunks(chunks)) if tool_calls: for value in tool_calls.values(): - # value.function looks like this: - # ChoiceDeltaToolCallFunction(arguments='{"city":"San Francisco"}', name='get_weather') response.add_tool_call( llm.ToolCall( tool_call_id=value.id, name=value.function.name, - arguments=json.loads(value.function.arguments), + arguments=json.loads(value.function.arguments or "{}"), ) ) else: @@ -815,12 +1465,29 @@ def execute( llm.ToolCall( tool_call_id=tool_call.id, name=tool_call.function.name, - arguments=json.loads(tool_call.function.arguments), + arguments=json.loads(tool_call.function.arguments or "{}"), ) ) + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + tool_call_id=tool_call.id, + ) + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments or "", + tool_call_id=tool_call.id, + ) if completion.choices[0].message.content is not None: - yield completion.choices[0].message.content + yield StreamEvent( + type="text", + chunk=completion.choices[0].message.content, + ) self.set_usage(response, usage) + if usage and (usage.get("completion_tokens_details") or {}).get( + "reasoning_tokens" + ): + yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data({"messages": messages}) @@ -829,23 +1496,23 @@ class AsyncChat(_Shared, AsyncKeyModel): key_env_var = "OPENAI_API_KEY" default_max_tokens = None - class Options(SharedOptions): - json_object: Optional[bool] = Field( - description="Output a valid JSON object {...}. Prompt must mention JSON.", - default=None, - ) + Options = build_options_class() async def execute( self, prompt: Prompt, stream: bool, response: AsyncResponse, - conversation: Optional[AsyncConversation] = None, - key: Optional[str] = None, - ) -> AsyncGenerator[str, None]: + conversation: AsyncConversation | None = None, + key: str | None = None, + ) -> AsyncGenerator[str | StreamEvent, None]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") - messages = self.build_messages(prompt, conversation) + messages = self.build_messages( + prompt, + conversation, + image_detail=getattr(prompt.options, "image_detail", None), + ) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key, async_=True) usage = None @@ -862,34 +1529,41 @@ async def execute( if chunk.usage: usage = chunk.usage.model_dump() chunks.append(chunk) - if chunk.usage: - usage = chunk.usage.model_dump() if chunk.choices and chunk.choices[0].delta: for tool_call in chunk.choices[0].delta.tool_calls or []: if tool_call.function.arguments is None: tool_call.function.arguments = "" - index = tool_call.index - if index not in tool_calls: - tool_calls[index] = tool_call + idx = tool_call.index + if idx not in tool_calls: + tool_calls[idx] = tool_call + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + tool_call_id=tool_call.id, + ) else: tool_calls[ - index + idx ].function.arguments += tool_call.function.arguments + if tool_call.function.arguments: + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments, + tool_call_id=tool_calls[idx].id, + ) try: content = chunk.choices[0].delta.content except IndexError: content = None - if content is not None: - yield content + if content: + yield StreamEvent(type="text", chunk=content) if tool_calls: for value in tool_calls.values(): - # value.function looks like this: - # ChoiceDeltaToolCallFunction(arguments='{"city":"San Francisco"}', name='get_weather') response.add_tool_call( llm.ToolCall( tool_call_id=value.id, name=value.function.name, - arguments=json.loads(value.function.arguments), + arguments=json.loads(value.function.arguments or "{}"), ) ) response.response_json = remove_dict_none_values(combine_chunks(chunks)) @@ -907,18 +1581,1333 @@ async def execute( llm.ToolCall( tool_call_id=tool_call.id, name=tool_call.function.name, - arguments=json.loads(tool_call.function.arguments), + arguments=json.loads(tool_call.function.arguments or "{}"), ) ) + yield StreamEvent( + type="tool_call_name", + chunk=tool_call.function.name or "", + tool_call_id=tool_call.id, + ) + yield StreamEvent( + type="tool_call_args", + chunk=tool_call.function.arguments or "", + tool_call_id=tool_call.id, + ) if completion.choices[0].message.content is not None: - yield completion.choices[0].message.content + yield StreamEvent( + type="text", + chunk=completion.choices[0].message.content, + ) self.set_usage(response, usage) + if usage and (usage.get("completion_tokens_details") or {}).get( + "reasoning_tokens" + ): + yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data({"messages": messages}) +def _responses_attachment(attachment, image_detail=None): + """Translate an llm Attachment into a Responses-API content part.""" + url = attachment.url + base64_content = "" + if not url or attachment.resolve_type().startswith("audio/"): + base64_content = attachment.base64_content() + url = f"data:{attachment.resolve_type()};base64,{base64_content}" + if attachment.resolve_type() == "application/pdf": + if not base64_content: + base64_content = attachment.base64_content() + return { + "type": "input_file", + "filename": f"{attachment.id()}.pdf", + "file_data": f"data:application/pdf;base64,{base64_content}", + } + if attachment.resolve_type().startswith("image/"): + item = {"type": "input_image", "image_url": url} + if image_detail: + item["detail"] = image_detail + return item + # Audio is not yet supported on the Responses input shape we use; fall + # back to image_url for unknown types so we don't silently drop content. + return {"type": "input_image", "image_url": url} + + +class WebSearch(llm.ServerSideTool): + """Search the web using OpenAI's hosted search tool. + + Configure domain filters, approximate location, result context, live web + access and image search through constructor arguments. Set + ``include_sources`` to retain every consulted URL or ``include_results`` + to retain raw results such as image search results. + """ + + name = "web_search" + _search_context_sizes = frozenset({"low", "medium", "high"}) + _return_token_budgets = frozenset({"default", "unlimited"}) + _search_content_types = frozenset({"text", "image"}) + + def __init__( + self, + allowed_domains: list[str] | None = None, + blocked_domains: list[str] | None = None, + user_location: dict | None = None, + search_context_size: Literal["low", "medium", "high"] | None = None, + external_web_access: bool | None = None, + return_token_budget: Literal["default", "unlimited"] | None = None, + search_content_types: list[Literal["text", "image"]] | None = None, + image_settings: dict | None = None, + include_sources: bool = False, + include_results: bool = False, + ): + super().__init__() + self.allowed_domains = self._validate_domains( + "allowed_domains", allowed_domains + ) + self.blocked_domains = self._validate_domains( + "blocked_domains", blocked_domains + ) + if ( + search_context_size is not None + and search_context_size not in self._search_context_sizes + ): + raise ValueError("search_context_size must be one of: low, medium or high") + if external_web_access is not None and not isinstance( + external_web_access, bool + ): + raise TypeError("external_web_access must be a boolean") + if ( + return_token_budget is not None + and return_token_budget not in self._return_token_budgets + ): + raise ValueError("return_token_budget must be default or unlimited") + if search_content_types is not None: + if not isinstance(search_content_types, list): + raise TypeError("search_content_types must be a list") + invalid_content_types = set(search_content_types).difference( + self._search_content_types + ) + if invalid_content_types: + raise ValueError("search_content_types must contain text and/or image") + if user_location is not None: + if not isinstance(user_location, dict): + raise TypeError("user_location must be a dictionary") + user_location = dict(user_location) + user_location.setdefault("type", "approximate") + if user_location["type"] != "approximate": + raise ValueError("user_location type must be approximate") + if image_settings is not None: + if not isinstance(image_settings, dict): + raise TypeError("image_settings must be a dictionary") + image_settings = dict(image_settings) + max_results = image_settings.get("max_results") + if max_results is not None and ( + isinstance(max_results, bool) + or not isinstance(max_results, int) + or max_results < 1 + ): + raise ValueError( + "image_settings max_results must be a positive integer" + ) + caption = image_settings.get("caption") + if caption is not None and not isinstance(caption, bool): + raise TypeError("image_settings caption must be a boolean") + if not isinstance(include_sources, bool): + raise TypeError("include_sources must be a boolean") + if not isinstance(include_results, bool): + raise TypeError("include_results must be a boolean") + self.user_location = user_location + self.search_context_size = search_context_size + self.external_web_access = external_web_access + self.return_token_budget = return_token_budget + self.search_content_types = ( + list(search_content_types) if search_content_types is not None else None + ) + self.image_settings = image_settings + self.include_sources = include_sources + self.include_results = include_results + + @staticmethod + def _validate_domains(name, domains): + if domains is None: + return None + if not isinstance(domains, list): + raise TypeError(f"{name} must be a list") + if len(domains) > 100: + raise ValueError(f"{name} cannot contain more than 100 domains") + for domain in domains: + if not isinstance(domain, str) or not domain: + raise TypeError(f"{name} entries must be non-empty strings") + if domain.lower().startswith(("http://", "https://")): + raise ValueError(f"{name} entries must omit the URL scheme") + return list(domains) + + def tool_spec(self, model): + spec = {"type": "web_search"} + if self.allowed_domains is not None or self.blocked_domains is not None: + filters = {} + if self.allowed_domains is not None: + filters["allowed_domains"] = list(self.allowed_domains) + if self.blocked_domains is not None: + filters["blocked_domains"] = list(self.blocked_domains) + spec["filters"] = filters + for key in ( + "user_location", + "search_context_size", + "external_web_access", + "return_token_budget", + "search_content_types", + "image_settings", + ): + value = getattr(self, key) + if value is not None: + if isinstance(value, dict): + value = dict(value) + elif isinstance(value, list): + value = list(value) + spec[key] = value + return spec + + def prepare_request(self, model, kwargs): + if not self.include_sources and not self.include_results: + return + include = kwargs.setdefault("include", []) + if self.include_sources and "web_search_call.action.sources" not in include: + include.append("web_search_call.action.sources") + if self.include_results and "web_search_call.results" not in include: + include.append("web_search_call.results") + + +class CodeInterpreter(llm.ServerSideTool): + """Run Python in an OpenAI-managed container. + + With no ``container`` argument OpenAI creates or reuses an automatic + container. ``memory_limit`` and ``file_ids`` configure that automatic + container. Pass an existing ``cntr_`` ID as ``container`` to use it + explicitly instead. + """ + + name = "code_interpreter" + _memory_limits = frozenset({"1g", "4g", "16g", "64g"}) + + def __init__( + self, + container: str | None = None, + memory_limit: Literal["1g", "4g", "16g", "64g"] | None = None, + file_ids: list[str] | None = None, + ): + super().__init__() + if container is not None and not isinstance(container, str): + raise TypeError("container must be a string container ID") + if memory_limit is not None and memory_limit not in self._memory_limits: + raise ValueError("memory_limit must be one of: 1g, 4g, 16g or 64g") + if container is not None and (memory_limit is not None or file_ids is not None): + raise ValueError( + "container cannot be combined with memory_limit or file_ids" + ) + self.container = container + self.memory_limit = memory_limit + self.file_ids = list(file_ids) if file_ids is not None else None + + def tool_spec(self, model): + if self.container is not None: + return {"type": "code_interpreter", "container": self.container} + container = {"type": "auto"} + if self.memory_limit is not None: + container["memory_limit"] = self.memory_limit + if self.file_ids is not None: + container["file_ids"] = list(self.file_ids) + return {"type": "code_interpreter", "container": container} + + def prepare_request(self, model, kwargs): + include = kwargs.setdefault("include", []) + if "code_interpreter_call.outputs" not in include: + include.append("code_interpreter_call.outputs") + + +class _SharedResponses(_Shared): + """Mixin that translates llm.Prompt into Responses API parameters.""" + + @property + def supported_server_side_tools(self): + return (WebSearch, CodeInterpreter, llm.ServerSideTool) + + # Recurring boilerplate in Responses API payloads. Same contract as + # _Shared.json_replacements, which this replaces for Responses + # models: NEVER remove or change an existing entry - only ever + # append new ones. + json_replacements: ClassVar[dict] = { + "tool_usage_0": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0, + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0, + }, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "input_tokens_details_0": { + "cached_tokens": 0, + "cache_write_tokens": 0, + }, + "reasoning_settings_0": { + "effort": "medium", + "summary": "detailed", + "context": "all_turns", + "mode": "standard", + }, + "reasoning_settings_1": { + "effort": "medium", + "summary": "detailed", + "context": "current_turn", + "mode": "standard", + }, + # The default text block on non-schema replies + "text_format_0": {"format": {"type": "text"}, "verbosity": "medium"}, + # The static envelope of a Responses payload + "response_env_0": { + "object": "response", + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "top_p": 1.0, + "background": False, + "service_tier": "default", + "status": "completed", + "top_logprobs": 0, + "truncation": "disabled", + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "store": False, + "tools": [], + }, + "message_completed": { + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + }, + } + + def __str__(self) -> str: + return f"OpenAI Responses: {self.model_id}" + + def _delegate_chat_kwargs(self): + """Return constructor kwargs that mirror this Responses model so we + can build a sibling Chat / AsyncChat instance for the + ``-o chat_completions 1`` opt-out path.""" + return { + "model_id": self.model_id, + "key": self.key, + "model_name": self.model_name, + "api_base": self.api_base, + "api_type": self.api_type, + "api_version": self.api_version, + "api_engine": self.api_engine, + "headers": self.headers, + "can_stream": self.can_stream, + "vision": self.vision, + "reasoning": self._reasoning, + "verbosity": self._verbosity, + "image_detail_original": self._image_detail_original, + "service_tier": self._service_tier, + "supports_schema": self.supports_schema, + "supports_tools": self.supports_tools, + "allows_system_prompt": self.allows_system_prompt, + } + + def _build_responses_input(self, prompt, image_detail=None): + """Translate prompt.messages into a (input_items, instructions) tuple + for the Responses API. + + The most recent system Message is hoisted into ``instructions``; + earlier system messages are dropped (mirroring the way the Chat + path collapses repeated identical system prompts). + """ + from llm.parts import ( + AttachmentPart, + ReasoningPart, + TextPart, + ToolCallPart, + ToolResultPart, + ) + + items: list[dict[str, Any]] = [] + instructions: str | None = None + + for msg in prompt.messages: + if msg.role == "system": + text = "".join(p.text for p in msg.parts if isinstance(p, TextPart)) + if text: + instructions = text + continue + + text_bits: list[str] = [] + attachment_items: list[dict[str, Any]] = [] + tool_call_items: list[dict[str, Any]] = [] + tool_result_items: list[dict[str, Any]] = [] + reasoning_items: list[dict[str, Any]] = [] + + for part in msg.parts: + if isinstance(part, TextPart): + text_bits.append(part.text) + elif isinstance(part, AttachmentPart) and part.attachment: + attachment_items.append( + _responses_attachment( + part.attachment, image_detail=image_detail + ) + ) + elif isinstance(part, ToolCallPart): + if part.server_executed: + # Server-side tool calls (web_search, + # code_interpreter) ran inside OpenAI's + # infrastructure - they must not be replayed as + # client function_call items. + continue + tool_call_items.append( + { + "type": "function_call", + "call_id": part.tool_call_id, + "name": part.name, + "arguments": json.dumps(part.arguments), + } + ) + elif isinstance(part, ToolResultPart): + if part.server_executed: + continue + tool_result_items.append( + { + "type": "function_call_output", + "call_id": part.tool_call_id, + "output": part.output, + } + ) + elif isinstance(part, ReasoningPart): + pm = (part.provider_metadata or {}).get("openai") or {} + enc = pm.get("encrypted_content") + rid = pm.get("id") + if enc or rid: + # Round-trip a previous reasoning item so the model + # can pick up where it left off mid-tool-call. + item: dict[str, Any] = {"type": "reasoning"} + if rid: + item["id"] = rid + if enc: + item["encrypted_content"] = enc + if pm.get("summary"): + item["summary"] = pm["summary"] + else: + item["summary"] = [] + reasoning_items.append(item) + + # Reasoning items must precede the assistant message / function + # call they belonged to. + items.extend(reasoning_items) + + if msg.role == "tool": + items.extend(tool_result_items) + continue + + if msg.role == "user": + if attachment_items: + content: list[dict[str, Any]] = [] + if text_bits: + content.append( + {"type": "input_text", "text": "".join(text_bits)} + ) + content.extend(attachment_items) + items.append({"role": "user", "content": content}) + elif text_bits: + items.append({"role": "user", "content": "".join(text_bits)}) + elif msg.role == "assistant": + if text_bits: + items.append({"role": "assistant", "content": "".join(text_bits)}) + items.extend(tool_call_items) + + return items, instructions + + def _build_responses_kwargs(self, prompt, stream): + """Build the keyword arguments for client.responses.create().""" + opts = dict(not_nulls(prompt.options)) + # Strip options that are either internal to llm or not accepted by + # the Responses API. + opts.pop("json_object", None) + opts.pop("chat_completions", None) + opts.pop("image_detail", None) + max_tokens = opts.pop("max_tokens", None) + reasoning_effort = opts.pop("reasoning_effort", None) + reasoning_summary = opts.pop("reasoning_summary", None) + verbosity = opts.pop("verbosity", None) + temperature = opts.pop("temperature", None) + top_p = opts.pop("top_p", None) + seed = opts.pop("seed", None) + + kwargs: dict[str, Any] = {} + if max_tokens is None and self.default_max_tokens is not None: + max_tokens = self.default_max_tokens + if max_tokens is not None: + kwargs["max_output_tokens"] = max_tokens + if temperature is not None: + kwargs["temperature"] = temperature + if top_p is not None: + kwargs["top_p"] = top_p + if seed is not None: + kwargs["seed"] = seed + if self._reasoning: + reasoning = {} + if not getattr(prompt, "hide_reasoning", False): + if reasoning_summary is not None: + reasoning["summary"] = reasoning_summary + elif self._reasoning_summary: + reasoning["summary"] = "auto" + if reasoning_effort: + reasoning["effort"] = reasoning_effort + if reasoning: + kwargs["reasoning"] = reasoning + + text: dict[str, Any] = {} + if verbosity: + text["verbosity"] = verbosity + if prompt.options.json_object: + text["format"] = {"type": "json_object"} + if prompt.schema: + # ``strict: False`` mirrors the looser behaviour of the + # /v1/chat/completions json_schema response_format - required + # because the Responses API otherwise insists on + # ``additionalProperties: false`` everywhere. + text["format"] = { + "type": "json_schema", + "name": "output", + "schema": prompt.schema, + "strict": False, + } + if text: + kwargs["text"] = text + + if prompt.tools: + _partition_tools(self, prompt.tools) + kwargs["tools"] = [ + ( + { + "type": "function", + "name": tool.name, + "description": tool.description or None, + "parameters": tool.input_schema, + } + if isinstance(tool, llm.Tool) + else tool.tool_spec(self) + ) + for tool in prompt.tools + ] + + # Pass anything we did not consume through verbatim - this lets + # extras like ``parallel_tool_calls`` flow into the API. + kwargs.update(opts) + return kwargs + + def _finalize_responses_kwargs(self, prompt, stream, instructions=None): + """Build complete request kwargs, then run server-tool hooks in order.""" + kwargs = self._build_responses_kwargs(prompt, stream) + if instructions is not None: + kwargs["instructions"] = instructions + kwargs["store"] = False + if self._reasoning and ( + self._reasoning_summary + or getattr(prompt.options, "reasoning_summary", None) + or getattr(prompt.options, "reasoning_effort", None) + ): + include = kwargs.setdefault("include", []) + if "reasoning.encrypted_content" not in include: + include.append("reasoning.encrypted_content") + _, server_side_tools = _partition_tools(self, prompt.tools) + for tool in server_side_tools: + tool.prepare_request(self, kwargs) + return kwargs + + def _set_usage_responses(self, response, usage): + if not usage: + return + input_tokens = usage.get("input_tokens", 0) or 0 + output_tokens = usage.get("output_tokens", 0) or 0 + details = {} + for key in ("input_tokens_details", "output_tokens_details"): + value = usage.get(key) + if value: + details[key] = value + response.set_usage( + input=input_tokens, output=output_tokens, details=details or None + ) + + def _reasoning_text_from_item(self, item): + bits = [] + for attr in ("summary", "content"): + for part in getattr(item, attr, None) or []: + if isinstance(part, dict): + text = part.get("text") + else: + text = getattr(part, "text", None) + if text: + bits.append(text) + return "".join(bits) + + def _reasoning_event(self, item, *, include_text=True): + """Build a redacted-reasoning StreamEvent that carries the opaque + ``id`` and ``encrypted_content`` from a Responses-API reasoning + item. Echoing this metadata back on the next request via + ``_build_responses_input`` lets the model pick up its prior chain + of thought - critical for tool-using reasoning models, since + without it the model loses ~3% on SWE-bench (per OpenAI).""" + rid = getattr(item, "id", None) + enc = getattr(item, "encrypted_content", None) + summary = getattr(item, "summary", None) + text = self._reasoning_text_from_item(item) if include_text else "" + meta: dict[str, Any] = {} + if rid: + meta["id"] = rid + if enc: + meta["encrypted_content"] = enc + if summary: + # ``summary`` is a list of {type:"summary_text", text:"..."} + # objects when reasoning summaries are enabled. + try: + meta["summary"] = [ + s.model_dump() if hasattr(s, "model_dump") else dict(s) + for s in summary + ] + except Exception: # noqa: BLE001 + meta["summary"] = list(summary) + return StreamEvent( + type="reasoning", + chunk=text, + redacted=include_text and not text, + provider_metadata={"openai": meta} if meta else None, + ) + + def _reasoning_refresh_events(self, response_json, done_events): + """Metadata-only reasoning events rebuilt from the final payload. + + While streaming, reasoning metadata is first harvested from the + ``response.output_item.done`` event, but the ``response.completed`` + payload carries a *different* ciphertext of the same reasoning - + OpenAI encrypts per event. Re-emitting the metadata from the + final payload, aimed at the already-resolved part_index, makes + the stored part and ``response_json`` agree on one blob (which + also lets the log store condense the payload against the part). + + ``done_events`` maps reasoning item id to the StreamEvent + yielded at ``output_item.done``; the framework has resolved + ``part_index`` on it by the time the stream ends. + """ + events = [] + for item in response_json.get("output") or []: + if not isinstance(item, dict) or item.get("type") != "reasoning": + continue + prior = done_events.get(item.get("id")) + if prior is None or prior.part_index is None: + continue + meta = { + key: item[key] + for key in ("id", "encrypted_content", "summary") + if item.get(key) + } + if meta: + events.append( + StreamEvent( + type="reasoning", + chunk="", + part_index=prior.part_index, + provider_metadata={"openai": meta}, + message_index=prior.message_index, + ) + ) + return events + + def _server_tool_events(self, item, message_index): + """StreamEvents for a server-side tool call output item + (web_search_call / code_interpreter_call), or [] for other + item types. The call and its result both carry + ``server_executed=True`` so they are recorded in the message + parts without entering the locally-executable tool call list. + """ + item_type = getattr(item, "type", None) + item_id = getattr(item, "id", None) + events: list[StreamEvent] = [] + if item_type == "web_search_call": + action = getattr(item, "action", None) + if hasattr(action, "model_dump"): + action = action.model_dump() + events.append( + StreamEvent( + type="tool_call_name", + chunk="web_search", + tool_call_id=item_id, + server_executed=True, + message_index=message_index, + ) + ) + events.append( + StreamEvent( + type="tool_call_args", + chunk=json.dumps(action or {}), + tool_call_id=item_id, + server_executed=True, + message_index=message_index, + ) + ) + results = getattr(item, "results", None) or [] + results = [ + result.model_dump() if hasattr(result, "model_dump") else result + for result in results + ] + events.append( + StreamEvent( + type="tool_result", + chunk=( + json.dumps(results) + if results + else (getattr(item, "status", None) or "completed") + ), + tool_call_id=item_id, + server_executed=True, + tool_name="web_search", + message_index=message_index, + ) + ) + elif item_type == "code_interpreter_call": + code = getattr(item, "code", None) or "" + events.append( + StreamEvent( + type="tool_call_name", + chunk="code_interpreter", + tool_call_id=item_id, + server_executed=True, + message_index=message_index, + ) + ) + events.append( + StreamEvent( + type="tool_call_args", + chunk=json.dumps({"code": code}), + tool_call_id=item_id, + server_executed=True, + message_index=message_index, + ) + ) + output_bits = [] + for output in getattr(item, "outputs", None) or []: + if hasattr(output, "model_dump"): + output = output.model_dump() + if isinstance(output, dict): + text = output.get("logs") or output.get("url") + if text: + output_bits.append(text) + events.append( + StreamEvent( + type="tool_result", + chunk="\n".join(output_bits) + or (getattr(item, "status", None) or "completed"), + tool_call_id=item_id, + server_executed=True, + tool_name="code_interpreter", + message_index=message_index, + ) + ) + return events + + def _refresh_server_tool_events(self, output, done_events): + """Replace streamed server-tool payloads with their final values. + + OpenAI can return incomplete sources, results or outputs on a + ``response.output_item.done`` event and then provide the complete + item on ``response.completed``. The response stores yielded + StreamEvent objects by reference, so updating their chunks here + corrects the assembled Parts without emitting duplicate events. + """ + for item in output or []: + item_id = getattr(item, "id", None) + prior_events = done_events.get(item_id) + if not prior_events: + continue + final_events = { + event.type: event + for event in self._server_tool_events( + item, prior_events[0].message_index + ) + } + for prior_event in prior_events: + final_event = final_events.get(prior_event.type) + if final_event is not None: + prior_event.chunk = final_event.chunk + + def _non_streaming_output_events(self, output, response): + """Translate a non-streaming Responses ``output`` item list into + StreamEvents. Returns ``(events, had_reasoning)``. + + Each ``message`` item after the first starts a new + ``message_index``, so server-side tool execution that + interleaves multiple message items assembles into multiple + assistant Messages. Items between two message items (tool + calls, reasoning) group with the preceding message. + """ + events: list[StreamEvent] = [] + had_reasoning = False + message_index = 0 + seen_message = False + for item in output: + if item.type == "message" and seen_message: + message_index += 1 + if item.type == "reasoning": + had_reasoning = True + event = self._reasoning_event(item) + event.message_index = message_index + events.append(event) + elif item.type == "function_call": + try: + args = json.loads(item.arguments) if item.arguments else {} + except json.JSONDecodeError: + args = {"_raw": item.arguments} + response.add_tool_call( + llm.ToolCall( + tool_call_id=item.call_id, + name=item.name, + arguments=args, + ) + ) + events.append( + StreamEvent( + type="tool_call_name", + chunk=item.name or "", + tool_call_id=item.call_id, + message_index=message_index, + ) + ) + events.append( + StreamEvent( + type="tool_call_args", + chunk=item.arguments or "", + tool_call_id=item.call_id, + message_index=message_index, + ) + ) + elif item.type == "message": + seen_message = True + for content in item.content or []: + ctype = getattr(content, "type", None) + if ctype == "output_text" and content.text: + events.append( + StreamEvent( + type="text", + chunk=content.text, + message_index=message_index, + ) + ) + else: + events.extend(self._server_tool_events(item, message_index)) + return events, had_reasoning + + +class Responses(_SharedResponses, KeyModel): + needs_key = "openai" + key_env_var = "OPENAI_API_KEY" + default_max_tokens = None + + def __init__( + self, + model_id, + key=None, + model_name=None, + api_base=None, + api_type=None, + api_version=None, + api_engine=None, + headers=None, + can_stream=True, + vision=False, + audio=False, + reasoning=False, + verbosity=False, + image_detail_original=False, + service_tier=False, + supports_schema=False, + supports_tools=False, + allows_system_prompt=True, + reasoning_summary=True, + ): + super().__init__( + model_id, + key=key, + model_name=model_name, + api_base=api_base, + api_type=api_type, + api_version=api_version, + api_engine=api_engine, + headers=headers, + can_stream=can_stream, + vision=vision, + audio=audio, + reasoning=reasoning, + verbosity=verbosity, + image_detail_original=image_detail_original, + service_tier=service_tier, + supports_schema=supports_schema, + supports_tools=supports_tools, + allows_system_prompt=allows_system_prompt, + ) + self._reasoning = reasoning + self._reasoning_summary = reasoning_summary + self._verbosity = verbosity + self._image_detail_original = image_detail_original + self._service_tier = service_tier + # Override the Options class so that ``-o chat_completions 1`` is + # always available on Responses-routed models. + self.Options = build_options_class( + reasoning=reasoning, + reasoning_summary=reasoning, + verbosity=verbosity, + image_detail_original=image_detail_original, + chat_completions=True, + service_tier=service_tier, + ) + + def execute( + self, + prompt: Prompt, + stream: bool, + response: Response, + conversation: Conversation | None = None, + key: str | None = None, + ) -> Iterator[str | StreamEvent]: + if getattr(prompt.options, "chat_completions", None): + chat = Chat(**self._delegate_chat_kwargs()) + _partition_tools(chat, prompt.tools) + yield from chat.execute(prompt, stream, response, conversation, key) + return + + if prompt.system and not self.allows_system_prompt: + raise NotImplementedError("Model does not support system prompts") + + image_detail = getattr(prompt.options, "image_detail", None) + if image_detail is not None: + image_detail = image_detail.value + input_items, instructions = self._build_responses_input( + prompt, image_detail=image_detail + ) + kwargs = self._finalize_responses_kwargs(prompt, stream, instructions) + + client = self.get_client(key) + usage = None + had_reasoning = False + if stream: + stream_obj = client.responses.create( + model=self.model_name or self.model_id, + input=input_items, + stream=True, + **kwargs, + ) + tool_call_meta: dict[str, dict[str, str]] = {} + final_response_dict: dict[str, Any] | None = None + reasoning_items_with_streamed_text = set() + reasoning_done_events: dict[str, StreamEvent] = {} + server_tool_done_events: dict[str, list[StreamEvent]] = {} + message_index = 0 + seen_message = False + for event in stream_obj: + etype = getattr(event, "type", None) + if etype == "response.output_item.added": + item = event.item + if item.type == "message": + if seen_message: + message_index += 1 + seen_message = True + elif item.type == "function_call": + tool_call_meta[item.id] = { + "id": item.id, + "call_id": item.call_id, + "name": item.name, + } + yield StreamEvent( + type="tool_call_name", + chunk=item.name or "", + tool_call_id=item.call_id, + message_index=message_index, + ) + elif etype == "response.output_text.delta": + yield StreamEvent( + type="text", + chunk=event.delta or "", + message_index=message_index, + ) + elif etype == "response.function_call_arguments.delta": + item_id = getattr(event, "item_id", None) + meta = tool_call_meta.get(item_id) if item_id else None + call_id = meta["call_id"] if meta else None + yield StreamEvent( + type="tool_call_args", + chunk=event.delta or "", + tool_call_id=call_id, + message_index=message_index, + ) + elif etype in ( + "response.reasoning_summary_text.delta", + "response.reasoning_text.delta", + ): + item_id = getattr(event, "item_id", None) + if item_id: + reasoning_items_with_streamed_text.add(item_id) + yield StreamEvent( + type="reasoning", + chunk=event.delta or "", + message_index=message_index, + ) + elif etype in ( + "response.reasoning_summary_text.done", + "response.reasoning_text.done", + ): + item_id = getattr(event, "item_id", None) + if item_id not in reasoning_items_with_streamed_text: + text = getattr(event, "text", None) or "" + if text: + if item_id: + reasoning_items_with_streamed_text.add(item_id) + yield StreamEvent( + type="reasoning", + chunk=text, + message_index=message_index, + ) + elif etype == "response.output_item.done": + item = event.item + if item.type == "reasoning": + had_reasoning = True + item_id = getattr(item, "id", None) + reasoning_event = self._reasoning_event( + item, + include_text=( + item_id not in reasoning_items_with_streamed_text + ), + ) + reasoning_event.message_index = message_index + if item_id: + # Retained so the refresh after + # response.completed can target the part + # this event resolved to. + reasoning_done_events[item_id] = reasoning_event + yield reasoning_event + elif item.type == "function_call": + try: + args = json.loads(item.arguments) if item.arguments else {} + except json.JSONDecodeError: + args = {"_raw": item.arguments} + response.add_tool_call( + llm.ToolCall( + tool_call_id=item.call_id, + name=item.name, + arguments=args, + ) + ) + else: + server_events = self._server_tool_events(item, message_index) + item_id = getattr(item, "id", None) + if item_id and server_events: + server_tool_done_events[item_id] = server_events + yield from server_events + elif etype == "response.completed": + self._refresh_server_tool_events( + event.response.output, server_tool_done_events + ) + final_response_dict = event.response.model_dump(warnings=False) + if final_response_dict.get("usage"): + usage = final_response_dict["usage"] + if final_response_dict is not None: + response.response_json = remove_dict_none_values(final_response_dict) + yield from self._reasoning_refresh_events( + response.response_json, reasoning_done_events + ) + else: + completion = client.responses.create( + model=self.model_name or self.model_id, + input=input_items, + stream=False, + **kwargs, + ) + dumped = completion.model_dump(warnings=False) + response.response_json = remove_dict_none_values(dumped) + usage = dumped.get("usage") + events, had_reasoning = self._non_streaming_output_events( + completion.output, response + ) + yield from events + + self._set_usage_responses(response, usage) + # Fallback: usage said reasoning happened but the API gave us no + # reasoning items to harvest encrypted_content from. Emit the + # opaque "reasoning happened" marker for UI / token accounting. + if ( + not had_reasoning + and usage + and ((usage.get("output_tokens_details") or {}).get("reasoning_tokens")) + ): + yield StreamEvent(type="reasoning", chunk="", redacted=True) + response._prompt_json = redact_data( + {"input": input_items, "instructions": instructions} + ) + + +class AsyncResponses(_SharedResponses, AsyncKeyModel): + needs_key = "openai" + key_env_var = "OPENAI_API_KEY" + default_max_tokens = None + + def __init__( + self, + model_id, + key=None, + model_name=None, + api_base=None, + api_type=None, + api_version=None, + api_engine=None, + headers=None, + can_stream=True, + vision=False, + audio=False, + reasoning=False, + verbosity=False, + image_detail_original=False, + service_tier=False, + supports_schema=False, + supports_tools=False, + allows_system_prompt=True, + reasoning_summary=True, + ): + super().__init__( + model_id, + key=key, + model_name=model_name, + api_base=api_base, + api_type=api_type, + api_version=api_version, + api_engine=api_engine, + headers=headers, + can_stream=can_stream, + vision=vision, + audio=audio, + reasoning=reasoning, + verbosity=verbosity, + image_detail_original=image_detail_original, + service_tier=service_tier, + supports_schema=supports_schema, + supports_tools=supports_tools, + allows_system_prompt=allows_system_prompt, + ) + self._reasoning = reasoning + self._reasoning_summary = reasoning_summary + self._verbosity = verbosity + self._image_detail_original = image_detail_original + self._service_tier = service_tier + self.Options = build_options_class( + reasoning=reasoning, + reasoning_summary=reasoning, + verbosity=verbosity, + image_detail_original=image_detail_original, + chat_completions=True, + service_tier=service_tier, + ) + + async def execute( + self, + prompt: Prompt, + stream: bool, + response: AsyncResponse, + conversation: AsyncConversation | None = None, + key: str | None = None, + ) -> AsyncGenerator[str | StreamEvent, None]: + if getattr(prompt.options, "chat_completions", None): + chat = AsyncChat(**self._delegate_chat_kwargs()) + _partition_tools(chat, prompt.tools) + async for event in chat.execute( + prompt, stream, response, conversation, key + ): + yield event + return + + if prompt.system and not self.allows_system_prompt: + raise NotImplementedError("Model does not support system prompts") + + image_detail = getattr(prompt.options, "image_detail", None) + if image_detail is not None: + image_detail = image_detail.value + input_items, instructions = self._build_responses_input( + prompt, image_detail=image_detail + ) + kwargs = self._finalize_responses_kwargs(prompt, stream, instructions) + + client = self.get_client(key, async_=True) + usage = None + had_reasoning = False + if stream: + stream_obj = await client.responses.create( + model=self.model_name or self.model_id, + input=input_items, + stream=True, + **kwargs, + ) + tool_call_meta: dict[str, dict[str, str]] = {} + final_response_dict: dict[str, Any] | None = None + reasoning_items_with_streamed_text = set() + reasoning_done_events: dict[str, StreamEvent] = {} + server_tool_done_events: dict[str, list[StreamEvent]] = {} + message_index = 0 + seen_message = False + async for event in stream_obj: + etype = getattr(event, "type", None) + if etype == "response.output_item.added": + item = event.item + if item.type == "message": + if seen_message: + message_index += 1 + seen_message = True + elif item.type == "function_call": + tool_call_meta[item.id] = { + "id": item.id, + "call_id": item.call_id, + "name": item.name, + } + yield StreamEvent( + type="tool_call_name", + chunk=item.name or "", + tool_call_id=item.call_id, + message_index=message_index, + ) + elif etype == "response.output_text.delta": + yield StreamEvent( + type="text", + chunk=event.delta or "", + message_index=message_index, + ) + elif etype == "response.function_call_arguments.delta": + item_id = getattr(event, "item_id", None) + meta = tool_call_meta.get(item_id) if item_id else None + call_id = meta["call_id"] if meta else None + yield StreamEvent( + type="tool_call_args", + chunk=event.delta or "", + tool_call_id=call_id, + message_index=message_index, + ) + elif etype in ( + "response.reasoning_summary_text.delta", + "response.reasoning_text.delta", + ): + item_id = getattr(event, "item_id", None) + if item_id: + reasoning_items_with_streamed_text.add(item_id) + yield StreamEvent( + type="reasoning", + chunk=event.delta or "", + message_index=message_index, + ) + elif etype in ( + "response.reasoning_summary_text.done", + "response.reasoning_text.done", + ): + item_id = getattr(event, "item_id", None) + if item_id not in reasoning_items_with_streamed_text: + text = getattr(event, "text", None) or "" + if text: + if item_id: + reasoning_items_with_streamed_text.add(item_id) + yield StreamEvent( + type="reasoning", + chunk=text, + message_index=message_index, + ) + elif etype == "response.output_item.done": + item = event.item + if item.type == "reasoning": + had_reasoning = True + item_id = getattr(item, "id", None) + reasoning_event = self._reasoning_event( + item, + include_text=( + item_id not in reasoning_items_with_streamed_text + ), + ) + reasoning_event.message_index = message_index + if item_id: + # Retained so the refresh after + # response.completed can target the part + # this event resolved to. + reasoning_done_events[item_id] = reasoning_event + yield reasoning_event + elif item.type == "function_call": + try: + args = json.loads(item.arguments) if item.arguments else {} + except json.JSONDecodeError: + args = {"_raw": item.arguments} + response.add_tool_call( + llm.ToolCall( + tool_call_id=item.call_id, + name=item.name, + arguments=args, + ) + ) + else: + server_events = self._server_tool_events(item, message_index) + item_id = getattr(item, "id", None) + if item_id and server_events: + server_tool_done_events[item_id] = server_events + for server_event in server_events: + yield server_event + elif etype == "response.completed": + self._refresh_server_tool_events( + event.response.output, server_tool_done_events + ) + final_response_dict = event.response.model_dump(warnings=False) + if final_response_dict.get("usage"): + usage = final_response_dict["usage"] + if final_response_dict is not None: + response.response_json = remove_dict_none_values(final_response_dict) + for refresh in self._reasoning_refresh_events( + response.response_json, reasoning_done_events + ): + yield refresh + else: + completion = await client.responses.create( + model=self.model_name or self.model_id, + input=input_items, + stream=False, + **kwargs, + ) + dumped = completion.model_dump(warnings=False) + response.response_json = remove_dict_none_values(dumped) + usage = dumped.get("usage") + events, had_reasoning = self._non_streaming_output_events( + completion.output, response + ) + for event in events: + yield event + + self._set_usage_responses(response, usage) + if ( + not had_reasoning + and usage + and ((usage.get("output_tokens_details") or {}).get("reasoning_tokens")) + ): + yield StreamEvent(type="reasoning", chunk="", redacted=True) + response._prompt_json = redact_data( + {"input": input_items, "instructions": instructions} + ) + + class Completion(Chat): class Options(SharedOptions): - logprobs: Optional[int] = Field( + logprobs: int | None = Field( description="Include the log probabilities of most likely N per token", default=None, le=5, @@ -929,26 +2918,35 @@ def __init__(self, *args, default_max_tokens=None, **kwargs): self.default_max_tokens = default_max_tokens def __str__(self) -> str: - return "OpenAI Completion: {}".format(self.model_id) + return f"OpenAI Completion: {self.model_id}" def execute( self, prompt: Prompt, stream: bool, response: Response, - conversation: Optional[Conversation] = None, - key: Optional[str] = None, - ) -> Iterator[str]: + conversation: Conversation | None = None, + key: str | None = None, + ) -> Iterator[str | StreamEvent]: if prompt.system: raise NotImplementedError( "System prompts are not supported for OpenAI completion models" ) + from llm.parts import TextPart + + # prompt.messages carries the full history - including history + # reloaded from storage, which conversation.responses does not. messages = [] - if conversation is not None: - for prev_response in conversation.responses: - messages.append(prev_response.prompt.prompt) - messages.append(cast(Response, prev_response).text()) - messages.append(prompt.prompt) + for message in prompt.messages: + if message.role not in ("user", "assistant"): + continue + text = "".join( + part.text + for part in message.parts + if isinstance(part, TextPart) and part.text + ) + if text: + messages.append(text) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key) if stream: @@ -986,7 +2984,7 @@ def not_nulls(data) -> dict: return {key: value for key, value in data if value is not None} -def combine_chunks(chunks: List) -> dict: +def combine_chunks(chunks: list) -> dict: content = "" role = None finish_reason = None diff --git a/llm/embeddings.py b/llm/embeddings.py index 5c9bf8ffa..0afe0f8a9 100644 --- a/llm/embeddings.py +++ b/llm/embeddings.py @@ -1,21 +1,24 @@ -from .models import EmbeddingModel -from .embeddings_migrations import embeddings_migrations -from dataclasses import dataclass import hashlib -from itertools import islice import json +import time +from collections.abc import Iterable +from dataclasses import dataclass +from itertools import islice +from typing import Any, cast + from sqlite_utils import Database from sqlite_utils.db import Table -import time -from typing import cast, Any, Dict, Iterable, List, Optional, Tuple, Union + +from .embeddings_migrations import embeddings_migrations +from .models import EmbeddingModel @dataclass class Entry: id: str - score: Optional[float] - content: Optional[str] = None - metadata: Optional[Dict[str, Any]] = None + score: float | None + content: str | None = None + metadata: dict[str, Any] | None = None class Collection: @@ -25,10 +28,10 @@ class DoesNotExist(Exception): def __init__( self, name: str, - db: Optional[Database] = None, + db: Database | None = None, *, - model: Optional[EmbeddingModel] = None, - model_id: Optional[str] = None, + model: EmbeddingModel | None = None, + model_id: str | None = None, create: bool = True, ) -> None: """ @@ -115,8 +118,8 @@ def count(self) -> int: def embed( self, id: str, - value: Union[str, bytes], - metadata: Optional[Dict[str, Any]] = None, + value: str | bytes, + metadata: dict[str, Any] | None = None, store: bool = False, ) -> None: """ @@ -152,7 +155,7 @@ def embed( def embed_multi( self, - entries: Iterable[Tuple[str, Union[str, bytes]]], + entries: Iterable[tuple[str, str | bytes]], store: bool = False, batch_size: int = 100, ) -> None: @@ -172,7 +175,7 @@ def embed_multi( def embed_multi_with_metadata( self, - entries: Iterable[Tuple[str, Union[str, bytes], Optional[Dict[str, Any]]]], + entries: Iterable[tuple[str, str | bytes, dict[str, Any] | None]], store: bool = False, batch_size: int = 100, ) -> None: @@ -202,9 +205,7 @@ def embed_multi_with_metadata( """ select id from embeddings where collection_id = ? and content_hash in ({}) - """.format( - ",".join("?" for _ in items_and_hashes) - ), + """.format(",".join("?" for _ in items_and_hashes)), [collection_id] + [item_and_hash[1] for item_and_hash in items_and_hashes], ) @@ -213,7 +214,7 @@ def embed_multi_with_metadata( embeddings = list( self.model().embed_multi(item[1] for item in filtered_batch) ) - with self.db.conn: + with self.db.atomic(): cast(Table, self.db["embeddings"]).insert_all( ( { @@ -239,11 +240,11 @@ def embed_multi_with_metadata( def similar_by_vector( self, - vector: List[float], + vector: list[float], number: int = 10, - skip_id: Optional[str] = None, - prefix: Optional[str] = None, - ) -> List[Entry]: + skip_id: str | None = None, + prefix: str | None = None, + ) -> list[Entry]: """ Find similar items in the collection by a given vector. @@ -297,8 +298,8 @@ def distance_score(other_encoded): ] def similar_by_id( - self, id: str, number: int = 10, prefix: Optional[str] = None - ) -> List[Entry]: + self, id: str, number: int = 10, prefix: str | None = None + ) -> list[Entry]: """ Find similar items in the collection by a given ID. @@ -326,8 +327,8 @@ def similar_by_id( ) def similar( - self, value: Union[str, bytes], number: int = 10, prefix: Optional[str] = None - ) -> List[Entry]: + self, value: str | bytes, number: int = 10, prefix: str | None = None + ) -> list[Entry]: """ Find similar items in the collection by a given value. @@ -357,12 +358,12 @@ def delete(self): """ Delete the collection and its embeddings from the database """ - with self.db.conn: + with self.db.atomic(): self.db.execute("delete from embeddings where collection_id = ?", [self.id]) self.db.execute("delete from collections where id = ?", [self.id]) @staticmethod - def content_hash(input: Union[str, bytes]) -> bytes: + def content_hash(input: str | bytes) -> bytes: "Hash content for deduplication. Override to change hashing behavior." if isinstance(input, str): input = input.encode("utf8") diff --git a/llm/embeddings_migrations.py b/llm/embeddings_migrations.py index 600ad204d..196d2abbb 100644 --- a/llm/embeddings_migrations.py +++ b/llm/embeddings_migrations.py @@ -1,7 +1,8 @@ -from sqlite_migrate import Migrations import hashlib import time +from sqlite_utils import Migrations + embeddings_migrations = Migrations("llm.embeddings") @@ -32,7 +33,7 @@ def m003_add_updated(db): # Pretty-print the schema db["embeddings"].transform() # Assume anything existing was last updated right now - db.query( + db.execute( "update embeddings set updated = ? where updated is null", [int(time.time())] ) @@ -62,21 +63,17 @@ def random_md5(): db.conn.create_function("temp_md5", 1, md5) db.conn.create_function("temp_random_md5", 0, random_md5) - with db.conn: - db.execute( - """ + with db.atomic(): + db.execute(""" update embeddings set content_hash = temp_md5(content) where content is not null - """ - ) - db.execute( - """ + """) + db.execute(""" update embeddings set content_hash = temp_random_md5() where content is null - """ - ) + """) db["embeddings"].create_index(["content_hash"]) diff --git a/llm/hookspecs.py b/llm/hookspecs.py index a244b007f..0cf46b546 100644 --- a/llm/hookspecs.py +++ b/llm/hookspecs.py @@ -1,5 +1,4 @@ -from pluggy import HookimplMarker -from pluggy import HookspecMarker +from pluggy import HookimplMarker, HookspecMarker hookspec = HookspecMarker("llm") hookimpl = HookimplMarker("llm") @@ -11,7 +10,7 @@ def register_commands(cli): @hookspec -def register_models(register): +def register_models(register, model_aliases): "Register additional model instances representing LLM models that can be called" diff --git a/llm/logs.py b/llm/logs.py new file mode 100644 index 000000000..159ad2996 --- /dev/null +++ b/llm/logs.py @@ -0,0 +1,1893 @@ +"""Content-addressed storage for conversation message trees. + +A conversation is a parent-linked chain of :class:`llm.Message` objects. +Each message is identified by a hash over its own canonical content plus +its parent's hash, so two conversations that share a prefix share the +rows that store it. Forking a conversation, or re-sending a history from +a client that holds the conversation state itself, both write only the +messages that are genuinely new. + +The identity of a message is its *resolved* content, but storage is by +reference: text that borrows from a fragment stores the fragment's id in +place of a copy, and attachments store an id into the existing +content-addressed ``attachments`` table. Ask a hundred questions about a +novel and the novel is stored once. Reading resolves the references +again, so the hash always covers the content as the model saw it - +``LogStore.verify()`` re-derives it to prove that stays true. +""" + +import datetime +import hashlib +import json +from typing import Any + +from condense_json import UncondenseError, condense_json, uncondense_json + +from .migrations import migrate +from .models import Attachment, ServerSideTool, _conversation_name +from .parts import ( + AttachmentPart, + Message, + Part, + ReasoningPart, + TextPart, + ToolCallPart, + ToolResultPart, +) +from .utils import ( + ensure_fragment, + ensure_tool, + ensure_tool_instance, + make_schema_id, + monotonic_ulid, +) + +__all__ = [ + "HASH_PREFIX", + "LogStore", + "canonical_json", + "content_hash", + "message_hash", +] + +# Hashes are tagged with the algorithm that produced them so a future +# change to the canonical form or the digest is detectable rather than +# silently splitting the dedup space into two incompatible halves. +HASH_PREFIX = "b2:" + +_DIGEST_SIZE = 16 + + +def canonical_json(obj: Any) -> str: + """Serialize to the canonical JSON form used for hashing. + + Keys sorted, no insignificant whitespace, non-ASCII left as-is. This + form is part of the documented contract: changing it changes every + hash. + """ + return json.dumps( + obj, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + + +def content_hash(obj: Any) -> str: + "Tagged hash of the canonical JSON form of ``obj``." + canonical = canonical_json(obj) + digest = hashlib.blake2b( + canonical.encode("utf-8"), digest_size=_DIGEST_SIZE + ).hexdigest() + return f"{HASH_PREFIX}{digest}" + + +def _canonical_attachment(attachment) -> dict: + """The hashed form of an attachment: content identity plus the + model-visible media type. + + The content hash is recomputed from the actual bytes every time, + never taken from Attachment.id()'s cache - that is what lets + verify() notice that a path-backed file changed after logging. + Path-backed attachments keep no copy of their bytes in the store + (by design: logs.db does not swallow large media), so their + fidelity depends on the file staying put; a changed or deleted + file shows up as a broken hash rather than passing silently. + + The type participates because the model sees it: identical bytes + sent as image/png and as text/plain are different requests. URL + attachments hash the URL itself - the log records which URL was + sent, not whatever it served that day. + """ + if attachment.content: + content_id = hashlib.sha256(attachment.content).hexdigest() + elif attachment.path: + try: + with open(attachment.path, "rb") as fp: + content_id = hashlib.sha256(fp.read()).hexdigest() + except OSError: + content_id = f"missing:{attachment.path}" + else: + content_id = hashlib.sha256( + json.dumps({"url": attachment.url}).encode("utf-8") + ).hexdigest() + try: + type_ = attachment.resolve_type() + except OSError: + # A deleted path-backed file - the content hash above already + # carries the missing marker + type_ = attachment.type + return {"id": content_id, "type": type_} + + +def message_hash(message: Message, parent_hash: str | None) -> str: + """Identity of ``message`` when reached via ``parent_hash``. + + The parent participates, so the same content at a different point in + a conversation is a different node. That is what makes a shared + prefix collapse to shared rows without any explicit comparison. + + Attachments are hashed by content id, via _canonical_attachment. + """ + d: Any = message.to_dict() + for part, part_dict in zip(message.parts, d["parts"]): + attachment = getattr(part, "attachment", None) + if attachment is not None: + part_dict["attachment"] = _canonical_attachment(attachment) + attachments = getattr(part, "attachments", None) + if attachments: + part_dict["attachments"] = [_canonical_attachment(a) for a in attachments] + return content_hash({"parent": parent_hash, "message": d}) + + +class LogStore: + """Read and write conversation history in a SQLite database. + + Wraps a ``sqlite_utils.Database`` and applies any outstanding + migrations on construction, so a fresh database and an existing one + are handled the same way:: + + store = LogStore(sqlite_utils.Database("logs.db")) + """ + + def __init__(self, db): + self.db = db + migrate(db) + + # -- writing ------------------------------------------------------- + + def ensure_chain( + self, + messages, + parent: str | None = None, + fragments=None, + ) -> str | None: + """Store ``messages`` as a chain and return the hash of the tip. + + Messages already present are left alone, so a caller that + re-sends a whole conversation - a client holding the state + itself, or a fork of an existing thread - writes only the + messages that are new. Passing ``parent`` appends to an existing + chain instead of starting a new one. + + ``fragments`` is an optional list of fragment contents that may + appear inside these messages. Any that do are stored as a + reference rather than a copy, which is the point of the fragments + feature: ask a hundred questions about a novel and the novel is + stored once. It never affects the hashes - identity is always the + resolved text. + """ + fragment_map = self._fragment_map(fragments) + tip = parent + for message in messages: + tip = self._ensure_message(message, tip, fragment_map) + return tip + + def _fragment_map(self, fragments) -> dict[str, int]: + "Map fragment content to its id, registering any that are new." + if not fragments: + return {} + return { + str(fragment): ensure_fragment(self.db, fragment) + for fragment in fragments + if str(fragment) + } + + def _ensure_message( + self, + message: Message, + parent_hash: str | None, + fragment_map: dict[str, int], + ) -> str: + hash = message_hash(message, parent_hash) + if self.db["messages"].count_where("hash = ?", [hash]): + # Already stored - and because the hash covers the parent, + # everything below it is stored too. + return hash + with self.db.atomic(): + # Another writer can store the same hash between the check + # above and this insert. Insert-or-ignore settles who won, + # and only the winner writes the parts. + cursor = self.db.execute( + "insert or ignore into messages" + " (hash, parent_hash, role, provider_metadata)" + " values (?, ?, ?, ?)", + [hash, parent_hash, message.role, _dump(message.provider_metadata)], + ) + if cursor.rowcount: + for position, part in enumerate(message.parts): + self._write_part(hash, position, part, fragment_map) + return hash + + def _write_part( + self, + message_hash_: str, + position: int, + part, + fragment_map: dict[str, int], + ) -> None: + payload = part.to_dict() + # The type key is redundant with the type column; readers put it + # back from there. + part_type = payload.pop("type") + attachments = _attachments_of(part) + + # Large content out of the payload and into the tables that + # already store it once: fragments for text, attachments for + # bytes. Both are resolved again on the way back out. + used_fragments = _encode_text_refs(payload, fragment_map) + if attachments: + attachment_ids = [ + ensure_attachment(self.db, attachment) for attachment in attachments + ] + _encode_attachment_refs(payload, attachment_ids, part_type) + else: + attachment_ids = [] + + # Pure literal text lives in its own column - raw, unescaped, + # never parsed - so prose reads as prose in SQL. Text that + # borrows fragments stays structured in the payload as text_ref. + text = None + if part_type in ("text", "reasoning") and "text" in payload: + text = payload.pop("text") + + part_id = ( + self.db["parts"] + .insert( + { + "message_hash": message_hash_, + "position": position, + "type": part_type, + "tool_name": payload.get("name"), + "text": text, + # Plain dumps, not canonical_json: sorting keys is + # for hashing. Storage keeps the order the model + # produced, so tool call arguments read back as + # they were written. NULL when the text column + # carries the whole part. + "payload": json.dumps(payload) if payload else None, + } + ) + .last_pk + ) + for order, attachment_id in enumerate(attachment_ids): + self.db["part_attachments"].insert( + { + "part_id": part_id, + "attachment_id": attachment_id, + "order": order, + } + ) + for order, fragment_id in enumerate(used_fragments): + self.db["part_fragments"].insert( + { + "part_id": part_id, + "fragment_id": fragment_id, + "order": order, + } + ) + + # -- reading ------------------------------------------------------- + + def load_chain(self, tip: str | None) -> list[Message]: + """Return the full chain ending at ``tip``, oldest message first. + + Raises ``KeyError`` if ``tip`` is not in the store. + """ + if tip is None: + return [] + rows = [] + hash: str | None = tip + while hash is not None: + found = list(self.db.query("select * from messages where hash = ?", [hash])) + if not found: + raise KeyError(hash) + rows.append(found[0]) + hash = found[0]["parent_hash"] + rows.reverse() + parts_by_message = self._load_parts([row["hash"] for row in rows]) + return [ + Message( + role=row["role"], + parts=parts_by_message.get(row["hash"], []), + provider_metadata=_load(row["provider_metadata"]), + ) + for row in rows + ] + + def _load_parts(self, message_hashes: list[str]) -> dict[str, list[Any]]: + if not message_hashes: + return {} + placeholders = ",".join("?" * len(message_hashes)) + part_rows = list( + self.db.query( + f""" + select * from parts + where message_hash in ({placeholders}) + order by message_hash, position + """, + message_hashes, + ) + ) + # Rebuild each part's full dict from its columns: type from the + # type column, literal text from the text column, everything + # else from the JSON payload. + payloads: list[Any] = [] + for row in part_rows: + payload = json.loads(row["payload"]) if row["payload"] else {} + payload["type"] = row["type"] + if row["text"] is not None: + payload["text"] = row["text"] + payloads.append(payload) + # Resolve the references put in on the way in. Both lookups are + # batched across the whole chain rather than done per part. + fragments = self._load_fragments(payloads) + attachments = self._load_attachments(payloads) + out: dict[str, list[Any]] = {} + for row, payload in zip(part_rows, payloads): + _decode_text_refs(payload, fragments) + # Strip the attachment references before rebuilding, then + # hang the resolved objects back on. + ids = _attachment_ids(payload) + payload.pop("attachment", None) + payload.pop("attachments", None) + part = Part.from_dict(payload) + _resolve_attachments(part, ids, attachments) + out.setdefault(row["message_hash"], []).append(part) + return out + + def _load_fragments(self, payloads: list[dict]) -> dict[int, str]: + ids = sorted({id for payload in payloads for id in _fragment_ids(payload)}) + if not ids: + return {} + placeholders = ",".join("?" * len(ids)) + return { + row["id"]: row["content"] + for row in self.db.query( + f"select id, content from fragments where id in ({placeholders})", + ids, + ) + } + + def _load_attachments(self, payloads: list[dict]) -> dict[str, Any]: + ids = sorted({id for payload in payloads for id in _attachment_ids(payload)}) + if not ids: + return {} + placeholders = ",".join("?" * len(ids)) + return { + row["id"]: Attachment.from_row(row) + for row in self.db.query( + f"select * from attachments where id in ({placeholders})", ids + ) + } + + # -- threads ------------------------------------------------------- + + def create_thread( + self, + name: str | None = None, + tip: str | None = None, + forked_from: str | None = None, + id: str | None = None, + ) -> str: + "Create a named pointer at a message and return its id." + thread_id = id or str(monotonic_ulid()).lower() + self.db["threads"].insert( + { + "id": thread_id, + "name": name, + "tip_message_hash": tip, + "forked_from": forked_from, + "datetime_utc": _now(), + } + ) + return thread_id + + def ensure_thread(self, thread_id: str, name: str | None = None) -> str: + """Return the thread with this id, creating it if it is new. + + Threads created from a conversation reuse the conversation's id, + so the two identifier spaces line up while both sets of tables + are being written. + """ + if not self.db["threads"].count_where("id = ?", [thread_id]): + self.create_thread(name=name, id=thread_id) + return thread_id + + def fork( + self, + message_hash_: str, + name: str | None = None, + forked_from: str | None = None, + ) -> str: + """Start a new thread from an existing message. + + Nothing is copied - the new thread points at a message that is + already stored, so its whole history is shared with the thread it + came from until the two diverge. + """ + if not self.db["messages"].count_where("hash = ?", [message_hash_]): + raise KeyError(message_hash_) + return self.create_thread(name=name, tip=message_hash_, forked_from=forked_from) + + def thread_tip(self, thread_id: str) -> str | None: + "The message a thread currently points at." + rows = list(self.db.query("select * from threads where id = ?", [thread_id])) + if not rows: + raise KeyError(thread_id) + return rows[0]["tip_message_hash"] + + def thread_messages(self, thread_id: str) -> list[Message]: + "The full history of a thread, oldest message first." + return self.load_chain(self.thread_tip(thread_id)) + + def append(self, thread_id: str, messages) -> str | None: + "Add messages to the end of a thread and return the new tip." + tip = self.ensure_chain(messages, parent=self.thread_tip(thread_id)) + self.db["threads"].update(thread_id, {"tip_message_hash": tip}) + return tip + + # -- turns --------------------------------------------------------- + + def log(self, response, thread_id: str | None = None) -> str: + """Record a completed response. + + The input chain and the response's own output are stored as + messages; everything that is specific to this particular call - + timings, usage, which model answered - goes on the turn, because + message rows are shared and so cannot carry provenance. + """ + with self.db.atomic(): + return self._log_in_transaction(response, thread_id) + + def _log_in_transaction(self, response, thread_id: str | None) -> str: + if thread_id is None: + conversation = getattr(response, "conversation", None) + # A response logged outside any conversation still gets a + # thread of its own, so `llm -c` and `llm logs` can always + # find it - the same guarantee the conversations table used + # to provide. + thread_id = self.ensure_thread( + conversation.id if conversation else str(monotonic_ulid()).lower(), + name=_conversation_name( + response.prompt.prompt or response.prompt.system or "" + ), + ) + + prompt_fragments = list(response.prompt.fragments or []) + system_fragments = list(response.prompt.system_fragments or []) + + parent = self.ensure_chain( + response.prompt.messages, + fragments=prompt_fragments + system_fragments, + ) + # _messages_now() rather than messages(), which is a coroutine on + # AsyncResponse. + own_messages = response._messages_now() + tip = self.ensure_chain(own_messages, parent=parent) + + schema_id = None + if response.prompt.schema: + schema_id, schema_json = make_schema_id(response.prompt.schema) + self.db["schemas"].insert( + {"id": schema_id, "content": schema_json}, ignore=True + ) + + turn_id = response.id or str(monotonic_ulid()).lower() + self.db["turns"].insert( + { + "id": turn_id, + "thread_id": thread_id, + "parent_message_hash": parent, + "tip_message_hash": tip, + "model": response.model.model_id, + "resolved_model": response.resolved_model, + "options_json": _dump( + { + key: value + for key, value in dict(response.prompt.options).items() + if value is not None + } + ), + "schema_id": schema_id, + "input_tokens": response.input_tokens, + "output_tokens": response.output_tokens, + "token_details": _dump(response.token_details), + "duration_ms": response.duration_ms(), + "datetime_utc": response.datetime_utc(), + "response_json": condense_payload( + getattr(response, "response_json", None), + own_messages, + [(tool.name, tool.description) for tool in response.prompt.tools], + schema=response.prompt.schema, + model_replacements=getattr( + response.model, "json_replacements", None + ), + ), + }, + replace=True, + ) + for tool in response.prompt.tools: + # Server-side tools are configured instances themselves. A + # toolbox-derived tool instead has an implementation method + # bound to its configured instance. Record either kind as a + # reference into the shared tool_instances table. + instance: Any | None + if isinstance(tool, ServerSideTool): + instance = tool + instance_name = tool.__class__.__name__ + else: + instance = getattr(tool.implementation, "__self__", None) + instance_name = tool.name.split("_")[0] + config = getattr(instance, "_config", None) + self.db["turn_tools"].insert( + { + "turn_id": turn_id, + "tool_id": ensure_tool(self.db, tool), + "instance_id": ( + ensure_tool_instance( + self.db, + instance_name, + tool.plugin, + json.dumps(config), + ) + if config is not None + else None + ), + }, + replace=True, + ) + # Which fragments this call was given - provenance, so it belongs + # on the turn rather than on the shared message rows. This is what + # answers "show me everything that used fragment X". + for kind, fragments in ( + ("prompt", prompt_fragments), + ("system", system_fragments), + ): + for order, fragment in enumerate(fragments): + self.db["turn_fragments"].insert( + { + "turn_id": turn_id, + "fragment_id": ensure_fragment(self.db, fragment), + "order": order, + "kind": kind, + }, + replace=True, + ) + # Which configured toolbox instance served each tool call. This + # is local execution provenance, so it lives outside the hashed + # message tree, keyed by the tool_call_id both worlds share. + for tool_result in response.prompt.tool_results: + # instance is annotated as Toolbox but a tool built from a + # bound method can carry an arbitrary __self__ here, so ask + # for the config rather than trusting the type. + config = getattr(tool_result.instance, "_config", None) + if config is None or not tool_result.tool_call_id: + continue + self.db["tool_instantiations"].insert( + { + "turn_id": turn_id, + "tool_call_id": tool_result.tool_call_id, + "instance_id": ensure_tool_instance( + self.db, + tool_result.name.split("_")[0], + next( + ( + tool.plugin + for tool in response.prompt.tools + if tool.name == tool_result.name + ), + None, + ), + json.dumps(config), + ), + }, + replace=True, + ) + # Refresh this turn's search row. Delete-then-derive rather than + # replace, so a re-logged turn with a different tip converges on + # what the turn now says. Derived in SQL from the stored + # payloads, not from load_chain - resolving references would put + # fragment content back into the searchable text. + self.db["turn_search"].delete_where("turn_id = ?", [turn_id]) + self.db.execute(TURN_SEARCH_INSERT_SQL, {"turn_id": turn_id}) + if thread_id is not None: + self.db["threads"].update(thread_id, {"tip_message_hash": tip}) + return turn_id + + def turn_response_json(self, turn_id: str) -> Any: + """The raw provider payload recorded for a turn, resolved. + + Returns ``None`` when the turn is unknown or recorded no + payload. Raises ``condense_json.UncondenseError`` when the + payload references message content that no longer resolves - + the payload was recorded but its context is gone. + """ + row = next( + iter( + self.db.query( + "select turns.model, turns.parent_message_hash," + " turns.tip_message_hash, turns.response_json," + " schemas.content as schema_json" + " from turns" + " left join schemas on turns.schema_id = schemas.id" + " where turns.id = ?", + [turn_id], + ) + ), + None, + ) + if row is None or row["response_json"] is None: + return None + inputs = self.load_chain(row["parent_message_hash"]) + outputs = self.load_chain(row["tip_message_hash"])[len(inputs) :] + return resolve_payload( + row["response_json"], + outputs, + self._turn_tool_pairs(turn_id), + schema=_load(row["schema_json"]), + model_replacements=_model_json_replacements(row["model"]), + ) + + def _turn_tool_pairs(self, turn_id: str) -> list[tuple[str, str]]: + return [ + (row["name"], row["description"]) + for row in self.db.query(TURN_TOOLS_SQL, [turn_id]) + ] + + # -- verification -------------------------------------------------- + + def verify(self) -> list[str]: + """Re-hash every stored message and return those that disagree. + + Reads resolve references - fragment text and attachment bytes are + stitched back in - so a bug there, or a fragment deleted out from + under a message, produces a chain that differs from the one that + was hashed. Nothing else would notice: the wrong text would just + be silently sent to the model. Re-deriving the hash from what + comes back out catches the whole class. + + An empty list means every message on disk still resolves to the + content its hash was taken over. + """ + broken = [] + for row in self.db.query("select hash, parent_hash from messages"): + parts = self._load_parts([row["hash"]]).get(row["hash"], []) + message_row = next( + iter( + self.db.query( + "select * from messages where hash = ?", [row["hash"]] + ) + ) + ) + message = Message( + role=message_row["role"], + parts=parts, + provider_metadata=_load(message_row["provider_metadata"]), + ) + if message_hash(message, row["parent_hash"]) != row["hash"]: + broken.append(row["hash"]) + return broken + + # -- pending work -------------------------------------------------- + + def pending_tool_calls(self, tip: str | None) -> list[Any]: + """Tool calls at the tip of a chain that have no result yet. + + A chain ending in tool calls with nothing after them is a paused + conversation waiting to be resumed - it needs no separate record. + """ + chain = self.load_chain(tip) + if not chain: + return [] + return [ + part + for part in chain[-1].parts + if isinstance(part, ToolCallPart) and not part.server_executed + ] + + +def ensure_attachment(db, attachment) -> str: + "Store an attachment, returning its content-addressed id." + attachment_id = attachment.id() + db["attachments"].insert( + { + "id": attachment_id, + "type": attachment.resolve_type(), + "path": attachment.path, + "url": attachment.url, + "content": attachment.content, + }, + replace=True, + ) + return attachment_id + + +# -- reference encoding ------------------------------------------------ +# +# A stored payload is Part.to_dict() with large content swapped for a +# reference: fragment ids in place of text, attachment ids in place of +# bytes. Resolving it reproduces the wire form exactly, which is what +# makes it safe for the hash to be taken over the resolved content and +# never over what is on disk. + + +def _attachments_of(part) -> list[Any]: + "The Attachment objects a part carries, in order." + if isinstance(part, ToolResultPart): + return list(part.attachments) + if isinstance(part, AttachmentPart) and part.attachment is not None: + return [part.attachment] + return [] + + +def _encode_text_refs(payload: dict, fragment_map: dict[str, int]) -> list[int]: + """Replace ``text`` with a ``text_ref`` list of fragments and + literals. Returns the fragment ids used, in order. + + Nothing is replaced unless a fragment actually occurs in the text, so + a part that borrows no fragments keeps its plain ``text`` key. + """ + text = payload.get("text") + if not text or not fragment_map: + return [] + pieces: list[dict] = [] + used: list[int] = [] + remaining = text + while remaining: + # Earliest occurrence wins; on a tie the longer fragment does, so + # a fragment that is a prefix of another cannot mask it. + best: tuple[int, str] | None = None + for content in fragment_map: + index = remaining.find(content) + if index == -1: + continue + if best is None or (index, -len(content)) < (best[0], -len(best[1])): + best = (index, content) + if best is None: + pieces.append({"literal": remaining}) + break + index, content = best + if index: + pieces.append({"literal": remaining[:index]}) + pieces.append({"fragment": fragment_map[content]}) + used.append(fragment_map[content]) + remaining = remaining[index + len(content) :] + if not used: + return [] + del payload["text"] + payload["text_ref"] = pieces + return used + + +def _decode_text_refs(payload: dict, fragments: dict[int, str]) -> None: + "Reverse of _encode_text_refs, restoring the exact original text." + pieces = payload.pop("text_ref", None) + if pieces is None: + return + payload["text"] = "".join( + ( + piece["literal"] + if "literal" in piece + else fragments.get(piece["fragment"], "") + ) + for piece in pieces + ) + + +def _fragment_ids(payload: dict) -> list[int]: + return [ + piece["fragment"] + for piece in payload.get("text_ref") or [] + if "fragment" in piece + ] + + +def _encode_attachment_refs( + payload: dict, attachment_ids: list[str], part_type: str +) -> None: + "Replace inline attachment dicts with their content-addressed ids." + if part_type == "attachment": + payload["attachment"] = {"id": attachment_ids[0]} + elif part_type == "tool_result": + payload["attachments"] = [{"id": id} for id in attachment_ids] + + +def _resolve_attachments(part, ids: list[str], attachments: dict[str, Any]) -> None: + """Hang the resolved Attachment objects back on a part. + + Done after ``Part.from_dict`` rather than by putting them back into + the payload, so the bytes are never round-tripped through base64 and + the objects keep the content-addressed id they were stored under. + """ + if not ids: + return + if isinstance(part, AttachmentPart): + part.attachment = attachments[ids[0]] + elif isinstance(part, ToolResultPart): + part.attachments = [attachments[id] for id in ids] + + +def _attachment_ids(payload: dict) -> list[str]: + if payload["type"] == "attachment" and "attachment" in payload: + return [payload["attachment"]["id"]] + if payload["type"] == "tool_result": + return [ref["id"] for ref in payload.get("attachments") or []] + return [] + + +def _dump(value: dict | None) -> str | None: + return json.dumps(value) if value else None + + +def _load(value: str | None) -> dict | None: + return json.loads(value) if value else None + + +def _now() -> str: + return str(datetime.datetime.now(datetime.timezone.utc)) + + +# -- condensed provider payloads ---------------------------------------- +# +# The raw response.json() payload mostly duplicates content the store +# already holds: the response text, reasoning summaries and their +# encrypted blobs, long tool arguments, and the tool definitions the +# provider echoes back on every call. The turn stores it condensed +# instead - strings that already live in the turn's own messages or its +# tools are swapped for {"$": key} references (condense-json), leaving +# roughly the provider envelope: ids, usage, fingerprints, settings +# echoes. The replacement dict is never stored; it is rebuilt on the way +# out from the chain segment (hash-frozen) and the turn_tools join +# (per-turn provenance), so the same walk over the same rows produces +# the same dict on both sides of the round trip. + +# Below this a {"$": key} marker costs about as much as the string it +# replaces. +_CONDENSE_MIN_LENGTH = 64 + +# The tools a turn was given, as the (name, description) pairs +# _payload_replacements expects. +TURN_TOOLS_SQL = """ +select tools.name, tools.description +from turn_tools join tools on tools.id = turn_tools.tool_id +where turn_tools.turn_id = ? +""" + + +def _payload_replacements( + messages, tools=(), schema=None, model_replacements=None +) -> dict[str, Any]: + """Replacement values for condensing a turn's provider payload. + + ``messages`` is the turn's own contribution - the chain segment + between its parent and tip. Keys are structural (message offset, + part position, field path) so the identical dict can be rebuilt + from the stored segment at read time. + + ``tools`` is the turn's tools as (name, description) pairs - + ``response.prompt.tools`` on the way in, the ``turn_tools`` join on + the way out. Descriptions are keyed by tool name; a name carrying + two different descriptions in one turn is dropped, the same + order-independent verdict from either side's view of the pairs. + (Parameter schemas are deliberately not offered: providers echo a + transformed schema - OpenAI strict mode adds keys - so the stored + form would not match even structurally.) + + ``schema`` is the turn's JSON schema dict, when one was used - the + provider echoes it back in the payload (OpenAI ``text.format``) and + the schemas table already stores it once. It is offered as a + structural value, so the echo's key order does not matter. + + ``model_replacements`` is the model class's ``json_replacements`` + dictionary of common boilerplate - the zstd-custom-dictionary idea: + payload fragments the plugin author knows recur in every reply, + keyed here under an ``m.`` prefix so they can never collide with + the derived keys. These resolve by looking the model up again at + read time, so plugins must treat their ``json_replacements`` as + append-only: removing or changing an entry breaks every payload + already stored against it. + + Values are strings (matched as substrings) and dicts or lists + (matched as whole subtrees by condense-json's structural equality). + Container values inside provider_metadata are offered as well as + their leaf strings - outermost match wins, so a payload that embeds + a whole metadata object (a reasoning ``summary`` list, say) + condenses to one reference instead of one per string. + """ + replacements: dict[str, Any] = {} + + def add(key: str, value: Any) -> None: + if isinstance(value, str) and len(value) >= _CONDENSE_MIN_LENGTH: + replacements[key] = value + + def add_value(key: str, value: Any) -> None: + if isinstance(value, (dict, list)) and value: + # The same length bar as strings, applied to the canonical + # form, so a reference is always a clear win. + try: + size = len(json.dumps(value, separators=(",", ":"))) + except (TypeError, ValueError): + return + if size >= _CONDENSE_MIN_LENGTH: + replacements[key] = value + + def walk(prefix: str, obj: Any) -> None: + if isinstance(obj, dict): + add_value(prefix, obj) + for key, value in obj.items(): + walk(f"{prefix}.{key}", value) + elif isinstance(obj, list): + add_value(prefix, obj) + for index, value in enumerate(obj): + walk(f"{prefix}.{index}", value) + else: + add(prefix, obj) + + for mi, message in enumerate(messages): + message_dict = message.to_dict() + for pi, part in enumerate(message_dict.get("parts", [])): + base = f"{mi}.{pi}" + add(f"{base}.text", part.get("text")) + add(f"{base}.output", part.get("output")) + arguments = part.get("arguments") + if arguments: + # Providers that carry tool arguments as a JSON-encoded + # string need a byte-exact serialization - OpenAI uses + # the compact form, so offer both. Providers that embed + # them as an object (Anthropic, Gemini) match the dict + # itself structurally. + compact = json.dumps(arguments, separators=(",", ":")) + spaced = json.dumps(arguments) + add(f"{base}.args", compact) + if spaced != compact: + add(f"{base}.args2", spaced) + add_value(f"{base}.argsv", arguments) + walk(f"{base}.pm", part.get("provider_metadata") or {}) + walk(f"{mi}.pm", message_dict.get("provider_metadata") or {}) + + descriptions: dict[str, str] = {} + conflicting = set() + for name, description in tools: + if ( + not name + or not isinstance(description, str) + or len(description) < _CONDENSE_MIN_LENGTH + ): + continue + if descriptions.get(name, description) != description: + conflicting.add(name) + continue + descriptions[name] = description + for name, description in descriptions.items(): + if name not in conflicting: + replacements[f"tool.{name}.description"] = description + + add_value("schema", schema) + + # Model-declared boilerplate passes through as curated: the plugin + # author chose these entries, so no length threshold applies. + for key, value in (model_replacements or {}).items(): + replacements[f"m.{key}"] = value + return replacements + + +def condense_payload( + payload: Any, messages, tools=(), schema=None, model_replacements=None +) -> str | None: + "JSON text of ``payload`` condensed against the turn's stored content." + if payload is None: + return None + return json.dumps( + condense_json( + payload, + _payload_replacements(messages, tools, schema, model_replacements), + ) + ) + + +def resolve_payload( + condensed: str | None, messages, tools=(), schema=None, model_replacements=None +) -> Any: + """Reverse of :func:`condense_payload`. + + Raises ``condense_json.UncondenseError`` when the stored payload + references content the segment no longer produces. + """ + if condensed is None: + return None + return uncondense_json( + json.loads(condensed), + _payload_replacements(messages, tools, schema, model_replacements), + ) + + +def _model_json_replacements(model_id: str | None): + """The ``json_replacements`` boilerplate dictionary for a model id. + + Resolved through the registry at read time, so payloads condensed + against a model's dictionary need that model's plugin installed to + resolve again. An unknown model returns None; any markers that + depended on it surface as UncondenseError from resolve_payload. + """ + if not model_id: + return None + from llm import UnknownModelError, get_model + + try: + model = get_model(model_id) + except UnknownModelError: + return None + return getattr(model, "json_replacements", None) + + +# -- llm logs support --------------------------------------------------- +# +# Rows shaped like the ones the older `responses` query produced, so the +# existing rendering in llm.cli works unchanged, but derived entirely +# from the content-addressed tables. + +LOG_ROWS_SQL = """ +select + turns.id, + turns.model, + turns.resolved_model, + turns.options_json, + turns.thread_id as conversation_id, + turns.duration_ms, + turns.datetime_utc, + turns.input_tokens, + turns.output_tokens, + turns.token_details, + turns.parent_message_hash, + turns.tip_message_hash, + turns.response_json, + threads.name as conversation_name, + turns.model as conversation_model, + schemas.content as schema_json{rank_select} +from turns +left join threads on turns.thread_id = threads.id +left join schemas on turns.schema_id = schemas.id{join} +{where} +order by {order_by}{limit} +""" + +# Literal text of one parts row: the text column when the part's text +# is pure literal, otherwise the literal segments of a text_ref payload +# - fragment content is deliberately not searchable. +TURN_SEARCH_LITERAL = """coalesce( + parts.text, + (select group_concat(json_extract(je.value, '$.literal'), '') + from json_each(parts.payload, '$.text_ref') je + where json_extract(je.value, '$.literal') is not null) +)""" + +# Derives the searchable prompt and response text for turns. Serves both +# the migration backfill (turn_filter="") and the per-turn refresh in +# LogStore.log (turn_filter="and turns.id = :turn_id" - the slot appears +# in three places so the filtered form touches only that turn's chain). +TURN_SEARCH_INSERT_SQL = """ +with recursive output_messages(turn_id, hash) as ( + select turns.id, turns.tip_message_hash + from turns + where turns.tip_message_hash is not null + and (turns.parent_message_hash is null + or turns.tip_message_hash != turns.parent_message_hash) + and turns.id = :turn_id + union all + select om.turn_id, messages.parent_hash + from output_messages om + join messages on messages.hash = om.hash + join turns on turns.id = om.turn_id + where messages.parent_hash is not null + and (turns.parent_message_hash is null + or messages.parent_hash != turns.parent_message_hash) +), +prompt_text as ( + select turns.id as turn_id, + (select group_concat({LITERAL}, '') + from parts + where parts.message_hash = turns.parent_message_hash + and parts.type = 'text' + order by parts.position) as text + from turns + join messages on messages.hash = turns.parent_message_hash + where messages.role = 'user' and turns.id = :turn_id +), +response_text as ( + select om.turn_id, group_concat(part_text.text, '') as text + from output_messages om + join messages on messages.hash = om.hash and messages.role = 'assistant' + join ( + select parts.message_hash, parts.position, {LITERAL} as text + from parts where parts.type = 'text' + ) part_text on part_text.message_hash = om.hash + group by om.turn_id +) +insert into turn_search (turn_id, prompt, response) +select turns.id, + coalesce(prompt_text.text, ''), + coalesce(response_text.text, '') + from turns + left join prompt_text on prompt_text.turn_id = turns.id + left join response_text on response_text.turn_id = turns.id + where (coalesce(prompt_text.text, '') != '' + or coalesce(response_text.text, '') != '') and turns.id = :turn_id +""".replace("{LITERAL}", TURN_SEARCH_LITERAL) + +# Relevance ranking for -q. bm25 scores are negative-better, ascending +# order is best-first. The prompt column is weighted well above the +# response: what you typed is a stronger signal of what a turn is about +# than what the model said back. +TURN_SEARCH_RANK = "bm25(turn_search_fts, 10.0, 1.0)" +LEGACY_SEARCH_RANK = "bm25(responses_fts, 10.0, 1.0)" + + +def _text_of(parts, kind) -> str: + "Concatenated text of every part of ``kind`` in order." + return "".join(part.text for part in parts if isinstance(part, kind) and part.text) + + +class _LogRowBuilder: + """Turns a turn row into the shape `llm logs` renders. + + A turn's prompt is the last message it was given and its response is + whatever it appended, so both are derived from the parent/tip pair + rather than stored a second time. The chain up to the parent is a + prefix of the chain up to the tip, so splitting them is a matter of + length. + """ + + def __init__(self, store: "LogStore"): + self.store = store + + def build(self, row: dict) -> dict: + inputs = self.store.load_chain(row["parent_message_hash"]) + outputs = self.store.load_chain(row["tip_message_hash"])[len(inputs) :] + + # The turn's own input is the trailing run of user and tool + # messages: everything after the last assistant (or system) + # message belongs to this turn, because a turn's new input + # never contains an assistant message. A turn carrying both + # tool results and a fresh user prompt therefore keeps both. + boundary = len(inputs) + while boundary and inputs[boundary - 1].role in ("user", "tool"): + boundary -= 1 + input_messages = inputs[boundary:] + + prompt_parts = [ + part + for message in input_messages + if message.role == "user" + for part in message.parts + ] + input_parts = [part for message in input_messages for part in message.parts] + system_parts = inputs[0].parts if inputs and inputs[0].role == "system" else [] + out_parts = [part for message in outputs for part in message.parts] + + built = { + key: row[key] + for key in ( + "id", + "model", + "resolved_model", + "options_json", + "conversation_id", + "duration_ms", + "datetime_utc", + "input_tokens", + "output_tokens", + "token_details", + "conversation_name", + "conversation_model", + "schema_json", + ) + } + if "_search_rank" in row: + built["_search_rank"] = row["_search_rank"] + built.update( + { + # The turn stores null when no options were set; the + # responses table always recorded "{}". + "options_json": row["options_json"] or "{}", + "prompt": _text_of(prompt_parts, TextPart), + # None rather than "" when there was no system + # message, matching what was recorded before. + "system": _text_of(system_parts, TextPart) or None, + "response": _text_of(out_parts, TextPart), + "reasoning": _text_of(out_parts, ReasoningPart) or None, + # No longer stored: the chain holds the structure. + "prompt_json": None, + # Stored condensed against the turn's own messages; + # resolved here so the row carries the payload as the + # provider sent it. A payload whose references no longer + # resolve renders as absent rather than failing the + # whole listing. + "response_json": self._resolve_response_json(row, outputs), + "_input_parts": input_parts, + "_output_parts": out_parts, + # Internal, stripped before rendering - the enrichment + # needs them to find the parts rows behind these parts. + "_parent_message_hash": row["parent_message_hash"], + "_input_message_hashes": self._input_segment_hashes( + row["parent_message_hash"] + ), + "_tip_message_hash": row["tip_message_hash"], + } + ) + return built + + def _resolve_response_json(self, row: dict, outputs) -> str | None: + condensed = row.get("response_json") + if not condensed: + return None + try: + return json.dumps( + resolve_payload( + condensed, + outputs, + self.store._turn_tool_pairs(row["id"]), + schema=_load(row.get("schema_json")), + model_replacements=_model_json_replacements(row.get("model")), + ) + ) + except UncondenseError: + return None + + def _input_segment_hashes(self, parent_hash: str | None) -> list[str]: + """Hashes of the turn's own input messages - the same trailing + user/tool run build() derives, walked directly in the table.""" + hashes: list[str] = [] + hash_ = parent_hash + while hash_: + message_row = next( + iter( + self.store.db.query( + "select parent_hash, role from messages where hash = ?", + [hash_], + ) + ), + None, + ) + if message_row is None or message_row["role"] not in ("user", "tool"): + break + hashes.append(hash_) + hash_ = message_row["parent_hash"] + hashes.reverse() + return hashes + + +def log_rows( + store: "LogStore", + *, + count: int | None = None, + model_id: str | None = None, + thread_id: str | None = None, + fragment_hashes=(), + tool_names=(), + any_tools: bool = False, + schema_id: str | None = None, + id_gt: str | None = None, + id_gte: str | None = None, + ids=(), + query: str | None = None, + latest: bool = False, +) -> list[dict]: + """Rows for `llm logs`, newest first, drawn from the new tables. + + Sees only conversations with turns - merged_log_rows adds the rows + that exist solely in the legacy `responses` table. + + With ``query`` the rows are the best matches from the turn_search + index, most relevant first - or newest first when ``latest`` is + also set. + """ + where: list[str] = [] + params: dict[str, Any] = {} + + if model_id: + where.append("(turns.model = :model or turns.resolved_model = :model)") + params["model"] = model_id + if thread_id: + where.append("turns.thread_id = :thread_id") + params["thread_id"] = thread_id + if id_gt: + where.append("turns.id > :id_gt") + params["id_gt"] = id_gt + if id_gte: + where.append("turns.id >= :id_gte") + params["id_gte"] = id_gte + if schema_id: + where.append("turns.schema_id = :schema_id") + params["schema_id"] = schema_id + if ids: + keys = [f"row_id_{index}" for index in range(len(ids))] + where.append("turns.id in ({})".format(", ".join(f":{key}" for key in keys))) + params.update(dict(zip(keys, ids))) + + # Fragments come from turn_fragments - what this call was given - + # rather than from the message text, so it matches what -f means. + for index, fragment_hash in enumerate(fragment_hashes): + key = f"fragment_{index}" + where.append(f"""turns.id in ( + select turn_fragments.turn_id from turn_fragments + join fragments on fragments.id = turn_fragments.fragment_id + where fragments.hash = :{key} + )""") + params[key] = fragment_hash + + # A turn "used" a tool when a tool result was among its inputs, + # matching what -T has always meant: the result came back and was + # fed to the model. The result sits in the turn's parent message. + if any_tools: + where.append(_tool_result_clause()) + for index, tool_name in enumerate(tool_names): + key = f"tool_{index}" + where.append(_tool_result_clause(f"and parts.tool_name = :{key}")) + params[key] = tool_name + + rank_select = "" + join = "" + order_by = "turns.id desc" + if query: + rank_select = f",\n {TURN_SEARCH_RANK} as _search_rank" + join = ( + "\njoin turn_search on turn_search.turn_id = turns.id" + "\njoin turn_search_fts on turn_search_fts.rowid = turn_search.id" + ) + where.append("turn_search_fts match :query") + params["query"] = query + if not latest: + order_by = TURN_SEARCH_RANK + + sql = LOG_ROWS_SQL.format( + rank_select=rank_select, + join=join, + where=("where " + " and ".join(where)) if where else "", + order_by=order_by, + limit=f" limit {count}" if count else "", + ) + builder = _LogRowBuilder(store) + return [builder.build(row) for row in store.db.query(sql, params)] + + +def _tool_result_clause(extra: str = "") -> str: + # A turn's tool results sit either in its parent message directly, + # or - when a fresh user prompt followed the results - in the + # parent's own parent, one step further up the same input segment. + return f"""(turns.parent_message_hash in ( + select parts.message_hash from parts + where parts.type = 'tool_result' {extra} + ) or turns.parent_message_hash in ( + select messages.hash from messages + join parts on parts.message_hash = messages.parent_hash + where messages.role = 'user' + and parts.type = 'tool_result' {extra} + ))""" + + +def log_row_extras(store: "LogStore", row: dict) -> dict: + """Attachments, fragments and tool info for one `llm logs` row. + + Attachments and tool calls come from the row's parts, which the row + builder kept hold of. Fragments come from turn_fragments - what the + call was given - so `-f` and the displayed list agree. + """ + attachments = [ + _attachment_summary(part.attachment) + for part in row.get("_input_parts", []) + if isinstance(part, AttachmentPart) and part.attachment is not None + ] + + # parts.id and tools.id stand in for the row ids the old + # tool_calls / tool_results tables exposed, so the JSON shape of + # `llm logs` is unchanged. + call_ids = _part_ids(store, [row.get("_tip_message_hash")], "tool_call") + result_ids = _part_ids(store, row.get("_input_message_hashes") or [], "tool_result") + + # input_schema is rendered as a dict, so decode it here rather than + # handing the caller the raw JSON text out of the column. + tools = [ + { + "id": tool_row["id"], + "hash": tool_row["hash"], + "name": tool_row["name"], + "description": tool_row["description"], + "input_schema": json.loads(tool_row["input_schema"] or "{}"), + "instance": ( + { + "name": tool_row["instance_name"], + "arguments": tool_row["instance_arguments"], + } + if tool_row["instance_name"] + else None + ), + } + for tool_row in store.db.query( + """ + select tools.id, tools.hash, tools.name, tools.description, + tools.input_schema, tool_instances.name as instance_name, + tool_instances.arguments as instance_arguments + from tools join turn_tools on turn_tools.tool_id = tools.id + left join tool_instances + on tool_instances.id = turn_tools.instance_id + where turn_tools.turn_id = ? + """, + [row["id"]], + ) + ] + # Resolved through this turn's own turn_tools rows - definitions + # sharing a name can differ between turns, and a global map would + # attribute the wrong one. + tool_ids = {tool["name"]: tool["id"] for tool in tools} + + tool_calls = [ + { + "id": call_ids.get(part.tool_call_id), + "tool_id": tool_ids.get(part.name), + "name": part.name, + "arguments": part.arguments, + "tool_call_id": part.tool_call_id, + } + for part in row.get("_output_parts", []) + if isinstance(part, ToolCallPart) + ] + result_parts = [ + part for part in row.get("_input_parts", []) if isinstance(part, ToolResultPart) + ] + instances = _instances_by_tool_call_id( + store, + row["id"], + [part.tool_call_id for part in result_parts if part.tool_call_id], + ) + tool_results = [ + { + "id": result_ids.get(part.tool_call_id), + "tool_id": tool_ids.get(part.name), + "name": part.name, + "output": part.output, + "tool_call_id": part.tool_call_id, + "exception": part.exception, + "instance": instances.get(part.tool_call_id), + "attachments": [_attachment_summary(a) for a in part.attachments], + } + for part in result_parts + ] + + fragments: dict[str, list[dict]] = { + "prompt_fragments": [], + "system_fragments": [], + } + for fragment_row in store.db.query( + """ + select turn_fragments.kind, fragments.hash, fragments.content, + (select json_group_array(fragment_aliases.alias) + from fragment_aliases + where fragment_aliases.fragment_id = fragments.id) as aliases + from turn_fragments + join fragments on fragments.id = turn_fragments.fragment_id + where turn_fragments.turn_id = ? + order by turn_fragments."order" + """, + [row["id"]], + ): + key = f"{fragment_row['kind']}_fragments" + fragments[key].append(dict(fragment_row)) + + return { + "attachments": attachments, + "tools": tools, + "tool_calls": tool_calls, + "tool_results": tool_results, + **fragments, + } + + +def _attachment_summary(attachment) -> dict: + "The attachment shape `llm logs` renders." + content = attachment.content or b"" + return { + "id": attachment.id(), + "type": attachment.resolve_type(), + "path": attachment.path, + "url": attachment.url, + "content": bool(content) or None, + "content_length": len(content) or None, + } + + +def _part_ids(store: "LogStore", message_hashes: list, type: str) -> dict: + "Map tool_call_id to the parts row id, across the given messages." + message_hashes = [hash_ for hash_ in message_hashes if hash_] + if not message_hashes: + return {} + placeholders = ",".join("?" * len(message_hashes)) + return { + json.loads(part_row["payload"]).get("tool_call_id"): part_row["id"] + for part_row in store.db.query( + f"select id, payload from parts" + f" where message_hash in ({placeholders}) and type = ?", + message_hashes + [type], + ) + } + + +def _instances_by_tool_call_id( + store: "LogStore", turn_id: str, tool_call_ids: list +) -> dict: + """Which configured toolbox instance served each call, for display. + + Scoped to the turn: providers with per-request counters can reuse + the same tool_call_id across independent turns. + """ + if not tool_call_ids: + return {} + placeholders = ",".join("?" * len(tool_call_ids)) + return { + row["tool_call_id"]: { + "name": row["name"], + "plugin": row["plugin"], + "arguments": row["arguments"], + } + for row in store.db.query( + f""" + select tool_instantiations.tool_call_id, tool_instances.name, + tool_instances.plugin, tool_instances.arguments + from tool_instantiations + join tool_instances + on tool_instances.id = tool_instantiations.instance_id + where tool_instantiations.turn_id = ? + and tool_instantiations.tool_call_id in ({placeholders}) + """, + [turn_id] + tool_call_ids, + ) + } + + +# -- legacy rows --------------------------------------------------------- +# +# History logged by older versions of llm lives only in the `responses` +# table. Those rows are merged into `llm logs` output, shaped like the +# rows _LogRowBuilder produces. A response whose id also exists in +# `turns` is suppressed - that is what a dual-write-era row or a +# backfilled conversation looks like, and the turn is the richer record. + +LEGACY_LOG_ROWS_SQL = """ +select + responses.id, + responses.model, + responses.resolved_model, + responses.prompt, + responses.system, + responses.prompt_json, + responses.options_json, + responses.response, + responses.reasoning, + responses.response_json, + responses.conversation_id, + responses.duration_ms, + responses.datetime_utc, + responses.input_tokens, + responses.output_tokens, + responses.token_details, + conversations.name as conversation_name, + conversations.model as conversation_model, + schemas.content as schema_json{rank_select} +from responses +left join schemas on responses.schema_id = schemas.id +left join conversations on responses.conversation_id = conversations.id{join} +where responses.id not in (select id from turns){extra_where} +order by {order_by}{limit} +""" + + +def legacy_log_rows( + db, + *, + count: int | None = None, + model_id: str | None = None, + thread_id: str | None = None, + fragment_hashes=(), + tool_names=(), + any_tools: bool = False, + schema_id: str | None = None, + id_gt: str | None = None, + id_gte: str | None = None, + ids=(), + query: str | None = None, + latest: bool = False, +) -> list[dict]: + """Rows for `llm logs` that exist only in the legacy tables. + + Applies the same filters as log_rows, translated to the legacy + schema. Thread ids are conversation ids, so the two filters match + the same conversations on either side of the upgrade. + """ + if query and "responses_fts" not in db.table_names(): + # A database that never saw legacy llm has no legacy index. + return [] + where: list[str] = [] + params: dict[str, Any] = {} + + if model_id: + where.append("(responses.model = :model or responses.resolved_model = :model)") + params["model"] = model_id + if thread_id: + where.append("responses.conversation_id = :thread_id") + params["thread_id"] = thread_id + if id_gt: + where.append("responses.id > :id_gt") + params["id_gt"] = id_gt + if id_gte: + where.append("responses.id >= :id_gte") + params["id_gte"] = id_gte + if schema_id: + where.append("responses.schema_id = :schema_id") + params["schema_id"] = schema_id + if ids: + keys = [f"row_id_{index}" for index in range(len(ids))] + where.append( + "responses.id in ({})".format(", ".join(f":{key}" for key in keys)) + ) + params.update(dict(zip(keys, ids))) + + for index, fragment_hash in enumerate(fragment_hashes): + key = f"fragment_{index}" + where.append(f"""( + exists ( + select 1 from prompt_fragments + where prompt_fragments.response_id = responses.id + and prompt_fragments.fragment_id in ( + select fragments.id from fragments where hash = :{key} + ) + ) + or exists ( + select 1 from system_fragments + where system_fragments.response_id = responses.id + and system_fragments.fragment_id in ( + select fragments.id from fragments where hash = :{key} + ) + ) + )""") + params[key] = fragment_hash + + if any_tools: + where.append("""exists ( + select 1 from tool_results + where tool_results.response_id = responses.id + )""") + for index, tool_name in enumerate(tool_names): + key = f"tool_{index}" + where.append(f"""exists ( + select 1 from tool_results + join tools on tools.id = tool_results.tool_id + where tool_results.response_id = responses.id + and tools.name = :{key} + )""") + params[key] = tool_name + + rank_select = "" + join = "" + order_by = "responses.id desc" + if query: + rank_select = f",\n {LEGACY_SEARCH_RANK} as _search_rank" + join = "\njoin responses_fts on responses_fts.rowid = responses.rowid" + where.append("responses_fts match :query") + params["query"] = query + if not latest: + order_by = LEGACY_SEARCH_RANK + + sql = LEGACY_LOG_ROWS_SQL.format( + rank_select=rank_select, + join=join, + extra_where=(" and " + " and ".join(where)) if where else "", + order_by=order_by, + limit=f" limit {count}" if count else "", + ) + rows = [dict(row) for row in db.query(sql, params)] + for row in rows: + row["_legacy"] = True + return rows + + +def merged_log_rows( + store: "LogStore", + *, + count: int | None = None, + query: str | None = None, + latest: bool = False, + **filters, +): + """Rows for `llm logs`: the new tables plus legacy-only responses. + + Both sides are newest-first and ids are ULIDs on both sides of the + upgrade, so a straight sort interleaves the two histories + chronologically and the top ``count`` of the union is always within + the top ``count`` of each side. + + With ``query``, each side returns its best matches and the union is + ordered most-relevant first. The two bm25 scores come from separate + indexes so the interleave is approximate - close enough in practice, + since both index the same kind of corpus with the same tokenizer. + ``latest`` makes the query a pure filter and keeps recency order. + """ + rows = log_rows(store, count=count, query=query, latest=latest, **filters) + rows.extend( + legacy_log_rows(store.db, count=count, query=query, latest=latest, **filters) + ) + if query and not latest: + rows.sort(key=lambda row: row["_search_rank"]) + else: + rows.sort(key=lambda row: row["id"], reverse=True) + if count: + rows = rows[:count] + return rows + + +LEGACY_ATTACHMENTS_SQL = """ +select + response_id, + attachments.id, + attachments.type, + attachments.path, + attachments.url, + length(attachments.content) as content_length +from attachments +join prompt_attachments + on attachments.id = prompt_attachments.attachment_id +where prompt_attachments.response_id in ({placeholders}) +order by prompt_attachments."order" +""" + +LEGACY_FRAGMENTS_SQL = """ +select + {table}.response_id, + fragments.hash, + fragments.id as fragment_id, + fragments.content, + ( + select json_group_array(fragment_aliases.alias) + from fragment_aliases + where fragment_aliases.fragment_id = fragments.id + ) as aliases +from {table} +join fragments on {table}.fragment_id = fragments.id +where {table}.response_id in ({placeholders}) +order by {table}."order" +""" + +LEGACY_TOOLS_SQL = """ +select responses.id, + coalesce( + (select json_group_array(json_object( + 'id', t.id, + 'hash', t.hash, + 'name', t.name, + 'description', t.description, + 'input_schema', json(t.input_schema), + 'instance', null + )) + from tools t + join tool_responses tr on t.id = tr.tool_id + where tr.response_id = responses.id + ), + '[]' + ) as tools, + coalesce( + (select json_group_array(json_object( + 'id', tc.id, + 'tool_id', tc.tool_id, + 'name', tc.name, + 'arguments', json(tc.arguments), + 'tool_call_id', tc.tool_call_id + )) + from tool_calls tc + where tc.response_id = responses.id + ), + '[]' + ) as tool_calls, + coalesce( + (select json_group_array(json_object( + 'id', tr.id, + 'tool_id', tr.tool_id, + 'name', tr.name, + 'output', tr.output, + 'tool_call_id', tr.tool_call_id, + 'exception', tr.exception, + 'instance', case when ti.id is not null then json_object( + 'name', ti.name, + 'plugin', ti.plugin, + 'arguments', ti.arguments + ) else null end, + 'attachments', coalesce( + (select json_group_array(json_object( + 'id', a.id, + 'type', a.type, + 'path', a.path, + 'url', a.url, + 'content', a.content + )) + from tool_results_attachments tra + join attachments a on tra.attachment_id = a.id + where tra.tool_result_id = tr.id + ), + '[]' + ) + )) + from tool_results tr + left join tool_instances ti on tr.instance_id = ti.id + where tr.response_id = responses.id + ), + '[]' + ) as tool_results +from responses +where id in ({placeholders}) +""" + + +def legacy_log_row_extras(db, ids: list[str]) -> dict[str, dict]: + """Extras for legacy rows, batch-fetched, keyed by response id. + + Same shape as log_row_extras, with each entry shaped the way the + pre-turns `llm logs` rendered it. + """ + extras: dict[str, dict] = { + id: { + "attachments": [], + "prompt_fragments": [], + "system_fragments": [], + "tools": [], + "tool_calls": [], + "tool_results": [], + } + for id in ids + } + if not ids: + return extras + placeholders = ",".join("?" * len(ids)) + + for attachment in db.query( + LEGACY_ATTACHMENTS_SQL.format(placeholders=placeholders), ids + ): + attachment = dict(attachment) + response_id = attachment.pop("response_id") + extras[response_id]["attachments"].append(attachment) + + for table in ("prompt_fragments", "system_fragments"): + for fragment in db.query( + LEGACY_FRAGMENTS_SQL.format(table=table, placeholders=placeholders), ids + ): + fragment = dict(fragment) + response_id = fragment.pop("response_id") + extras[response_id][table].append(fragment) + + for row in db.query(LEGACY_TOOLS_SQL.format(placeholders=placeholders), ids): + extras[row["id"]].update( + { + "tools": json.loads(row["tools"]), + "tool_calls": json.loads(row["tool_calls"]), + "tool_results": json.loads(row["tool_results"]), + } + ) + + return extras diff --git a/llm/migrations.py b/llm/migrations.py index f2ca04651..bc26a5759 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -1,7 +1,7 @@ import datetime -from typing import Callable, List +from collections.abc import Callable -MIGRATIONS: List[Callable] = [] +MIGRATIONS: list[Callable] = [] migration = MIGRATIONS.append @@ -134,8 +134,7 @@ def m007_finish_logs_table(db): rename={"timestamp_utc": "datetime_utc"}, drop_foreign_keys=("chat_id",), ) - with db.conn: - db.execute("alter table log rename to logs") + db.execute("alter table log rename to logs") @migration @@ -418,3 +417,318 @@ def m020_tool_results_attachments(db): @migration def m021_tool_results_exception(db): db["tool_results"].add_column("exception", str) + + +@migration +def m022_response_reasoning(db): + # Concatenated visible reasoning text emitted during the response. + # NULL/empty when no reasoning was emitted or when the provider + # only reported an opaque token count (the redacted-marker case). + db["responses"].add_column("reasoning", str) + + +@migration +def m023_message_store(db): + # The content-addressed message store + db["messages"].create( + { + "hash": str, + "parent_hash": str, + "role": str, + "provider_metadata": str, + }, + pk="hash", + foreign_keys=(("parent_hash", "messages", "hash"),), + ) + # Needed by the recursive descent through the tree and by the + # child count that identifies a fork. + db["messages"].create_index(["parent_hash"]) + + db["parts"].create( + { + "id": int, + "message_hash": str, + "position": int, + # text | reasoning | tool_call | tool_result | attachment + "type": str, + # Tool name for tool_call and tool_result parts, else NULL. + "tool_name": str, + # The part's literal text, for text and reasoning parts whose + # text borrows no fragments. Raw and unescaped - this column + # is never parsed as anything. + "text": str, + # The part's remaining structure as JSON, with large content + # replaced by references (fragment ids for text, attachment + # ids for binary) and the type key left to the column above. + # NULL when the text column carries everything. + "payload": str, + }, + pk="id", + foreign_keys=(("message_hash", "messages", "hash"),), + ) + # The hot read path, and it enforces one part per position. + db["parts"].create_index(["message_hash", "position"], unique=True) + + # Attachment and fragment ids are in the payload too. These tables + # exist so referential integrity, garbage collection reachability and + # "everything that used X" are plain joins rather than a json_each + # over every payload in the database. + db["part_attachments"].create( + { + "part_id": int, + "attachment_id": str, + "order": int, + }, + pk=("part_id", "attachment_id", "order"), + foreign_keys=( + ("part_id", "parts", "id"), + ("attachment_id", "attachments", "id"), + ), + ) + + db["part_fragments"].create( + { + "part_id": int, + "fragment_id": int, + "order": int, + }, + pk=("part_id", "fragment_id", "order"), + foreign_keys=( + ("part_id", "parts", "id"), + ("fragment_id", "fragments", "id"), + ), + ) + db["part_fragments"].create_index(["fragment_id"]) + + # The only mutable rows in the schema. A fork is a second thread + # pointing at a message that already exists. + db["threads"].create( + { + "id": str, + "name": str, + "tip_message_hash": str, + "forked_from": str, + "datetime_utc": str, + }, + pk="id", + foreign_keys=( + ("tip_message_hash", "messages", "hash"), + ("forked_from", "threads", "id"), + ), + ) + + # One model call. Self-contained - it does not read anything from the + # older responses table. parent_ and tip_message_hash together + # delimit what this turn contributed, which nothing else records, + # because message rows are shared and cannot carry provenance. + db["turns"].create( + { + "id": str, + "thread_id": str, + "parent_message_hash": str, + "tip_message_hash": str, + "model": str, + "resolved_model": str, + "options_json": str, + "schema_id": str, + "input_tokens": int, + "output_tokens": int, + "token_details": str, + "duration_ms": int, + "datetime_utc": str, + }, + pk="id", + foreign_keys=( + ("thread_id", "threads", "id"), + ("parent_message_hash", "messages", "hash"), + ("tip_message_hash", "messages", "hash"), + ("schema_id", "schemas", "id"), + ), + ) + db["turns"].create_index(["thread_id"]) + + db["turn_tools"].create( + {"turn_id": str, "tool_id": int}, + pk=("turn_id", "tool_id"), + foreign_keys=(("turn_id", "turns", "id"), ("tool_id", "tools", "id")), + ) + + # Provenance: which fragments this call was given. Distinct from + # part_fragments, which says what a message's text is built from. + db["turn_fragments"].create( + { + "turn_id": str, + "fragment_id": int, + "order": int, + "kind": str, # 'prompt' | 'system' + }, + pk=("turn_id", "fragment_id", "kind", "order"), + foreign_keys=( + ("turn_id", "turns", "id"), + ("fragment_id", "fragments", "id"), + ), + ) + db["turn_fragments"].create_index(["fragment_id"]) + + # Searchable text per turn: the user's typed prompt (fragment + # content excluded) and the assistant's text output, kept fresh by + # LogStore.log. An explicit id primary key because external-content + # FTS is keyed by rowid, and implicit rowids are not stable across + # VACUUM. + db["turn_search"].create( + { + "id": int, + "turn_id": str, + "prompt": str, + "response": str, + }, + pk="id", + foreign_keys=(("turn_id", "turns", "id"),), + ) + db["turn_search"].create_index(["turn_id"], unique=True) + db["turn_search"].enable_fts(["prompt", "response"], create_triggers=True) + + # Which configured toolbox instance served a tool call: the toolbox + # name, its plugin and its constructor arguments. Local execution + # provenance, so it lives outside the hashed message tree, joined + # to the chain by tool_call_id - and keyed by (turn_id, + # tool_call_id), because provider-supplied call ids are not + # guaranteed unique across turns. Deliberately the seed of a fuller + # execution-events table: duration or exception details would be + # additive columns here. + db["tool_instantiations"].create( + { + "turn_id": str, + "tool_call_id": str, + "name": str, + "plugin": str, + "arguments": str, + }, + pk=("turn_id", "tool_call_id"), + foreign_keys=(("turn_id", "turns", "id"),), + ) + + +@migration +def m024_tool_instance_references(db): + # Tool instance configurations - e.g. Datasette("https://...") - + # are stored once in the shared tool_instances table and referenced + # by id, instead of being copied onto every row that mentions them: + # tool_instantiations gains instance_id in place of its + # name/plugin/arguments copies, and turn_tools gains instance_id so + # the tools list can show which configured instance provided each + # tool. + from .utils import ensure_tool_instance + + db["turn_tools"].add_column("instance_id", int, fk="tool_instances", fk_col="id") + db["tool_instantiations"].add_column( + "instance_id", int, fk="tool_instances", fk_col="id" + ) + for row in list(db["tool_instantiations"].rows): + db["tool_instantiations"].update( + (row["turn_id"], row["tool_call_id"]), + { + "instance_id": ensure_tool_instance( + db, row["name"], row["plugin"], row["arguments"] + ) + }, + ) + db["tool_instantiations"].transform(drop={"name", "plugin", "arguments"}) + + +@migration +def m025_turn_tools_instance_backfill(db): + # turn_tools rows written before instance_id existed have NULL + # there, so the tools list shows nothing for them. The instance + # that served calls in the same thread, matched by toolbox name + # prefix, is the right value for these development-era rows. + with db.atomic(): + db.execute(""" + update turn_tools set instance_id = ( + select ti.instance_id from tool_instantiations ti + join turns turn_a on turn_a.id = ti.turn_id + join turns turn_b on turn_b.id = turn_tools.turn_id + and turn_b.thread_id = turn_a.thread_id + join tool_instances instance + on instance.id = ti.instance_id + join tools on tools.id = turn_tools.tool_id + where tools.name = instance.name + or tools.name like instance.name || '\\_%' escape '\\' + limit 1 + ) + where instance_id is null + """) + + +MESSAGE_TREE_SQL = """ +with recursive msg as ( + select m.hash, m.parent_hash, m.role, m.rowid as rid, + replace(coalesce( + nullif(p.text, ''), + (select f.content from part_fragments pf + join fragments f on f.id = pf.fragment_id + where pf.part_id = p.id + order by pf."order" limit 1), + '[' || coalesce(p.type, 'empty') || ']' + ), char(10), ' ') as text, + (select group_concat(p2.tool_name, ', ') from parts p2 + where p2.message_hash = m.hash and p2.type = 'tool_result' + and p2.tool_name is not null) as tools + from messages m + left join parts p on p.message_hash = m.hash and p.position = 0 +), +tree as ( + select hash, text, tools, 0 as depth, + printf('%012d', rid) as path, hash as root_hash + from msg where parent_hash is null + union all + select msg.hash, msg.text, msg.tools, t.depth + 1, + t.path || '/' || printf('%012d', msg.rid), + t.root_hash + from msg join tree t on msg.parent_hash = t.hash +), +turn_chain as ( + select t.id as turn_id, t.datetime_utc, m.hash, m.parent_hash + from turns t join messages m on m.hash = t.tip_message_hash + union all + select tc.turn_id, tc.datetime_utc, m.hash, m.parent_hash + from turn_chain tc join messages m on m.hash = tc.parent_hash +) +select + t.root_hash, + strftime('%Y-%m-%d %H:%M:%S', + (select min(tc.datetime_utc) from turn_chain tc where tc.hash = t.hash) + ) as datetime, + replace(hex(zeroblob(t.depth)), '00', ' ') || substr(t.text, 1, 60) + as message, + coalesce(t.tools, '') as tools, + t.hash as message_hash, + t.path +from tree t +order by t.path +""".strip() + + +@migration +def m026_message_tree_view(db): + # A readable rendering of the message store: every conversation + # tree as indented text, one row per message, depth-first with + # siblings in insertion order. root_hash identifies a tree - filter + # on it to isolate one conversation and its forks. datetime is the + # earliest turn that recorded the message, since shared message + # rows carry no timestamp of their own. Kept ordered by including + # path in the output - selecting from the view preserves tree order + # only while sorted by path. + db.create_view("message_tree", MESSAGE_TREE_SQL) + + +@migration +def m027_turns_response_json(db): + # The raw provider payload for the call, stored condensed: strings + # that already live in the turn's message parts (response text, + # reasoning blobs, long tool arguments) are replaced with references + # via condense-json, so the column costs roughly the provider + # envelope - ids, usage, fingerprints - not a second copy of the + # response. NULL for turns logged before this column existed and + # for models that expose no raw payload. + db["turns"].add_column("response_json", str) diff --git a/llm/models.py b/llm/models.py index 5e7676eb1..dc66c5c39 100644 --- a/llm/models.py +++ b/llm/models.py @@ -1,63 +1,75 @@ import asyncio import base64 -from condense_json import condense_json -from dataclasses import dataclass, field +import dataclasses import datetime -from .errors import NeedsKeyException +import functools import hashlib -import httpx -from itertools import islice -from pathlib import Path import re import time -from types import MethodType -from typing import ( - Any, +from collections.abc import ( AsyncGenerator, AsyncIterator, Awaitable, Callable, - Dict, Iterable, Iterator, - List, +) +from dataclasses import dataclass, field +from itertools import islice +from pathlib import Path +from types import MethodType +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, Optional, - Set, Union, + cast, get_type_hints, ) + +import httpx + +from .errors import NeedsKeyException +from .serialization import ResponseDict + +if TYPE_CHECKING: + from .parts import StreamEvent +import inspect +import json +from abc import ABC, abstractmethod + +from pydantic import BaseModel, ConfigDict, create_model + from .utils import ( - ensure_fragment, - ensure_tool, - make_schema_id, + Fragment, mimetype_from_path, mimetype_from_string, - token_usage_string, monotonic_ulid, - Fragment, + token_usage_string, ) -from abc import ABC, abstractmethod -import inspect -import json -from pydantic import BaseModel, ConfigDict, create_model CONVERSATION_NAME_LENGTH = 32 @dataclass class Usage: - input: Optional[int] = None - output: Optional[int] = None - details: Optional[Dict[str, Any]] = None + "Token usage information from a model response." + + input: int | None = None + output: int | None = None + details: dict[str, Any] | None = None @dataclass class Attachment: - type: Optional[str] = None - path: Optional[str] = None - url: Optional[str] = None - content: Optional[bytes] = None - _id: Optional[str] = None + "An attachment (image, audio, etc) to include with a prompt." + + type: str | None = None + path: str | None = None + url: str | None = None + content: bytes | None = None + _id: str | None = None def id(self): # Hash of the binary content, or of '{"url": "https://..."}' for URL attachments @@ -73,13 +85,15 @@ def id(self): return self._id def resolve_type(self): + "Return the content type, guessing from content if not specified." if self.type: return self.type # Derive it from path or url or content if self.path: return mimetype_from_path(self.path) if self.url: - response = httpx.head(self.url) + with httpx.Client(follow_redirects=True, max_redirects=3) as client: + response = client.head(self.url) response.raise_for_status() return response.headers.get("content-type") if self.content: @@ -87,17 +101,20 @@ def resolve_type(self): raise ValueError("Attachment has no type and no content to derive it from") def content_bytes(self): + "Return the binary content, reading from path or URL if needed." content = self.content if not content: if self.path: content = Path(self.path).read_bytes() elif self.url: - response = httpx.get(self.url) + with httpx.Client(follow_redirects=True, max_redirects=3) as client: + response = client.get(self.url) response.raise_for_status() content = response.content return content def base64_content(self): + "Return the content as a base64-encoded string." return base64.b64encode(self.content_bytes()).decode("utf-8") def __repr__(self): @@ -125,11 +142,13 @@ def from_row(cls, row): @dataclass class Tool: + "A tool that can be called by a model." + name: str - description: Optional[str] = None - input_schema: Dict = field(default_factory=dict) - implementation: Optional[Callable] = None - plugin: Optional[str] = None # plugin tool came from, e.g. 'llm_tools_sqlite' + description: str | None = None + input_schema: dict = field(default_factory=dict) + implementation: Callable | None = None + plugin: str | None = None # plugin tool came from, e.g. 'llm_tools_sqlite' def __post_init__(self): # Convert Pydantic model to JSON schema if needed @@ -168,12 +187,83 @@ def function(cls, function, name=None, description=None): ) +class ServerSideTool: + """A tool executed inside the provider's infrastructure. + + Instances are passed in ``tools=[...]`` alongside function tools. The + framework transports and validates them but never executes them. Provider + plugins subclass this class for their own tools; the base class can be + instantiated directly with a raw provider tool specification. + """ + + name: ClassVar[str] = "server_side_tool" + plugin: ClassVar[str | None] = None + implementation: ClassVar[None] = None + input_schema: ClassVar[dict] = {} + + def __init__(self, spec: dict | None = None): + self.spec = spec + self._config = {"spec": spec} + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + original_init = cls.__init__ + + @functools.wraps(original_init) + def wrapped_init(self, *args, **kwargs): + signature = inspect.signature(original_init) + bound = signature.bind(self, *args, **kwargs) + bound.apply_defaults() + original_init(self, *args, **kwargs) + self._config = { + name: value + for name, value in bound.arguments.items() + if name != "self" + and signature.parameters[name].kind + not in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ) + } + + cls.__init__ = wrapped_init + + @property + def description(self) -> str | None: + return inspect.getdoc(self.__class__) + + def tool_spec(self, model) -> dict: + """Return this tool's provider-specific request specification.""" + if self.spec is None: + raise TypeError( + f"{self.__class__.__name__} does not define a raw provider tool spec" + ) + return self.spec + + def prepare_request(self, model, kwargs: dict) -> None: + """Add any other values this tool needs to provider request kwargs.""" + + def hash(self): + """Hash the definition separately from configured instances.""" + to_hash = { + "name": self.name, + "description": self.description, + "input_schema": self.input_schema, + "server_side": True, + } + if self.plugin: + to_hash["plugin"] = self.plugin + return hashlib.sha256(json.dumps(to_hash).encode("utf-8")).hexdigest() + + def _get_arguments_input_schema(function, name): signature = inspect.signature(function) type_hints = get_type_hints(function) fields = {} for param_name, param in signature.parameters.items(): - if param_name == "self": + if param_name in ("self", "llm_tool_call"): + # llm_tool_call is reserved: populated with the ToolCall object + # at execution time, never exposed to the model. continue # Determine the type annotation (default to string if missing) annotated_type = type_hints.get(param_name, str) @@ -187,9 +277,29 @@ def _get_arguments_input_schema(function, name): return create_model(f"{name}InputSchema", **fields) +def _accepts_llm_tool_call(implementation) -> bool: + try: + signature = inspect.signature(implementation) + except (TypeError, ValueError): + return False + return "llm_tool_call" in signature.parameters + + +def _implementation_arguments(tool: "Tool", tool_call: "ToolCall") -> dict: + """Arguments to invoke a tool implementation with. + + Implementations with an explicit ``llm_tool_call`` parameter receive + the ToolCall object itself - a ``**kwargs`` catch-all does not count. + """ + arguments = dict(tool_call.arguments) + if _accepts_llm_tool_call(tool.implementation): + arguments["llm_tool_call"] = tool_call + return arguments + + class Toolbox: - name: Optional[str] = None - instance_id: Optional[int] = None + name: str | None = None + instance_id: int | None = None _blocked = ( "tools", "add_tool", @@ -198,8 +308,8 @@ class Toolbox: "prepare", "prepare_async", ) - _extra_tools: List[Tool] = [] - _config: Dict[str, Any] = {} + _extra_tools: ClassVar[list[Tool]] = [] + _config: ClassVar[dict[str, Any]] = {} _prepared: bool = False _async_prepared: bool = False @@ -208,6 +318,7 @@ def __init_subclass__(cls, **kwargs): original_init = cls.__init__ + @functools.wraps(original_init) def wrapped_init(self, *args, **kwargs): # Track args/kwargs passed to constructor in self._config # so we can serialize them to a database entry later on @@ -229,7 +340,7 @@ def wrapped_init(self, *args, **kwargs): cls.__init__ = wrapped_init @classmethod - def method_tools(cls) -> List[Tool]: + def method_tools(cls) -> list[Tool]: tools = [] for method_name in dir(cls): if method_name.startswith("_") or method_name in cls._blocked: @@ -238,7 +349,7 @@ def method_tools(cls) -> List[Tool]: if callable(method): tool = Tool.function( method, - name="{}_{}".format(cls.__name__, method_name), + name=f"{cls.__name__}_{method_name}", ) tools.append(tool) return tools @@ -257,7 +368,7 @@ def tools(self) -> Iterable[Tool]: yield from self._extra_tools def add_tool( - self, tool_or_function: Union[Tool, Callable[..., Any]], pass_self: bool = False + self, tool_or_function: Tool | Callable[..., Any], pass_self: bool = False ): "Add a tool to this toolbox" @@ -271,71 +382,112 @@ def _upgrade(fn): elif callable(tool_or_function): self._extra_tools.append(Tool.function(_upgrade(tool_or_function))) else: - raise ValueError("Tool must be an instance of Tool or a callable function") + raise TypeError("Tool must be an instance of Tool or a callable function") def prepare(self): """ Over-ride this to perform setup (and .add_tool() calls) before the toolbox is used. Implement a similar prepare_async() method for async setup. """ - pass async def prepare_async(self): """ Over-ride this to perform async setup (and .add_tool() calls) before the toolbox is used. """ - pass @dataclass class ToolCall: + "A request by the model to call a tool." + name: str arguments: dict - tool_call_id: Optional[str] = None + tool_call_id: str | None = None + + +def _ensure_tool_call_id(tool_call: ToolCall) -> ToolCall: + # Generate a tool call ID if one has not yet been specified + if tool_call.tool_call_id is not None: + return tool_call + return dataclasses.replace( + tool_call, + tool_call_id=f"tc_{str(monotonic_ulid()).lower()}", + ) @dataclass class ToolResult: + "The result of executing a tool call." + name: str output: str - attachments: List[Attachment] = field(default_factory=list) - tool_call_id: Optional[str] = None - instance: Optional[Toolbox] = None - exception: Optional[Exception] = None + attachments: list[Attachment] = field(default_factory=list) + tool_call_id: str | None = None + instance: Toolbox | None = None + exception: Exception | None = None @dataclass class ToolOutput: "Tool functions can return output with extra attachments" - output: Optional[Union[str, dict, list, bool, int, float]] = None - attachments: List[Attachment] = field(default_factory=list) + output: str | dict | list | bool | int | float | None = None + attachments: list[Attachment] = field(default_factory=list) -ToolDef = Union[Tool, Toolbox, Callable[..., Any]] -BeforeCallSync = Callable[[Optional[Tool], ToolCall], None] +ToolDef = Tool | Toolbox | ServerSideTool | Callable[..., Any] +BeforeCallSync = Callable[[Tool | None, ToolCall], None] AfterCallSync = Callable[[Tool, ToolCall, ToolResult], None] -BeforeCallAsync = Callable[[Optional[Tool], ToolCall], Union[None, Awaitable[None]]] -AfterCallAsync = Callable[[Tool, ToolCall, ToolResult], Union[None, Awaitable[None]]] +BeforeCallAsync = Callable[[Tool | None, ToolCall], None | Awaitable[None]] +AfterCallAsync = Callable[[Tool, ToolCall, ToolResult], None | Awaitable[None]] class CancelToolCall(Exception): pass +class PauseChain(Exception): + """Raise inside a tool implementation to pause the chain. + + Unlike other exceptions - which are converted into error ToolResults + and sent back to the model - PauseChain propagates out of + ``execute_tool_calls()`` and ``chain()``. Before it is re-raised the + framework populates two attributes: + + - ``tool_call``: the ToolCall whose implementation paused + - ``tool_results``: ToolResults of sibling calls in the same batch + that completed + + Concurrent (async) sibling tool calls always run to completion + before the exception propagates; sequential (sync) execution stops + at the paused call, leaving later calls unexecuted so they can + safely run when the chain is resumed. Resume by re-running the + chain with a ``messages=`` history that ends in the unresolved tool + calls. + """ + + def __init__(self, *args): + super().__init__(*args) + self.tool_call: ToolCall | None = None + self.tool_results: list[ToolResult] = [] + + @dataclass class Prompt: - _prompt: Optional[str] + "The prompt being sent to the model." + + _prompt: str | None model: "Model" - fragments: Optional[List[Union[str, Fragment]]] - attachments: Optional[List[Attachment]] - _system: Optional[str] - system_fragments: Optional[List[Union[str, Fragment]]] - prompt_json: Optional[str] - schema: Optional[Union[Dict, type[BaseModel]]] - tools: List[Tool] - tool_results: List[ToolResult] + fragments: list[str | Fragment] | None + attachments: list[Attachment] | None + _system: str | None + system_fragments: list[str | Fragment] | None + prompt_json: str | None + schema: dict | type[BaseModel] | None + tools: list[Tool | ServerSideTool] + tool_results: list[ToolResult] options: "Options" + hide_reasoning: bool def __init__( self, @@ -351,6 +503,8 @@ def __init__( schema=None, tools=None, tool_results=None, + messages=None, + hide_reasoning=False, ): self._prompt = prompt self.model = model @@ -365,70 +519,336 @@ def __init__( self.tools = _wrap_tools(tools or []) self.tool_results = tool_results or [] self.options = options or {} + self.hide_reasoning = hide_reasoning + # Explicit messages= list, if the caller supplied one. Copied so + # later mutation by the caller doesn't alter the Prompt. + self._explicit_messages = list(messages) if messages is not None else None @property def prompt(self): + "The text of the prompt, with any fragments concatenated." return "\n".join(self.fragments + ([self._prompt] if self._prompt else [])) @property def system(self): - bits = [ - bit.strip() - for bit in (self.system_fragments + [self._system or ""]) - if bit.strip() - ] - return "\n\n".join(bits) + "The system prompt, with any system fragments concatenated." + return _combine_system(self._system, self.system_fragments) + + @property + def messages(self): + """Canonical list of Message objects for this prompt. + + **Invariant:** this property returns exactly what the model + was (or will be) sent for this turn — the full chain including + any prior conversation history. + + - If ``messages=`` was passed explicitly, it is authoritative: + returned verbatim. Other kwargs (``prompt=``, ``system=``, + ``attachments=``, ``tool_results=``) are ignored for the + messages list (they remain available via ``prompt.prompt``, + ``prompt.system``, etc., for adapters that still read them). + - Otherwise the list is synthesized from the legacy kwargs + (system, tool_results, prompt, attachments), producing just + the current turn — prior history is not folded in, because + no conversation context is reachable here. + + Conversation.prompt / AsyncConversation.prompt / reply() all + pre-compute the full chain and pass it as ``messages=``, so + ``response.prompt.messages`` after those paths is the full + chain. + """ + from .parts import ( + AttachmentPart, + Message, + TextPart, + ToolResultPart, + ) + + if self._explicit_messages is not None: + return list(self._explicit_messages) + + result: list[Message] = [] + + if self.system: + result.append(Message(role="system", parts=[TextPart(text=self.system)])) + + if self.tool_results: + result.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + exception=_format_tool_exception(tr.exception), + attachments=list(tr.attachments or []), + ) + for tr in self.tool_results + ], + ) + ) + + user_parts: list[Any] = [] + if self.prompt: + user_parts.append(TextPart(text=self.prompt)) + for att in self.attachments: + user_parts.append(AttachmentPart(attachment=att)) + if user_parts: + result.append(Message(role="user", parts=user_parts)) + return result -def _wrap_tools(tools: List[ToolDef]) -> List[Tool]: + +def _wrap_tools(tools: list[ToolDef]) -> list[Tool | ServerSideTool]: wrapped_tools = [] for tool in tools: - if isinstance(tool, Tool): + if isinstance(tool, (Tool, ServerSideTool)): wrapped_tools.append(tool) elif isinstance(tool, Toolbox): wrapped_tools.extend(tool.tools()) elif callable(tool): wrapped_tools.append(Tool.function(tool)) else: - raise ValueError(f"Invalid tool: {tool}") + raise TypeError(f"Invalid tool: {tool}") return wrapped_tools +def _partition_tools( + model: "_BaseModel", tools: Iterable[Tool | ServerSideTool] +) -> tuple[list[Tool], list[ServerSideTool]]: + """Partition tools and reject server-side tools the model did not claim.""" + function_tools = [] + server_side_tools = [] + declared = tuple(model.supported_server_side_tools) + for tool in tools: + if isinstance(tool, ServerSideTool): + # Declaring ServerSideTool itself claims only direct raw-spec + # instances. Without this exact-type exception it would also + # accidentally claim every provider-specific subclass. + claimed = any( + ( + type(tool) is candidate + if candidate is ServerSideTool + else isinstance(tool, candidate) + ) + for candidate in declared + ) + if not claimed: + raise ValueError( + f"Model '{model.model_id}' does not support server-side tool " + f"'{tool.name}'. Run: llm tools -m {model.model_id}" + ) + server_side_tools.append(tool) + else: + function_tools.append(tool) + return function_tools, server_side_tools + + +def _append_turn_input( + chain: list[Any], + prompt: str | None, + fragments=None, + attachments=None, + tool_results=None, +) -> list[Any]: + """Append a turn's new input to a message chain. + + A tool-role message for any tool results, then a user-role message + built from fragments + prompt text + attachments. This is what makes + ``messages=`` mean "authoritative history" without the other prompt + arguments being silently dropped from the chain. Mutates and returns + ``chain``. + """ + from .parts import ( + AttachmentPart, + Message, + TextPart, + ToolResultPart, + ) + + if tool_results: + chain.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + exception=_format_tool_exception(tr.exception), + attachments=list(tr.attachments or []), + ) + for tr in tool_results + ], + ) + ) + + user_parts: list[Any] = [] + # Fragments are concatenated into the prompt text before it is + # sent, so they have to be in the chain too - prompt.messages is + # meant to be exactly what the model sees. Matches Prompt.prompt. + prompt_text = "\n".join( + [str(fragment) for fragment in fragments or []] + ([prompt] if prompt else []) + ) + if prompt_text: + user_parts.append(TextPart(text=prompt_text)) + for att in attachments or []: + user_parts.append(AttachmentPart(attachment=att)) + if user_parts: + chain.append(Message(role="user", parts=user_parts)) + return chain + + +def _combine_system(system, system_fragments): + "Concatenate the system prompt and any system fragments into one string." + bits = [ + bit.strip() + for bit in ((system_fragments or []) + [system or ""]) + if bit.strip() + ] + return "\n\n".join(bits) + + +def _merge_options(options: dict | None, kwargs: dict) -> dict: + if not options: + return kwargs + overlap = set(options) & set(kwargs) + if overlap: + raise TypeError( + "Got values for these options both in options= and as keyword " + f"arguments: {sorted(overlap)}" + ) + return {**options, **kwargs} + + @dataclass class _BaseConversation: model: "_BaseModel" id: str = field(default_factory=lambda: str(monotonic_ulid()).lower()) - name: Optional[str] = None - responses: List["_BaseResponse"] = field(default_factory=list) - tools: Optional[List[ToolDef]] = None - chain_limit: Optional[int] = None + name: str | None = None + responses: list["_BaseResponse"] = field(default_factory=list) + tools: list[ToolDef] | None = None + chain_limit: int | None = None + # History read back from storage, used as the chain for the next turn + # when this conversation has not yet produced a response in this + # process. Unlike a chain rebuilt from logged responses this is the + # exact message list, so reasoning signatures and provider metadata + # survive being reloaded. + loaded_messages: list[Any] | None = None + # Plugin and server-side tool names and configured specs (e.g. + # 'Datasette({"url": ...})' or 'CodeInterpreter({"memory_limit": + # "4g"})') recorded against this conversation's first turn in storage. + # Read when the conversation was loaded from the message store, where + # there are no rebuilt responses to copy prompt.tools from. + loaded_tools: list[str] | None = None @classmethod @abstractmethod def from_row(cls, row: Any) -> "_BaseConversation": raise NotImplementedError + def _record_response(self, response: "_BaseResponse") -> None: + "Record a completed response as part of this conversation." + self.responses.append(response) + # History now comes from the live responses, so anything read + # back from storage is superseded. + self.loaded_messages = None + + def _build_full_chain( + self, + prompt: str | None, + attachments, + tool_results, + explicit_messages, + system=None, + system_fragments=None, + fragments=None, + ) -> list[Any]: + """Build the full message chain for the next turn. + + Uses the last response's stored prompt chain to recover prior + history, then appends the new turn's content (explicit messages + first, or synthesized from prompt/attachments/tool_results). + + Returns the list that should be passed as ``messages=`` to the + Prompt constructor so that ``response.prompt.messages`` equals + exactly what the model sees. + + If ``explicit_messages`` is provided, the caller has opted out + of history reconstruction: the list is the authoritative history, + and the new turn's input is appended to it. + """ + from .parts import Message, TextPart + + if explicit_messages is not None: + return _append_turn_input( + list(explicit_messages), + prompt, + fragments, + attachments, + tool_results, + ) + + chain: list[Any] = [] + if self.loaded_messages: + # Storage holds the exact chain, so prefer it over anything + # rebuilt from logged responses. Cleared as soon as this + # conversation produces a response of its own. + chain.extend(self.loaded_messages) + elif self.responses: + last = self.responses[-1] + # last.prompt.messages already contains the full input chain + # under the invariant, so use the last response only and then + # append that response's structured output. + chain.extend(last.prompt.messages) + chain.extend(last._messages_now()) + else: + # Start with the system prompt as the first message so adapters + # that build from prompt.messages see it. On later turns it + # is already carried forward in last.prompt.messages. + system_text = _combine_system(system, system_fragments) + if system_text: + chain.append(Message(role="system", parts=[TextPart(text=system_text)])) + + return _append_turn_input(chain, prompt, fragments, attachments, tool_results) + @dataclass class Conversation(_BaseConversation): - before_call: Optional[BeforeCallSync] = None - after_call: Optional[AfterCallSync] = None + before_call: BeforeCallSync | None = None + after_call: AfterCallSync | None = None def prompt( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[Union[str, Fragment]]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - system_fragments: Optional[List[Union[str, Fragment]]] = None, + fragments: list[str | Fragment] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + system_fragments: list[str | Fragment] | None = None, + messages: list[Any] | None = None, stream: bool = True, - key: Optional[str] = None, - **options, + key: str | None = None, + options: dict | None = None, + hide_reasoning: bool = False, + **kwargs, ) -> "Response": + merged = _merge_options(options, kwargs) + # Build the authoritative chain so response.prompt.messages + # equals exactly what the model sees for this turn. + chain = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + system=system, + system_fragments=system_fragments, + fragments=fragments, + ) return Response( Prompt( prompt, @@ -440,33 +860,52 @@ def prompt( tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, - options=self.model.Options(**options), + messages=chain, + options=self.model.Options(**merged), + hide_reasoning=hide_reasoning, ), self.model, stream, conversation=self, key=key, + before_call=self.before_call, + after_call=self.after_call, ) def chain( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, + fragments: list[str] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + system_fragments: list[str] | None = None, + messages: list[Any] | None = None, stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - chain_limit: Optional[int] = None, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + chain_limit: int | None = None, + before_call: BeforeCallSync | None = None, + after_call: AfterCallSync | None = None, + key: str | None = None, + options: dict | None = None, + hide_reasoning: bool = False, ) -> "ChainResponse": self.model._validate_attachments(attachments) + # Parity with Conversation.prompt: pre-bake the full chain so + # response.prompt.messages is authoritative for the first turn + # of the chain loop. Subsequent tool-result turns extend the + # chain via _chain_for_tool_results. + chain_messages = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + system=system, + system_fragments=system_fragments, + fragments=fragments, + ) return ChainResponse( Prompt( prompt, @@ -477,8 +916,10 @@ def chain( tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, + messages=chain_messages, model=self.model, options=self.model.Options(**(options or {})), + hide_reasoning=hide_reasoning, ), model=self.model, stream=stream, @@ -507,28 +948,39 @@ def __repr__(self): @dataclass class AsyncConversation(_BaseConversation): - before_call: Optional[BeforeCallAsync] = None - after_call: Optional[AfterCallAsync] = None + before_call: BeforeCallAsync | None = None + after_call: AfterCallAsync | None = None def chain( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, + fragments: list[str] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + system_fragments: list[str] | None = None, + messages: list[Any] | None = None, stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - chain_limit: Optional[int] = None, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + chain_limit: int | None = None, + before_call: BeforeCallAsync | None = None, + after_call: AfterCallAsync | None = None, + key: str | None = None, + options: dict | None = None, + hide_reasoning: bool = False, ) -> "AsyncChainResponse": self.model._validate_attachments(attachments) + chain_messages = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + system=system, + system_fragments=system_fragments, + fragments=fragments, + ) return AsyncChainResponse( Prompt( prompt, @@ -539,8 +991,10 @@ def chain( tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, + messages=chain_messages, model=self.model, options=self.model.Options(**(options or {})), + hide_reasoning=hide_reasoning, ), model=self.model, stream=stream, @@ -553,19 +1007,32 @@ def chain( def prompt( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - system_fragments: Optional[List[str]] = None, + fragments: list[str] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + system_fragments: list[str] | None = None, + messages: list[Any] | None = None, stream: bool = True, - key: Optional[str] = None, - **options, + key: str | None = None, + options: dict | None = None, + hide_reasoning: bool = False, + **kwargs, ) -> "AsyncResponse": + merged = _merge_options(options, kwargs) + chain = self._build_full_chain( + prompt=prompt, + attachments=attachments, + tool_results=tool_results, + explicit_messages=messages, + system=system, + system_fragments=system_fragments, + fragments=fragments, + ) return AsyncResponse( Prompt( prompt, @@ -577,12 +1044,16 @@ def prompt( tools=tools, tool_results=tool_results, system_fragments=system_fragments, - options=self.model.Options(**options), + messages=chain, + options=self.model.Options(**merged), + hide_reasoning=hide_reasoning, ), self.model, stream, conversation=self, key=key, + before_call=self.before_call, + after_call=self.after_call, ) def to_sync_conversation(self): @@ -637,18 +1108,19 @@ class _BaseResponse: id: str prompt: "Prompt" stream: bool - resolved_model: Optional[str] = None + resolved_model: str | None = None conversation: Optional["_BaseConversation"] = None - _key: Optional[str] = None - _tool_calls: List[ToolCall] = [] + _key: str | None = None def __init__( self, prompt: Prompt, model: "_BaseModel", stream: bool, - conversation: Optional[_BaseConversation] = None, - key: Optional[str] = None, + conversation: _BaseConversation | None = None, + key: str | None = None, + before_call: BeforeCallSync | BeforeCallAsync | None = None, + after_call: AfterCallSync | AfterCallAsync | None = None, ): self.id = str(monotonic_ulid()).lower() self.prompt = prompt @@ -656,35 +1128,393 @@ def __init__( self.model = model self.stream = stream self._key = key - self._chunks: List[str] = [] + self.before_call = before_call + self.after_call = after_call + self._chunks: list[str] = [] + # Every StreamEvent ever yielded by execute(), in order. Plain + # str yields are wrapped as text events (with part_index resolved + # by _resolve_part_index) so this buffer is the single source of + # truth for replay and for assembling response.messages. + self._stream_events: list[Any] = [] + # Auto-allocator state for resolving StreamEvent.part_index=None. + # Plugins yield events with part_index=None (the default) and + # the framework assigns concrete integers based on context: + # consecutive same-family text/reasoning events concatenate, + # tool calls group by tool_call_id, and tool_result is always + # its own part. _auto_index_max tracks the highest index seen + # (explicit or allocated); _auto_last_index / _auto_last_family + # remember the previously-resolved event so same-family runs + # share an index; _auto_tool_id_to_index maps known tool ids to + # their assigned index for parallel-tool-call grouping. + self._auto_index_max: int = -1 + self._auto_last_index: int | None = None + self._auto_last_family: str | None = None + self._auto_tool_id_to_index: dict[str, int] = {} + self._auto_last_message_index: int = 0 self._done = False - self._tool_calls: List[ToolCall] = [] - self.response_json: Optional[Dict[str, Any]] = None + self._tool_calls: list[ToolCall] = [] + self.response_json: dict[str, Any] | None = None self.conversation = conversation - self.attachments: List[Attachment] = [] - self._start: Optional[float] = None - self._end: Optional[float] = None - self._start_utcnow: Optional[datetime.datetime] = None - self.input_tokens: Optional[int] = None - self.output_tokens: Optional[int] = None - self.token_details: Optional[dict] = None - self.done_callbacks: List[Callable] = [] + self.attachments: list[Attachment] = [] + self._start: float | None = None + self._end: float | None = None + self._start_utcnow: datetime.datetime | None = None + self.input_tokens: int | None = None + self.output_tokens: int | None = None + self.token_details: dict | None = None + self.done_callbacks: list[Callable] = [] if self.prompt.schema and not self.model.supports_schema: raise ValueError(f"{self.model} does not support schemas") - if self.prompt.tools and not self.model.supports_tools: + function_tools, _ = _partition_tools(self.model, self.prompt.tools) + if function_tools and not self.model.supports_tools: raise ValueError(f"{self.model} does not support tools") + def _messages_now(self) -> list[Any]: + """Assemble messages assuming the response is already drained. + + Public ``messages()`` forces / awaits first, then delegates here. + Internal sync paths (``_response_to_dict``, + ``_chain_for_tool_results``) call this directly so they don't + have to await on async responses. + """ + from .parts import Message + + loaded = getattr(self, "_loaded_messages", None) + if loaded is not None: + return list(loaded) + return [ + Message(role="assistant", parts=parts) + for parts in self._build_message_parts() + ] + + @staticmethod + def _event_family(event_type: str) -> str: + if event_type in ("tool_call_name", "tool_call_args"): + return "tool_call" + return event_type + + def _resolve_part_index(self, event): + """Mutate event.part_index in place when the plugin left it None. + + Resolution rules: consecutive same-family text/reasoning events + share an index; tool-call events are grouped by tool_call_id; + tool_result always allocates a fresh index. Explicit indices + pass through but update the allocator's bookkeeping so future + None resolutions avoid collisions. + """ + fam = self._event_family(event.type) + + # A message_index change always starts a fresh part - text on + # either side of a message boundary must not concatenate. + if event.message_index != self._auto_last_message_index: + self._auto_last_family = None + self._auto_last_index = None + self._auto_last_message_index = event.message_index + + if event.part_index is not None: + self._auto_index_max = max(self._auto_index_max, event.part_index) + if ( + event.type in ("tool_call_name", "tool_call_args") + and event.tool_call_id + ): + self._auto_tool_id_to_index[event.tool_call_id] = event.part_index + self._auto_last_index = event.part_index + self._auto_last_family = fam + return + + if event.type in ("tool_call_name", "tool_call_args"): + if event.tool_call_id: + existing = self._auto_tool_id_to_index.get(event.tool_call_id) + if existing is not None: + event.part_index = existing + self._auto_last_index = existing + self._auto_last_family = "tool_call" + return + self._auto_index_max += 1 + new_idx = self._auto_index_max + self._auto_tool_id_to_index[event.tool_call_id] = new_idx + event.part_index = new_idx + self._auto_last_index = new_idx + self._auto_last_family = "tool_call" + return + # No tool_call_id — providers like Gemini omit the id on + # parallel tool calls. tool_call_args events glue onto the + # most recent tool-call index; a fresh tool_call_name + # always starts a new part (otherwise N parallel tool calls + # collapse into one with concatenated names and args). + if ( + event.type == "tool_call_args" + and self._auto_last_family == "tool_call" + and self._auto_last_index is not None + ): + event.part_index = self._auto_last_index + return + self._auto_index_max += 1 + new_idx = self._auto_index_max + event.part_index = new_idx + self._auto_last_index = new_idx + self._auto_last_family = "tool_call" + return + + if event.type == "tool_result": + self._auto_index_max += 1 + new_idx = self._auto_index_max + event.part_index = new_idx + self._auto_last_index = new_idx + self._auto_last_family = "tool_result" + return + + # text / reasoning: same family as previous → reuse, else new. + if self._auto_last_family == fam and self._auto_last_index is not None: + event.part_index = self._auto_last_index + return + self._auto_index_max += 1 + new_idx = self._auto_index_max + event.part_index = new_idx + self._auto_last_index = new_idx + self._auto_last_family = fam + + def _process_chunk(self, chunk): + """Normalize a chunk from execute() into a StreamEvent and return + the text str (or None) that __iter__ should yield. + + Plain str yields from legacy plugins are wrapped as text events + with an auto-allocated part_index. Side effects: populates + self._stream_events and self._chunks. + """ + from .parts import StreamEvent + + if isinstance(chunk, StreamEvent): + self._resolve_part_index(chunk) + self._stream_events.append(chunk) + if chunk.type == "text": + self._chunks.append(chunk.chunk) + return chunk.chunk + return None + # Legacy plain-str plugin. + event = StreamEvent(type="text", chunk=chunk) + self._resolve_part_index(event) + self._stream_events.append(event) + self._chunks.append(chunk) + return chunk + + def _build_parts(self) -> list[Any]: + """All Parts from the accumulated stream events, flattened + across message boundaries. See ``_build_message_parts``.""" + return [part for parts in self._build_message_parts() for part in parts] + + def _build_message_parts(self) -> list[list[Any]]: + """Assemble Part objects from the accumulated stream events, + grouped into one parts-list per assistant message. + + Most providers emit a single assistant message, so the result + is usually a one-element list. Events carrying an explicit + ``message_index`` (OpenAI Responses server-side tool execution + interleaves multiple ``message`` output items in one response) + split into one parts-list per distinct index, in first-seen + order. + + Events sharing a part_index group into one Part. Mixing + families (text vs tool_call vs reasoning vs tool_result) at the + same index is a plugin bug — raises ValueError instead of + silently dropping content. + + Fallback: when no stream events were recorded (response was + rehydrated from SQLite via ``from_row``), synthesize a + TextPart from ``self._chunks`` plus any ``self._tool_calls`` + restored by the row loader. Reasoning signatures are not + recoverable from SQLite in this fallback — use + ``response.to_dict()`` / ``Response.from_dict()`` for + structure-preserving persistence. + """ + from .parts import ( + ReasoningPart, + TextPart, + ToolCallPart, + ToolResultPart, + ) + + if not self._stream_events: + # Rehydrated-from-SQLite path: assemble from _chunks + + # _tool_calls so response.messages isn't empty after + # from_row, and Conversation.prompt-built chains include + # the assistant turn on follow-up calls. + fallback_parts: list[Any] = [] + text = "".join(self._chunks) + if text: + fallback_parts.append(TextPart(text=text)) + for tc in self._tool_calls: + fallback_parts.append( + ToolCallPart( + name=tc.name, + arguments=tc.arguments or {}, + tool_call_id=tc.tool_call_id, + ) + ) + return [fallback_parts] if fallback_parts else [] + + # Group events by their (resolved) part_index, preserving the + # order in which each index was first seen. Then build one Part + # per group. This handles non-adjacent same-index events (e.g. + # text → tool_call → text where the plugin pinned both text + # bursts to part_index=0) by merging them into one Part. Each + # group belongs to the message of its first event. + groups: dict[int, list[Any]] = {} + order: list[int] = [] + group_message: dict[int, int] = {} + for event in self._stream_events: + pi = event.part_index + if pi not in groups: + groups[pi] = [] + order.append(pi) + group_message[pi] = event.message_index + groups[pi].append(event) + + built: list[tuple[int, Any]] = [] + for pi in order: + evs = groups[pi] + fam_first = self._event_family(evs[0].type) + for e in evs: + if self._event_family(e.type) != fam_first: + raise ValueError( + f"StreamEvent type {e.type!r} is incompatible with " + f"prior type at part_index={pi}. " + "Allocate a new part_index for a different content type." + ) + + pm_merged: dict[str, Any] | None = None + for e in evs: + if e.provider_metadata: + merged = dict(pm_merged) if pm_merged else {} + for k, v in e.provider_metadata.items(): + merged[k] = v + pm_merged = merged + + mi = group_message[pi] + if fam_first == "text": + text = "".join(e.chunk for e in evs) + if text: + built.append((mi, TextPart(text=text, provider_metadata=pm_merged))) + elif fam_first == "reasoning": + text = "".join(e.chunk for e in evs) + redacted = any(e.redacted for e in evs) + if text or redacted or pm_merged: + built.append( + ( + mi, + ReasoningPart( + text=text, + redacted=redacted, + provider_metadata=pm_merged, + ), + ) + ) + elif fam_first == "tool_call": + tool_name = "".join(e.chunk for e in evs if e.type == "tool_call_name") + args_str = "".join(e.chunk for e in evs if e.type == "tool_call_args") + try: + arguments = json.loads(args_str) if args_str else {} + except json.JSONDecodeError: + arguments = {"_raw": args_str} + tool_call_id = next( + (e.tool_call_id for e in evs if e.tool_call_id), None + ) + server_executed = any(e.server_executed for e in evs) + built.append( + ( + mi, + ToolCallPart( + name=tool_name, + arguments=arguments, + tool_call_id=tool_call_id, + server_executed=server_executed, + provider_metadata=pm_merged, + ), + ) + ) + elif fam_first == "tool_result": + tool_result_name = next((e.tool_name for e in evs if e.tool_name), "") + tool_call_id = next( + (e.tool_call_id for e in evs if e.tool_call_id), None + ) + server_executed = any(e.server_executed for e in evs) + built.append( + ( + mi, + ToolResultPart( + name=tool_result_name, + output="".join(e.chunk for e in evs), + tool_call_id=tool_call_id, + server_executed=server_executed, + provider_metadata=pm_merged, + ), + ) + ) + + # Split into per-message parts lists, message indexes in + # first-seen order. + message_order: list[int] = [] + by_message: dict[int, list[Any]] = {} + for mi, part in built: + if mi not in by_message: + by_message[mi] = [] + message_order.append(mi) + by_message[mi].append(part) + messages_parts = [by_message[mi] for mi in message_order] + if not messages_parts: + messages_parts = [[]] + + # Merge in any tool calls registered via add_tool_call() that the + # plugin didn't also emit as StreamEvents. Dedup by tool_call_id so + # plugins using both APIs in tandem don't double-count. They join + # the final message - matching the old append-at-end behavior. + seen_ids = { + p.tool_call_id + for parts in messages_parts + for p in parts + if isinstance(p, ToolCallPart) and p.tool_call_id is not None + } + for tc in self._tool_calls: + if tc.tool_call_id is not None and tc.tool_call_id in seen_ids: + continue + messages_parts[-1].append( + ToolCallPart( + name=tc.name, + arguments=tc.arguments or {}, + tool_call_id=tc.tool_call_id, + ) + ) + + # Hoist redacted reasoning Parts to the start of their message. + # Plugins typically emit them late (when usage arrives in the + # final chunk), but UIs render reasoning before content, so the + # framework reorders. Relative order among redacted Parts is + # preserved. + for i, parts in enumerate(messages_parts): + redacted_parts = [ + p for p in parts if isinstance(p, ReasoningPart) and p.redacted + ] + if redacted_parts: + other_parts = [ + p + for p in parts + if not (isinstance(p, ReasoningPart) and p.redacted) + ] + messages_parts[i] = redacted_parts + other_parts + + return [parts for parts in messages_parts if parts] + def add_tool_call(self, tool_call: ToolCall): - self._tool_calls.append(tool_call) + self._tool_calls.append(_ensure_tool_call_id(tool_call)) def set_usage( self, *, - input: Optional[int] = None, - output: Optional[int] = None, - details: Optional[dict] = None, + input: int | None = None, + output: int | None = None, + details: dict | None = None, ): self.input_tokens = input self.output_tokens = output @@ -695,7 +1525,7 @@ def set_resolved_model(self, model_id: str): @classmethod def from_row(cls, db, row, _async=False): - from llm import get_model, get_async_model + from llm import get_async_model, get_model if _async: model = get_async_model(row["model"]) @@ -809,202 +1639,216 @@ def token_usage(self) -> str: ) def log_to_db(self, db): - conversation = self.conversation - if not conversation: - conversation = Conversation(model=self.model) - db["conversations"].insert( - { - "id": conversation.id, - "name": _conversation_name( - self.prompt.prompt or self.prompt.system or "" - ), - "model": conversation.model.model_id, - }, - ignore=True, + # Everything - thread, turn, messages, parts, fragments, + # attachments, tools - is recorded in the content-addressed + # tables. The legacy tables are no longer written; they hold + # history logged by older versions and are still read. + # This lives here rather than in the CLI because log_to_db() + # is what plugins call. + from .logs import LogStore + + LogStore(db).log(self) + + +def _response_to_dict(response: "_BaseResponse") -> ResponseDict: + """Shared serializer for Response.to_dict / AsyncResponse.to_dict. + + The output is a JSON-safe dict — store it anywhere (file, Redis, + Postgres, HTTP body) and round-trip via Response.from_dict or + AsyncResponse.from_dict. + """ + options = { + key: value + for key, value in dict(response.prompt.options).items() + if value is not None + } + payload: dict[str, Any] = { + "model": response.model.model_id, + "prompt": { + "messages": [m.to_dict() for m in response.prompt.messages], + }, + "messages": [m.to_dict() for m in response._messages_now()], + } + if options: + payload["prompt"]["options"] = options + if response.prompt._system: + payload["prompt"]["system"] = response.prompt._system + # Optional audit fields — helpful for debugging, not needed for reply(). + if response.id: + payload["id"] = response.id + if response._done: + if response.input_tokens is not None or response.output_tokens is not None: + usage: dict[str, Any] = {} + if response.input_tokens is not None: + usage["input"] = response.input_tokens + if response.output_tokens is not None: + usage["output"] = response.output_tokens + if response.token_details is not None: + usage["details"] = response.token_details + payload["usage"] = usage + if response._start_utcnow is not None: + payload["datetime_utc"] = response._start_utcnow.isoformat() + return cast(ResponseDict, payload) + + +def _response_from_dict( + data: ResponseDict, + cls, + *, + model=None, + async_: bool = False, +) -> "_BaseResponse": + """Shared deserializer for Response.from_dict / AsyncResponse.from_dict.""" + from .parts import Message + + if model is None: + from llm import get_async_model, get_model + + getter = get_async_model if async_ else get_model + model = getter(data["model"]) + + prompt_data = data.get("prompt", {}) + input_messages = [Message.from_dict(m) for m in prompt_data.get("messages", [])] + output_messages = [Message.from_dict(m) for m in data.get("messages", [])] + + options_kwargs = prompt_data.get("options") or {} + system = prompt_data.get("system") + + prompt = Prompt( + None, + model=model, + messages=input_messages, + system=system, + options=model.Options(**options_kwargs), + ) + response = cls(prompt, model=model, stream=False) + # Preserve id for audit continuity. + if "id" in data: + response.id = data["id"] + # Rebuild chunks from the assistant's text parts so response.text() + # works without re-running the assembler. + from .parts import TextPart + + response._chunks = [ + p.text + for m in output_messages + for p in m.parts + if isinstance(p, TextPart) and p.text + ] + # Stash the structured output so response.messages returns the + # full picture (reasoning, tool calls, signatures) without needing + # a StreamEvent replay. + response._loaded_messages = output_messages + # Rebuild _tool_calls from the restored parts so tool_calls() and + # reply(tools=...) can execute serialized pending calls. Server- + # executed calls stay out, matching add_tool_call() during live + # streaming. + from .parts import ToolCallPart + + response._tool_calls = [ + ToolCall( + name=p.name, + arguments=p.arguments or {}, + tool_call_id=p.tool_call_id, ) - schema_id = None - if self.prompt.schema: - schema_id, schema_json = make_schema_id(self.prompt.schema) - db["schemas"].insert({"id": schema_id, "content": schema_json}, ignore=True) - - response_id = self.id - replacements = {} - # Include replacements from previous responses - for previous_response in conversation.responses[:-1]: - for fragment in (previous_response.prompt.fragments or []) + ( - previous_response.prompt.system_fragments or [] - ): - fragment_id = ensure_fragment(db, fragment) - replacements[f"f:{fragment_id}"] = fragment - replacements[f"r:{previous_response.id}"] = ( - previous_response.text_or_raise() - ) - - for i, fragment in enumerate(self.prompt.fragments): - fragment_id = ensure_fragment(db, fragment) - replacements[f"f{fragment_id}"] = fragment - db["prompt_fragments"].insert( - { - "response_id": response_id, - "fragment_id": fragment_id, - "order": i, - }, - ) - for i, fragment in enumerate(self.prompt.system_fragments): - fragment_id = ensure_fragment(db, fragment) - replacements[f"f{fragment_id}"] = fragment - db["system_fragments"].insert( - { - "response_id": response_id, - "fragment_id": fragment_id, - "order": i, - }, - ) - - response_text = self.text_or_raise() - replacements[f"r:{response_id}"] = response_text - json_data = self.json() - - response = { - "id": response_id, - "model": self.model.model_id, - "prompt": self.prompt._prompt, - "system": self.prompt._system, - "prompt_json": condense_json(self._prompt_json, replacements), - "options_json": { - key: value - for key, value in dict(self.prompt.options).items() - if value is not None - }, - "response": response_text, - "response_json": condense_json(json_data, replacements), - "conversation_id": conversation.id, - "duration_ms": self.duration_ms(), - "datetime_utc": self.datetime_utc(), - "input_tokens": self.input_tokens, - "output_tokens": self.output_tokens, - "token_details": ( - json.dumps(self.token_details) if self.token_details else None - ), - "schema_id": schema_id, - "resolved_model": self.resolved_model, - } - db["responses"].insert(response) - - # Persist any attachments - loop through with index - for index, attachment in enumerate(self.prompt.attachments): - attachment_id = attachment.id() - db["attachments"].insert( - { - "id": attachment_id, - "type": attachment.resolve_type(), - "path": attachment.path, - "url": attachment.url, - "content": attachment.content, - }, - replace=True, - ) - db["prompt_attachments"].insert( - { - "response_id": response_id, - "attachment_id": attachment_id, - "order": index, - }, - ) - - # Persist any tools, tool calls and tool results - tool_ids_by_name = {} - for tool in self.prompt.tools: - tool_id = ensure_tool(db, tool) - tool_ids_by_name[tool.name] = tool_id - db["tool_responses"].insert( - { - "tool_id": tool_id, - "response_id": response_id, - } - ) - for tool_call in self.tool_calls(): # TODO Should be _or_raise() - db["tool_calls"].insert( - { - "response_id": response_id, - "tool_id": tool_ids_by_name.get(tool_call.name) or None, - "name": tool_call.name, - "arguments": json.dumps(tool_call.arguments), - "tool_call_id": tool_call.tool_call_id, - } - ) - for tool_result in self.prompt.tool_results: - instance_id = None - if tool_result.instance: - try: - if not tool_result.instance.instance_id: - tool_result.instance.instance_id = ( - db["tool_instances"] - .insert( - { - "plugin": tool.plugin, - "name": tool.name.split("_")[0], - "arguments": json.dumps( - tool_result.instance._config - ), - } - ) - .last_pk - ) - instance_id = tool_result.instance.instance_id - except AttributeError: - pass - tool_result_id = ( - db["tool_results"] - .insert( - { - "response_id": response_id, - "tool_id": tool_ids_by_name.get(tool_result.name) or None, - "name": tool_result.name, - "output": tool_result.output, - "tool_call_id": tool_result.tool_call_id, - "instance_id": instance_id, - "exception": ( - ( - "{}: {}".format( - tool_result.exception.__class__.__name__, - str(tool_result.exception), - ) - ) - if tool_result.exception - else None - ), - } - ) - .last_pk - ) - # Persist attachments for tool results - for index, attachment in enumerate(tool_result.attachments): - attachment_id = attachment.id() - db["attachments"].insert( - { - "id": attachment_id, - "type": attachment.resolve_type(), - "path": attachment.path, - "url": attachment.url, - "content": attachment.content, - }, - replace=True, - ) - db["tool_results_attachments"].insert( - { - "tool_result_id": tool_result_id, - "attachment_id": attachment_id, - "order": index, - }, - ) + for m in output_messages + for p in m.parts + if isinstance(p, ToolCallPart) and not p.server_executed + ] + response._done = True + # Restore usage if present. + usage = data.get("usage") + if usage: + response.input_tokens = usage.get("input") + response.output_tokens = usage.get("output") + response.token_details = usage.get("details") + return response class Response(_BaseResponse): + "Sync response from a model." + model: "Model" conversation: Optional["Conversation"] = None + def reply( + self, + prompt: str | None = None, + *, + messages: list[Any] | None = None, + tool_results: list[ToolResult] | None = None, + options: dict | None = None, + **kwargs, + ) -> "Response": + """Continue the conversation from this response. + + Builds the next turn's chain as + ``self.prompt.messages + self.messages + [tool_message] + + [user(prompt)] + messages`` and calls + ``self.model.prompt(messages=chain, ...)``. + + If this response made tool calls and ``tool_results=`` is not + passed, ``reply()`` runs ``self.execute_tool_calls()`` + automatically and threads the results into the chain. Pass an + explicit ``tool_results=`` list (e.g. results you mutated, or + synthetic ones for testing) to skip auto-execution. + """ + from .parts import Message, TextPart + + self._force() + # Forward original tools so the next turn can call them again + # (mirrors Conversation.prompt's `tools or self.tools` rule). + if "tools" not in kwargs and self.prompt.tools: + kwargs["tools"] = self.prompt.tools + if tool_results is None and self._tool_calls: + tool_results = self.execute_tool_calls(tools=kwargs.get("tools")) + chain: list[Any] = list(self.prompt.messages) + list(self._messages_now()) + if tool_results: + tool_attachments: list[Attachment] = [] + for tr in tool_results: + tool_attachments.extend(tr.attachments or []) + _append_tool_results_to_chain(chain, tool_results, tool_attachments) + if prompt: + chain.append(Message(role="user", parts=[TextPart(text=prompt)])) + if messages: + chain.extend(messages) + return self.model.prompt(messages=chain, options=options, **kwargs) + + def to_dict(self) -> ResponseDict: + """Serialize this response for JSON persistence. + + Captures exactly what is needed to continue the conversation: + model id, the input chain that was sent + (``response.prompt.messages``), the structured assistant output + (``response.messages``), and any explicit options. Pair with + :meth:`Response.from_dict` to rehydrate and + :meth:`Response.reply` to continue. + + Returns :class:`~llm.serialization.ResponseDict`. + """ + self._force() + return _response_to_dict(self) + + @classmethod + def from_dict( + cls, + data: ResponseDict, + *, + model: Optional["Model"] = None, + ) -> "Response": + """Rehydrate a Response from a ``to_dict()`` payload. + + The returned Response is in the ``_done`` state with + ``response.text()`` and ``response.messages`` populated. + ``model`` overrides the stored model id (useful for continuing + on a different model). + """ + return cast( + "Response", _response_from_dict(data, cls, model=model, async_=False) + ) + def on_done(self, callback): + "Register a callback to be called when the response is complete." if not self._done: self.done_callbacks.append(callback) else: @@ -1022,6 +1866,7 @@ def _force(self): list(self) def text(self) -> str: + "Return the full text of the response, executing the prompt if needed." self._force() return "".join(self._chunks) @@ -1031,11 +1876,27 @@ def text_or_raise(self) -> str: def execute_tool_calls( self, *, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - ) -> List[ToolResult]: + before_call: BeforeCallSync | None = None, + after_call: AfterCallSync | None = None, + tool_calls_list: list[ToolCall] | None = None, + tools: list[ToolDef] | None = None, + ) -> list[ToolResult]: + """Execute tool calls using this response's tools. + + By default executes ``self.tool_calls()``; pass + ``tool_calls_list=`` to execute an explicit list instead (used + when resuming a chain whose history ends in unresolved calls). + Pass ``tools=`` to resolve implementations from an explicit + list instead of ``self.prompt.tools`` (used when a rehydrated + response has pending calls but no tool implementations). + """ tool_results = [] - tools_by_name = {tool.name: tool for tool in self.prompt.tools} + effective_tools = _wrap_tools(tools) if tools is not None else self.prompt.tools + tools_by_name = { + tool.name: tool for tool in effective_tools if isinstance(tool, Tool) + } + if tool_calls_list is None: + tool_calls_list = self.tool_calls() # Run prepare() on all Toolbox instances that need it instances_to_prepare: list[Toolbox] = [] @@ -1048,8 +1909,8 @@ def execute_tool_calls( inst.prepare() inst._prepared = True - for tool_call in self.tool_calls(): - tool: Optional[Tool] = tools_by_name.get(tool_call.name) + for tool_call in tool_calls_list: + tool: Tool | None = tools_by_name.get(tool_call.name) # Tool could be None if the tool was not found in the prompt tools, # but we still call the before_call method: if before_call: @@ -1072,7 +1933,7 @@ def execute_tool_calls( continue if tool is None: - msg = 'tool "{}" does not exist'.format(tool_call.name) + msg = f'tool "{tool_call.name}" does not exist' tool_results.append( ToolResult( name=tool_call.name, @@ -1085,17 +1946,20 @@ def execute_tool_calls( if not tool.implementation: raise ValueError( - "No implementation available for tool: {}".format(tool_call.name) + f"No implementation available for tool: {tool_call.name}" ) attachments = [] exception = None try: + implementation_arguments = _implementation_arguments(tool, tool_call) if inspect.iscoroutinefunction(tool.implementation): - result = asyncio.run(tool.implementation(**tool_call.arguments)) + result = asyncio.run( + tool.implementation(**implementation_arguments) + ) else: - result = tool.implementation(**tool_call.arguments) + result = tool.implementation(**implementation_arguments) if isinstance(result, ToolOutput): attachments = result.attachments @@ -1103,7 +1967,14 @@ def execute_tool_calls( if not isinstance(result, str): result = json.dumps(result, default=repr) - except Exception as ex: + except PauseChain as ex: + # Pause: propagate instead of converting to an error + # result. Sequential execution stops here - later calls + # never started, so they can safely run on resume. + ex.tool_call = tool_call + ex.tool_results = list(tool_results) + raise + except Exception as ex: # noqa: BLE001 result = f"Error: {ex}" exception = ex @@ -1126,14 +1997,25 @@ def execute_tool_calls( tool_results.append(tool_result_obj) return tool_results - def tool_calls(self) -> List[ToolCall]: + def execute_tool_call(self, tool_call: ToolCall) -> ToolResult: + "Utility method for manually executing a tool call with callbacks" + tool_call = _ensure_tool_call_id(tool_call) + return self.execute_tool_calls( + before_call=cast(BeforeCallSync | None, self.before_call), + after_call=cast(AfterCallSync | None, self.after_call), + tool_calls_list=[tool_call], + )[0] + + def tool_calls(self) -> list[ToolCall]: + "Return the list of tool calls made during this response." self._force() return self._tool_calls - def tool_calls_or_raise(self) -> List[ToolCall]: + def tool_calls_or_raise(self) -> list[ToolCall]: return self.tool_calls() - def json(self) -> Optional[Dict[str, Any]]: + def json(self) -> dict[str, Any] | None: + "Return the raw JSON response from the model, if available." self._force() return self.response_json @@ -1146,6 +2028,7 @@ def datetime_utc(self) -> str: return self._start_utcnow.isoformat() if self._start_utcnow else "" def usage(self) -> Usage: + "Return token usage information for this response." self._force() return Usage( input=self.input_tokens, @@ -1153,59 +2036,170 @@ def usage(self) -> Usage: details=self.token_details, ) - def __iter__(self) -> Iterator[str]: - self._start = time.monotonic() - self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) - if self._done: - yield from self._chunks - return - + def _iter_events(self): + """Drive self.model.execute() once and yield each raw chunk it + produces. Callers normalize chunks through _process_chunk. + """ if isinstance(self.model, Model): - for chunk in self.model.execute( + generator = self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, - ): - assert chunk is not None - yield chunk - self._chunks.append(chunk) + ) elif isinstance(self.model, KeyModel): - for chunk in self.model.execute( + generator = self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, key=self.model.get_key(self._key), - ): - assert chunk is not None - yield chunk - self._chunks.append(chunk) + ) else: - raise Exception("self.model must be a Model or KeyModel") + raise TypeError("self.model must be a Model or KeyModel") + + for chunk in generator: + assert chunk is not None + yield chunk + + def __iter__(self) -> Iterator[str]: + self._start = time.monotonic() + self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) + if self._done: + yield from self._chunks + return + + for chunk in self._iter_events(): + text = self._process_chunk(chunk) + if text is not None: + yield text if self.conversation: - self.conversation.responses.append(self) + self.conversation._record_response(self) self._end = time.monotonic() self._done = True self._on_done() + def stream_events(self): + """Yield StreamEvent objects as the model produces them. + + Whichever of __iter__ and stream_events runs first during live + streaming consumes the underlying generator. After completion, + both work — each replays from its own buffer. + """ + if self._done: + yield from self._stream_events + return + + self._start = time.monotonic() + self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) + for chunk in self._iter_events(): + # _process_chunk appends to self._stream_events; use it as + # the canonical source for what to yield so the replay path + # matches the live path byte-for-byte. + self._process_chunk(chunk) + yield self._stream_events[-1] + + if self.conversation: + self.conversation._record_response(self) + self._end = time.monotonic() + self._done = True + self._on_done() + + def messages(self) -> list[Any]: + """List of Message objects produced by this response. + + Almost always a single assistant Message; multiple messages are + possible for providers that emit multi-message responses during + server-side tool execution. + + Forces execution if the response has not yet been drained, so + ``response.messages()`` is safe to call without a prior + ``response.text()`` / iteration. + + Responses rehydrated via ``Response.from_dict`` short-circuit + and return the stored messages directly. + """ + self._force() + return self._messages_now() + def __repr__(self): text = "... not yet done ..." if self._done: text = "".join(self._chunks) - return "".format(self.prompt.prompt, text) + return f"" class AsyncResponse(_BaseResponse): + "Async response from a model." + model: "AsyncModel" conversation: Optional["AsyncConversation"] = None + async def reply( + self, + prompt: str | None = None, + *, + messages: list[Any] | None = None, + tool_results: list[ToolResult] | None = None, + options: dict | None = None, + **kwargs, + ) -> "AsyncResponse": + """Async counterpart of Response.reply(). Requires this response + to have been awaited (so self.messages is available). + + Awaitable so the auto-execute path can ``await + self.execute_tool_calls()``. See ``Response.reply`` for the + ``tool_results=`` semantics. + """ + from .parts import Message, TextPart + + if not self._done: + raise ValueError( + "Response not yet awaited — call `await response` before reply()" + ) + if "tools" not in kwargs and self.prompt.tools: + kwargs["tools"] = self.prompt.tools + if tool_results is None and self._tool_calls: + tool_results = await self.execute_tool_calls(tools=kwargs.get("tools")) + chain: list[Any] = list(self.prompt.messages) + list(self._messages_now()) + if tool_results: + tool_attachments: list[Attachment] = [] + for tr in tool_results: + tool_attachments.extend(tr.attachments or []) + _append_tool_results_to_chain(chain, tool_results, tool_attachments) + if prompt: + chain.append(Message(role="user", parts=[TextPart(text=prompt)])) + if messages: + chain.extend(messages) + return self.model.prompt(messages=chain, options=options, **kwargs) + + def to_dict(self) -> ResponseDict: + """Async counterpart of Response.to_dict(). Requires awaiting.""" + if not self._done: + raise ValueError( + "Response not yet awaited — call `await response` before to_dict()" + ) + return _response_to_dict(self) + + @classmethod + def from_dict( + cls, + data: ResponseDict, + *, + model: Optional["AsyncModel"] = None, + ) -> "AsyncResponse": + """Async counterpart of Response.from_dict().""" + return cast( + "AsyncResponse", _response_from_dict(data, cls, model=model, async_=True) + ) + @classmethod def from_row(cls, db, row, _async=False): return super().from_row(db, row, _async=True) async def on_done(self, callback): + "Register a callback to be called when the response is complete." if not self._done: self.done_callbacks.append(callback) else: @@ -1229,11 +2223,26 @@ async def _on_done(self): async def execute_tool_calls( self, *, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - ) -> List[ToolResult]: - tool_calls_list = await self.tool_calls() - tools_by_name = {tool.name: tool for tool in self.prompt.tools} + before_call: BeforeCallAsync | None = None, + after_call: AfterCallAsync | None = None, + tool_calls_list: list[ToolCall] | None = None, + tools: list[ToolDef] | None = None, + ) -> list[ToolResult]: + """Execute tool calls using this response's tools. + + By default executes ``await self.tool_calls()``; pass + ``tool_calls_list=`` to execute an explicit list instead (used + when resuming a chain whose history ends in unresolved calls). + Pass ``tools=`` to resolve implementations from an explicit + list instead of ``self.prompt.tools`` (used when a rehydrated + response has pending calls but no tool implementations). + """ + if tool_calls_list is None: + tool_calls_list = await self.tool_calls() + effective_tools = _wrap_tools(tools) if tools is not None else self.prompt.tools + tools_by_name = { + tool.name: tool for tool in effective_tools if isinstance(tool, Tool) + } # Run async prepare_async() on all Toolbox instances that need it instances_to_prepare: list[Toolbox] = [] @@ -1248,20 +2257,62 @@ async def execute_tool_calls( await inst.prepare_async() inst._async_prepared = True - indexed_results: List[tuple[int, ToolResult]] = [] - async_tasks: List[asyncio.Task] = [] + indexed_results: list[tuple[int, ToolResult]] = [] + async_tasks: list[asyncio.Task] = [] + async_task_indexes: list[int] = [] + # Defined failure semantics: a pause or error in one call must not + # orphan concurrently-running siblings. Pauses and hook failures + # are collected here and raised only after every task that was + # started has finished. + paused: list[tuple[int, PauseChain]] = [] + failures: list[tuple[int, BaseException]] = [] for idx, tc in enumerate(tool_calls_list): - tool: Optional[Tool] = tools_by_name.get(tc.name) - exception: Optional[Exception] = None + tool: Tool | None = tools_by_name.get(tc.name) + exception: Exception | None = None + + if tool is None or not tool.implementation: + # Mirror the sync executor: append an error ToolResult so + # the provider still receives a result for every tool + # call. before_call fires even though the tool is + # unavailable. + if before_call: + try: + cb = before_call(tool, tc) + if inspect.isawaitable(cb): + await cb + except CancelToolCall as ex: + indexed_results.append( + ( + idx, + ToolResult( + name=tc.name, + output="Cancelled: " + str(ex), + tool_call_id=tc.tool_call_id, + exception=ex, + ), + ) + ) + continue + except Exception as ex: # noqa: BLE001 + failures.append((idx, ex)) + break + reason = "does not exist" if tool is None else "has no implementation" + msg = f'tool "{tc.name}" {reason}' + indexed_results.append( + ( + idx, + ToolResult( + name=tc.name, + output="Error: " + msg, + tool_call_id=tc.tool_call_id, + exception=KeyError(msg), + ), + ) + ) + continue - if tool is None: - output = f'Error: tool "{tc.name}" does not exist' - exception = KeyError(tc.name) - elif not tool.implementation: - output = f'Error: tool "{tc.name}" has no implementation' - exception = KeyError(tc.name) - elif inspect.iscoroutinefunction(tool.implementation): + if inspect.iscoroutinefunction(tool.implementation): async def run_async(tc=tc, tool=tool, idx=idx): # before_call inside the task @@ -1282,7 +2333,9 @@ async def run_async(tc=tc, tool=tool, idx=idx): attachments = [] try: - result = await tool.implementation(**tc.arguments) + result = await tool.implementation( + **_implementation_arguments(tool, tc) + ) if isinstance(result, ToolOutput): attachments.extend(result.attachments) result = result.output @@ -1291,7 +2344,12 @@ async def run_async(tc=tc, tool=tool, idx=idx): if isinstance(result, str) else json.dumps(result, default=repr) ) - except Exception as ex: + except PauseChain as ex: + # Propagates out of the task; collected after + # the gather so siblings finish first. + ex.tool_call = tc + raise + except Exception as ex: # noqa: BLE001 output = f"Error: {ex}" exception = ex @@ -1313,6 +2371,7 @@ async def run_async(tc=tc, tool=tool, idx=idx): return idx, tr async_tasks.append(asyncio.create_task(run_async())) + async_task_indexes.append(idx) else: # Sync implementation: do hooks and call inline @@ -1334,53 +2393,93 @@ async def run_async(tc=tc, tool=tool, idx=idx): ) ) continue + except Exception as ex: # noqa: BLE001 + failures.append((idx, ex)) + break exception = None attachments = [] - if tool is None: - output = f'Error: tool "{tc.name}" does not exist' - exception = KeyError(tc.name) - else: - try: - res = tool.implementation(**tc.arguments) - if inspect.isawaitable(res): - res = await res - if isinstance(res, ToolOutput): - attachments.extend(res.attachments) - res = res.output - output = ( - res - if isinstance(res, str) - else json.dumps(res, default=repr) - ) - except Exception as ex: - output = f"Error: {ex}" - exception = ex - - tr = ToolResult( - name=tc.name, - output=output, - attachments=attachments, - tool_call_id=tc.tool_call_id, - instance=_get_instance(tool.implementation), - exception=exception, + try: + res = tool.implementation(**_implementation_arguments(tool, tc)) + if inspect.isawaitable(res): + res = await res + if isinstance(res, ToolOutput): + attachments.extend(res.attachments) + res = res.output + output = ( + res if isinstance(res, str) else json.dumps(res, default=repr) ) + except PauseChain as ex: + # Inline execution stops here; later calls never + # start. Tasks already started are still awaited + # below before the pause propagates. + ex.tool_call = tc + paused.append((idx, ex)) + break + except Exception as ex: # noqa: BLE001 + output = f"Error: {ex}" + exception = ex + + tr = ToolResult( + name=tc.name, + output=output, + attachments=attachments, + tool_call_id=tc.tool_call_id, + instance=_get_instance(tool.implementation), + exception=exception, + ) - if tool is not None and after_call: + try: + if after_call: cb2 = after_call(tool, tc, tr) if inspect.isawaitable(cb2): await cb2 + except Exception as ex: # noqa: BLE001 + failures.append((idx, ex)) + break - indexed_results.append((idx, tr)) + indexed_results.append((idx, tr)) - # Await all async tasks in parallel + # Await every task that was started; return_exceptions so a pause + # or hook failure in one task cannot orphan its siblings mid-flight. if async_tasks: - indexed_results.extend(await asyncio.gather(*async_tasks)) + outcomes = await asyncio.gather(*async_tasks, return_exceptions=True) + for task_idx, outcome in zip(async_task_indexes, outcomes): + if isinstance(outcome, PauseChain): + paused.append((task_idx, outcome)) + elif isinstance(outcome, BaseException): + failures.append((task_idx, outcome)) + else: + indexed_results.append(outcome) # Reorder by original index indexed_results.sort(key=lambda x: x[0]) - return [tr for _, tr in indexed_results] + results = [tr for _, tr in indexed_results] + + # Hook failures are bugs: raise the first by call order. + if failures: + failures.sort(key=lambda item: item[0]) + raise failures[0][1] + + # Pauses propagate with the completed sibling results attached. + if paused: + paused.sort(key=lambda item: item[0]) + pause = paused[0][1] + pause.tool_results = results + raise pause + + return results + + async def execute_tool_call(self, tool_call: ToolCall) -> ToolResult: + "Asynchronous counterpart to :meth:`Response.execute_tool_call`." + tool_call = _ensure_tool_call_id(tool_call) + results = await self.execute_tool_calls( + before_call=cast(BeforeCallAsync | None, self.before_call), + after_call=cast(AfterCallAsync | None, self.after_call), + tool_calls_list=[tool_call], + ) + return results[0] def __aiter__(self): self._start = time.monotonic() @@ -1389,12 +2488,7 @@ def __aiter__(self): self._iter_chunks = list(self._chunks) # Make a copy for iteration return self - async def __anext__(self) -> str: - if self._done: - if hasattr(self, "_iter_chunks") and self._iter_chunks: - return self._iter_chunks.pop(0) - raise StopAsyncIteration - + def _ensure_async_generator(self): if not hasattr(self, "_generator"): if isinstance(self.model, AsyncModel): self._generator = self.model.execute( @@ -1414,20 +2508,69 @@ async def __anext__(self) -> str: else: raise ValueError("self.model must be an AsyncModel or AsyncKeyModel") - try: - chunk = await self._generator.__anext__() + async def _async_finalize(self): + if self.conversation: + self.conversation._record_response(self) + self._end = time.monotonic() + self._done = True + if hasattr(self, "_generator"): + del self._generator + await self._on_done() + + async def __anext__(self) -> str: + if self._done: + if hasattr(self, "_iter_chunks") and self._iter_chunks: + return self._iter_chunks.pop(0) + raise StopAsyncIteration + + self._ensure_async_generator() + # Skip non-text events — iteration yields only text. Loop until + # we find a text chunk or the generator is exhausted. + while True: + try: + chunk = await self._generator.__anext__() + except StopAsyncIteration: + await self._async_finalize() + raise assert chunk is not None - self._chunks.append(chunk) - return chunk - except StopAsyncIteration: - if self.conversation: - self.conversation.responses.append(self) - self._end = time.monotonic() - self._done = True - if hasattr(self, "_generator"): - del self._generator - await self._on_done() - raise + text = self._process_chunk(chunk) + if text is not None: + return text + + async def astream_events(self): + """Yield StreamEvent objects as the model produces them (async).""" + if self._done: + for event in self._stream_events: + yield event + return + + self._start = time.monotonic() + self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) + self._ensure_async_generator() + try: + while True: + try: + chunk = await self._generator.__anext__() + except StopAsyncIteration: + await self._async_finalize() + return + assert chunk is not None + self._process_chunk(chunk) + yield self._stream_events[-1] + finally: + pass + + async def messages(self) -> list[Any]: + """List of Message objects produced by this response. + + Awaits ``self._force()`` so ``await response.messages()`` is + safe to call without first awaiting ``response.text()`` or + iterating the stream. Responses rehydrated via + ``AsyncResponse.from_dict`` short-circuit and return the + stored messages. + """ + await self._force() + return self._messages_now() async def _force(self): if not self._done: @@ -1443,19 +2586,22 @@ def text_or_raise(self) -> str: return "".join(self._chunks) async def text(self) -> str: + "Return the full text of the response, executing the prompt if needed." await self._force() return "".join(self._chunks) - async def tool_calls(self) -> List[ToolCall]: + async def tool_calls(self) -> list[ToolCall]: + "Return the list of tool calls made during this response." await self._force() return self._tool_calls - def tool_calls_or_raise(self) -> List[ToolCall]: + def tool_calls_or_raise(self) -> list[ToolCall]: if not self._done: raise ValueError("Response not yet awaited") return self._tool_calls - async def json(self) -> Optional[Dict[str, Any]]: + async def json(self) -> dict[str, Any] | None: + "Return the raw JSON response from the model, if available." await self._force() return self.response_json @@ -1468,6 +2614,7 @@ async def datetime_utc(self) -> str: return self._start_utcnow.isoformat() if self._start_utcnow else "" async def usage(self) -> Usage: + "Return token usage information for this response." await self._force() return Usage( input=self.input_tokens, @@ -1514,6 +2661,11 @@ async def to_sync_response(self) -> Response: response._prompt_json = self._prompt_json response.response_json = self.response_json response._tool_calls = list(self._tool_calls) + # Without these the sync response falls back to assembling a bare + # TextPart from _chunks, so reasoning, redacted markers and every + # part's provider_metadata are lost. The CLI converts before + # logging, so that loss would apply to every async response. + response._stream_events = list(self._stream_events) response.attachments = list(self.attachments) response.resolved_model = self.resolved_model return response @@ -1523,7 +2675,7 @@ def fake( cls, model: "AsyncModel", prompt: str, - *attachments: List[Attachment], + *attachments: list[Attachment], system: str, response: str, ): @@ -1546,14 +2698,129 @@ def __repr__(self): text = "... not yet awaited ..." if self._done: text = "".join(self._chunks) - return "".format(self.prompt.prompt, text) + return f"" + + +def _append_tool_results_to_chain(chain, tool_results, attachments) -> list[Any]: + """Append a tool-role message carrying ToolResults to a message + chain, plus a trailing user-role message for any attachments the + tools returned (mimics the legacy attachments=[] kwarg behavior).""" + from .parts import ( + AttachmentPart, + Message, + ToolResultPart, + ) + + if tool_results: + chain.append( + Message( + role="tool", + parts=[ + ToolResultPart( + name=tr.name, + output=tr.output, + tool_call_id=tr.tool_call_id, + exception=_format_tool_exception(tr.exception), + ) + for tr in tool_results + ], + ) + ) + if attachments: + chain.append( + Message( + role="user", + parts=[AttachmentPart(attachment=a) for a in attachments], + ) + ) + return chain + + +def _chain_for_tool_results(prior_response, tool_results, attachments) -> list[Any]: + """Build the message chain for a tool-result turn in a chain loop. + + Takes the prior response's full input chain + its structured + output, then appends a tool-role message carrying the new + ToolResult outputs. + + This is what gives ``response.prompt.messages`` on the tool- + result turn the complete history for the next provider call — + including any reasoning signatures or thoughtSignatures from the + prior turn. + """ + chain: list[Any] = list(prior_response.prompt.messages) + list( + prior_response._messages_now() + ) + return _append_tool_results_to_chain(chain, tool_results, attachments) + + +def _trailing_pending_tool_calls(messages) -> list[ToolCall]: + """Find unresolved tool calls at the end of a message history. + + Returns ToolCall objects from the last assistant message containing + locally-executable tool_call parts, minus any that already have a + matching tool_result in subsequent tool-role messages. Returns [] + when the history has moved on past those calls (a user/assistant/ + system message follows them) - resuming only makes sense when the + calls are the latest thing that happened. + + Matching uses tool_call_id when present; id-less calls (histories + persisted before ids were guaranteed) match results by name, one + result consumed per call. + """ + from .parts import ToolCallPart, ToolResultPart + + last_index = None + call_parts: list[Any] = [] + for i, msg in enumerate(messages or []): + parts = getattr(msg, "parts", None) or [] + calls = [ + p for p in parts if isinstance(p, ToolCallPart) and not p.server_executed + ] + if getattr(msg, "role", None) == "assistant" and calls: + last_index = i + call_parts = calls + if last_index is None: + return [] + + results: list[Any] = [] + for msg in messages[last_index + 1 :]: + role = getattr(msg, "role", None) + if role == "tool": + results.extend( + p + for p in (getattr(msg, "parts", None) or []) + if isinstance(p, ToolResultPart) + ) + else: + # Conversation moved on past these calls + return [] + + matched_ids = {r.tool_call_id for r in results if r.tool_call_id} + unmatched_names = [r.name for r in results if not r.tool_call_id] + pending = [] + for part in call_parts: + if part.tool_call_id: + if part.tool_call_id in matched_ids: + continue + elif part.name in unmatched_names: + unmatched_names.remove(part.name) + continue + pending.append( + ToolCall( + name=part.name, + arguments=part.arguments or {}, + tool_call_id=part.tool_call_id, + ) + ) + return pending class _BaseChainResponse: prompt: "Prompt" stream: bool conversation: Optional["_BaseConversation"] = None - _key: Optional[str] = None + _key: str | None = None def __init__( self, @@ -1561,16 +2828,16 @@ def __init__( model: "_BaseModel", stream: bool, conversation: _BaseConversation, - key: Optional[str] = None, - chain_limit: Optional[int] = 10, - before_call: Optional[Union[BeforeCallSync, BeforeCallAsync]] = None, - after_call: Optional[Union[AfterCallSync, AfterCallAsync]] = None, + key: str | None = None, + chain_limit: int | None = 10, + before_call: BeforeCallSync | BeforeCallAsync | None = None, + after_call: AfterCallSync | AfterCallAsync | None = None, ): self.prompt = prompt self.model = model self.stream = stream self._key = key - self._responses: List[Any] = [] + self._responses: list[Any] = [] self.conversation = conversation self.chain_limit = chain_limit self.before_call = before_call @@ -1586,22 +2853,81 @@ def log_to_db(self, db): assert False, "Should have been a Response or AsyncResponse" sync_response.log_to_db(db) + def _pending_tool_calls(self) -> list[ToolCall]: + """Unresolved tool calls at the end of this chain's history. + + Non-empty when the supplied messages= end in an assistant + message whose tool calls have no results yet - e.g. a chain + that paused on PauseChain and is being resumed from persisted + history.""" + if not self.prompt.tools: + return [] + return _trailing_pending_tool_calls(self.prompt.messages) + + def _resume_prompt(self, tool_results: list[ToolResult]) -> Prompt: + """The first prompt for a resumed chain: the original history + plus a tool-role message carrying the freshly-executed results - + the same shape as the chain loop's own tool-result turns.""" + prompt = self.prompt + attachments = [] + for tool_result in tool_results: + attachments.extend(tool_result.attachments) + next_chain = _append_tool_results_to_chain( + list(prompt.messages), tool_results, attachments + ) + return Prompt( + "", + self.model, + tools=prompt.tools, + tool_results=tool_results, + messages=next_chain, + system=prompt._system, + system_fragments=prompt.system_fragments, + options=prompt.options, + attachments=attachments, + hide_reasoning=prompt.hide_reasoning, + ) + class ChainResponse(_BaseChainResponse): - _responses: List["Response"] - before_call: Optional[BeforeCallSync] = None - after_call: Optional[AfterCallSync] = None + _responses: list["Response"] + before_call: BeforeCallSync | None = None + after_call: AfterCallSync | None = None def responses(self) -> Iterator[Response]: prompt = self.prompt count = 0 - current_response: Optional[Response] = Response( + initial_response = Response( prompt, self.model, self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) + # Resume: a history ending in unresolved tool calls means a + # previous run stopped (paused or crashed) before executing + # them. Execute those calls first - through the normal + # before_call/after_call machinery - then start the loop on + # the tool-result turn. This could raise llm.PauseChain. + pending_tool_calls = self._pending_tool_calls() + if pending_tool_calls: + tool_results = initial_response.execute_tool_calls( + before_call=self.before_call, + after_call=self.after_call, + tool_calls_list=pending_tool_calls, + ) + initial_response = Response( + self._resume_prompt(tool_results), + self.model, + self.stream, + key=self._key, + conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, + ) + current_response: Response | None = initial_response while current_response: count += 1 yield current_response @@ -1617,47 +2943,129 @@ def responses(self) -> Iterator[Response]: for tool_result in tool_results: attachments.extend(tool_result.attachments) if tool_results: + # Pre-bake the full chain for the tool-result turn so + # response.prompt.messages is what gets sent — carries + # thoughtSignatures, thinking signatures, and everything + # else the model needs for the next call. + next_chain = _chain_for_tool_results( + current_response, tool_results, attachments + ) current_response = Response( Prompt( - "", # Next prompt is empty, tools drive it + "", # Next prompt text is empty; tool_results drive it self.model, tools=current_response.prompt.tools, tool_results=tool_results, + messages=next_chain, + # Carry system + system_fragments forward so + # stateless-per-turn adapters (OpenAI and + # friends that read prompt.system directly) + # keep seeing the system prompt on every call + # of the chain loop. + system=self.prompt._system, + system_fragments=self.prompt.system_fragments, options=self.prompt.options, attachments=attachments, + hide_reasoning=current_response.prompt.hide_reasoning, ), self.model, stream=self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) else: current_response = None break def __iter__(self) -> Iterator[str]: + # Rounds of a chain are separate model responses; joined with + # nothing between them the text of one runs straight into the + # next ("...have dragons.Now that I..."). Yield one space at + # each boundary where neither side brings its own whitespace. + # Display only: the separator never enters any response's + # recorded events, so it is not stored or hashed. + last_char = "" + for response_item in self.responses(): + first_chunk = True + for chunk in response_item: + if not chunk: + continue + if ( + first_chunk + and last_char + and not last_char.isspace() + and not chunk[0].isspace() + ): + yield " " + first_chunk = False + yield chunk + last_char = chunk[-1] + + def stream_events(self): + "Yield StreamEvents from every response in the chain." + from .parts import StreamEvent + + # The same round-boundary separator as __iter__, synthesized at + # the chain level so it is never part of a response's events. + last_char = "" for response_item in self.responses(): - yield from response_item + first_text = True + for event in response_item.stream_events(): + if event.type == "text" and event.chunk: + if ( + first_text + and last_char + and not last_char.isspace() + and not event.chunk[0].isspace() + ): + yield StreamEvent(type="text", chunk=" ") + first_text = False + last_char = event.chunk[-1] + yield event def text(self) -> str: return "".join(self) class AsyncChainResponse(_BaseChainResponse): - _responses: List["AsyncResponse"] - before_call: Optional[BeforeCallAsync] = None - after_call: Optional[AfterCallAsync] = None + _responses: list["AsyncResponse"] + before_call: BeforeCallAsync | None = None + after_call: AfterCallAsync | None = None async def responses(self) -> AsyncIterator[AsyncResponse]: prompt = self.prompt count = 0 - current_response: Optional[AsyncResponse] = AsyncResponse( + initial_response = AsyncResponse( prompt, self.model, self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) + # Resume: see ChainResponse.responses() - execute trailing + # unresolved tool calls before the first provider call. This + # could raise llm.PauseChain. + pending_tool_calls = self._pending_tool_calls() + if pending_tool_calls: + tool_results = await initial_response.execute_tool_calls( + before_call=self.before_call, + after_call=self.after_call, + tool_calls_list=pending_tool_calls, + ) + initial_response = AsyncResponse( + self._resume_prompt(tool_results), + self.model, + self.stream, + key=self._key, + conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, + ) + current_response: AsyncResponse | None = initial_response while current_response: count += 1 yield current_response @@ -1674,13 +3082,24 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: attachments = [] for tool_result in tool_results: attachments.extend(tool_result.attachments) + # Pre-bake chain so prompt.messages carries full history + # + any thinking/tool-call signatures from prior turn. + next_chain = _chain_for_tool_results( + current_response, tool_results, attachments + ) prompt = Prompt( "", self.model, tools=current_response.prompt.tools, tool_results=tool_results, + messages=next_chain, + # Carry system + system_fragments forward — same + # reasoning as the sync path. + system=self.prompt._system, + system_fragments=self.prompt.system_fragments, options=self.prompt.options, attachments=attachments, + hide_reasoning=current_response.prompt.hide_reasoning, ) current_response = AsyncResponse( prompt, @@ -1688,15 +3107,53 @@ async def responses(self) -> AsyncIterator[AsyncResponse]: stream=self.stream, key=self._key, conversation=self.conversation, + before_call=self.before_call, + after_call=self.after_call, ) else: current_response = None break async def __aiter__(self) -> AsyncIterator[str]: + # Round-boundary separator - same reasoning as the sync chain. + last_char = "" async for response_item in self.responses(): + first_chunk = True async for chunk in response_item: + if not chunk: + continue + if ( + first_chunk + and last_char + and not last_char.isspace() + and not chunk[0].isspace() + ): + yield " " + first_chunk = False yield chunk + last_char = chunk[-1] + + async def astream_events(self): + "Yield StreamEvents from every response in the chain." + from .parts import StreamEvent + + # Same round-boundary separator as __aiter__, synthesized at the + # chain level so it is never part of a response's events. + last_char = "" + async for response_item in self.responses(): + first_text = True + async for event in response_item.astream_events(): + if event.type == "text" and event.chunk: + if ( + first_text + and last_char + and not last_char.isspace() + and not event.chunk[0].isspace() + ): + yield StreamEvent(type="text", chunk=" ") + first_text = False + last_char = event.chunk[-1] + yield event async def text(self) -> str: all_chunks = [] @@ -1713,11 +3170,11 @@ class Options(BaseModel): class _get_key_mixin: - needs_key: Optional[str] = None - key: Optional[str] = None - key_env_var: Optional[str] = None + needs_key: str | None = None + key: str | None = None + key_env_var: str | None = None - def get_key(self, explicit_key: Optional[str] = None) -> Optional[str]: + def get_key(self, explicit_key: str | None = None) -> str | None: from llm import get_key if self.needs_key is None: @@ -1738,27 +3195,30 @@ def get_key(self, explicit_key: Optional[str] = None) -> Optional[str]: return key_value # Show a useful error message - message = "No key found - add one using 'llm keys set {}'".format( - self.needs_key - ) + message = f"No key found - add one using 'llm keys set {self.needs_key}'" if self.key_env_var: - message += " or set the {} environment variable".format(self.key_env_var) + message += f" or set the {self.key_env_var} environment variable" raise NeedsKeyException(message) class _BaseModel(ABC, _get_key_mixin): model_id: str can_stream: bool = False - attachment_types: Set = set() + attachment_types: set[str] | frozenset[str] = frozenset() supports_schema = False supports_tools = False + @property + def supported_server_side_tools(self) -> tuple[type[ServerSideTool], ...]: + """Server-side tool classes accepted by this model instance.""" + return () + class Options(_Options): pass def _validate_attachments( - self, attachments: Optional[List[Attachment]] = None + self, attachments: list[Attachment] | None = None ) -> None: if attachments and not self.attachment_types: raise ValueError("This model does not support attachments") @@ -1778,16 +3238,16 @@ def __str__(self) -> str: ) def __repr__(self) -> str: - return f"<{str(self)}>" + return f"<{self!s}>" class _Model(_BaseModel): def conversation( self, - tools: Optional[List[ToolDef]] = None, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - chain_limit: Optional[int] = None, + tools: list[ToolDef] | None = None, + before_call: BeforeCallSync | None = None, + after_call: AfterCallSync | None = None, + chain_limit: int | None = None, ) -> Conversation: return Conversation( model=self, @@ -1799,20 +3259,31 @@ def conversation( def prompt( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[Union[str, Fragment]]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[Union[str, Fragment]]] = None, + fragments: list[str | Fragment] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + system_fragments: list[str | Fragment] | None = None, + messages: list[Any] | None = None, stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - **options, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + options: dict | None = None, + hide_reasoning: bool = False, + **kwargs, ) -> Response: - key_value = options.pop("key", None) + key_value = kwargs.pop("key", None) + merged = _merge_options(options, kwargs) self._validate_attachments(attachments) + if messages is not None: + # messages= is the authoritative history; the other prompt + # arguments are this turn's new input, folded in so that + # response.prompt.messages stays exactly what the model sees. + messages = _append_turn_input( + list(messages), prompt, fragments, attachments, tool_results + ) return Response( Prompt( prompt, @@ -1823,8 +3294,10 @@ def prompt( tools=tools, tool_results=tool_results, system_fragments=system_fragments, + messages=messages, model=self, - options=self.Options(**options), + options=self.Options(**merged), + hide_reasoning=hide_reasoning, ), self, stream, @@ -1833,20 +3306,22 @@ def prompt( def chain( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, + fragments: list[str] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + system_fragments: list[str] | None = None, + messages: list[Any] | None = None, stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - before_call: Optional[BeforeCallSync] = None, - after_call: Optional[AfterCallSync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + before_call: BeforeCallSync | None = None, + after_call: AfterCallSync | None = None, + key: str | None = None, + options: dict | None = None, + hide_reasoning: bool = False, ) -> ChainResponse: return self.conversation().chain( prompt=prompt, @@ -1854,6 +3329,7 @@ def chain( attachments=attachments, system=system, system_fragments=system_fragments, + messages=messages, stream=stream, schema=schema, tools=tools, @@ -1862,6 +3338,7 @@ def chain( after_call=after_call, key=key, options=options, + hide_reasoning=hide_reasoning, ) @@ -1872,8 +3349,8 @@ def execute( prompt: Prompt, stream: bool, response: Response, - conversation: Optional[Conversation], - ) -> Iterator[str]: + conversation: Conversation | None, + ) -> Iterator[Union[str, "StreamEvent"]]: pass @@ -1884,19 +3361,19 @@ def execute( prompt: Prompt, stream: bool, response: Response, - conversation: Optional[Conversation], - key: Optional[str], - ) -> Iterator[str]: + conversation: Conversation | None, + key: str | None, + ) -> Iterator[Union[str, "StreamEvent"]]: pass class _AsyncModel(_BaseModel): def conversation( self, - tools: Optional[List[ToolDef]] = None, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - chain_limit: Optional[int] = None, + tools: list[ToolDef] | None = None, + before_call: BeforeCallAsync | None = None, + after_call: AfterCallAsync | None = None, + chain_limit: int | None = None, ) -> AsyncConversation: return AsyncConversation( model=self, @@ -1908,20 +3385,30 @@ def conversation( def prompt( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[Union[str, Fragment]]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - system_fragments: Optional[List[Union[str, Fragment]]] = None, + fragments: list[str | Fragment] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + system_fragments: list[str | Fragment] | None = None, + messages: list[Any] | None = None, stream: bool = True, - **options, + options: dict | None = None, + hide_reasoning: bool = False, + **kwargs, ) -> AsyncResponse: - key_value = options.pop("key", None) + key_value = kwargs.pop("key", None) + merged = _merge_options(options, kwargs) self._validate_attachments(attachments) + if messages is not None: + # Same fold as Model.prompt: messages= is the history, the + # other prompt arguments are this turn's new input. + messages = _append_turn_input( + list(messages), prompt, fragments, attachments, tool_results + ) return AsyncResponse( Prompt( prompt, @@ -1932,8 +3419,10 @@ def prompt( tools=tools, tool_results=tool_results, system_fragments=system_fragments, + messages=messages, model=self, - options=self.Options(**options), + options=self.Options(**merged), + hide_reasoning=hide_reasoning, ), self, stream, @@ -1942,20 +3431,22 @@ def prompt( def chain( self, - prompt: Optional[str] = None, + prompt: str | None = None, *, - fragments: Optional[List[str]] = None, - attachments: Optional[List[Attachment]] = None, - system: Optional[str] = None, - system_fragments: Optional[List[str]] = None, + fragments: list[str] | None = None, + attachments: list[Attachment] | None = None, + system: str | None = None, + system_fragments: list[str] | None = None, + messages: list[Any] | None = None, stream: bool = True, - schema: Optional[Union[dict, type[BaseModel]]] = None, - tools: Optional[List[ToolDef]] = None, - tool_results: Optional[List[ToolResult]] = None, - before_call: Optional[BeforeCallAsync] = None, - after_call: Optional[AfterCallAsync] = None, - key: Optional[str] = None, - options: Optional[dict] = None, + schema: dict | type[BaseModel] | None = None, + tools: list[ToolDef] | None = None, + tool_results: list[ToolResult] | None = None, + before_call: BeforeCallAsync | None = None, + after_call: AfterCallAsync | None = None, + key: str | None = None, + options: dict | None = None, + hide_reasoning: bool = False, ) -> AsyncChainResponse: return self.conversation().chain( prompt=prompt, @@ -1963,6 +3454,7 @@ def chain( attachments=attachments, system=system, system_fragments=system_fragments, + messages=messages, stream=stream, schema=schema, tools=tools, @@ -1971,6 +3463,7 @@ def chain( after_call=after_call, key=key, options=options, + hide_reasoning=hide_reasoning, ) @@ -1981,11 +3474,10 @@ async def execute( prompt: Prompt, stream: bool, response: AsyncResponse, - conversation: Optional[AsyncConversation], - ) -> AsyncGenerator[str, None]: + conversation: AsyncConversation | None, + ) -> AsyncGenerator[Union[str, "StreamEvent"], None]: if False: # Ensure it's a generator type yield "" - pass class AsyncKeyModel(_AsyncModel): @@ -1995,24 +3487,23 @@ async def execute( prompt: Prompt, stream: bool, response: AsyncResponse, - conversation: Optional[AsyncConversation], - key: Optional[str], - ) -> AsyncGenerator[str, None]: + conversation: AsyncConversation | None, + key: str | None, + ) -> AsyncGenerator[Union[str, "StreamEvent"], None]: if False: # Ensure it's a generator type yield "" - pass class EmbeddingModel(ABC, _get_key_mixin): model_id: str - key: Optional[str] = None - needs_key: Optional[str] = None - key_env_var: Optional[str] = None + key: str | None = None + needs_key: str | None = None + key_env_var: str | None = None supports_text: bool = True supports_binary: bool = False - batch_size: Optional[int] = None + batch_size: int | None = None - def _check(self, item: Union[str, bytes]): + def _check(self, item: str | bytes): if not self.supports_binary and isinstance(item, bytes): raise ValueError( "This model does not support binary data, only text strings" @@ -2022,14 +3513,14 @@ def _check(self, item: Union[str, bytes]): "This model does not support text strings, only binary data" ) - def embed(self, item: Union[str, bytes]) -> List[float]: + def embed(self, item: str | bytes) -> list[float]: "Embed a single text string or binary blob, return a list of floats" self._check(item) return next(iter(self.embed_batch([item]))) def embed_multi( - self, items: Iterable[Union[str, bytes]], batch_size: Optional[int] = None - ) -> Iterator[List[float]]: + self, items: Iterable[str | bytes], batch_size: int | None = None + ) -> Iterator[list[float]]: "Embed multiple items in batches according to the model batch_size" iter_items = iter(items) effective_batch_size = self.batch_size if batch_size is None else batch_size @@ -2051,28 +3542,29 @@ def checking_iter(inner_items): yield from self.embed_batch(batch_items) @abstractmethod - def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float]]: + def embed_batch(self, items: Iterable[str | bytes]) -> Iterator[list[float]]: """ Embed a batch of strings or blobs, return a list of lists of floats """ - pass def __str__(self) -> str: - return "{}: {}".format(self.__class__.__name__, self.model_id) + return f"{self.__class__.__name__}: {self.model_id}" def __repr__(self) -> str: - return f"<{str(self)}>" + return f"<{self!s}>" @dataclass class ModelWithAliases: + "A model with its optional async counterpart and aliases." + model: Model async_model: AsyncModel - aliases: Set[str] + aliases: set[str] def matches(self, query: str) -> bool: query_lower = query.lower() - all_strings: List[str] = [] + all_strings: list[str] = [] all_strings.extend(self.aliases) if self.model: all_strings.append(str(self.model)) @@ -2084,16 +3576,29 @@ def matches(self, query: str) -> bool: @dataclass class EmbeddingModelWithAliases: model: EmbeddingModel - aliases: Set[str] + aliases: set[str] def matches(self, query: str) -> bool: query_lower = query.lower() - all_strings: List[str] = [] + all_strings: list[str] = [] all_strings.extend(self.aliases) all_strings.append(str(self.model)) return any(query_lower in alias.lower() for alias in all_strings) +def _format_tool_exception(exception) -> str | None: + """Render a tool's exception the way it is recorded. + + ToolResult carries the exception object; ToolResultPart carries the + rendered string, so the chain has to convert rather than drop it. + """ + if exception is None: + return None + if isinstance(exception, str): + return exception + return f"{exception.__class__.__name__}: {exception!s}" + + def _conversation_name(text): # Collapse whitespace, including newlines text = re.sub(r"\s+", " ", text) diff --git a/llm/parts.py b/llm/parts.py new file mode 100644 index 000000000..4debba51f --- /dev/null +++ b/llm/parts.py @@ -0,0 +1,360 @@ +"""Part, Message, and StreamEvent value types. + +Parts represent the structured content of model interactions: text, +reasoning, tool calls, tool results, and attachments. A Message wraps a +list of Parts with a role. StreamEvent wraps a streaming chunk with type +information so consumers can distinguish text from reasoning from tool +call fragments as they arrive. + +These types are pure values — identity (ids, parent links, storage keys) +is a storage concern that lives elsewhere. Two Messages with identical +content are equal. +""" + +import base64 +from dataclasses import dataclass, field +from typing import Any + +from .models import Attachment +from .serialization import ( + AttachmentDict, + AttachmentPartDict, + MessageDict, + PartDict, + ReasoningPartDict, + TextPartDict, + ToolCallPartDict, + ToolResultPartDict, +) + + +def _attachment_to_dict(att: Attachment) -> AttachmentDict: + d: dict[str, Any] = {} + if att.type: + d["type"] = att.type + if att.url: + d["url"] = att.url + if att.path: + d["path"] = att.path + if att.content: + d["content"] = base64.b64encode(att.content).decode("ascii") + return d # type: ignore[return-value] + + +def _attachment_from_dict(d: AttachmentDict) -> Attachment: + raw_content = d.get("content") + content_bytes: bytes | None = None + if isinstance(raw_content, str): + content_bytes = base64.b64decode(raw_content) + return Attachment( + type=d.get("type"), + path=d.get("path"), + url=d.get("url"), + content=content_bytes, + ) + + +@dataclass +class Part: + """Base class for all parts. Role lives on the enclosing Message.""" + + def to_dict(self) -> PartDict: + raise NotImplementedError + + @staticmethod + def from_dict(d: PartDict) -> "Part": + if d["type"] == "text": + return TextPart( + text=d["text"], + provider_metadata=d.get("provider_metadata"), + ) + if d["type"] == "reasoning": + return ReasoningPart( + text=d["text"], + redacted=d.get("redacted", False), + provider_metadata=d.get("provider_metadata"), + ) + if d["type"] == "tool_call": + return ToolCallPart( + name=d["name"], + arguments=d["arguments"], + tool_call_id=d.get("tool_call_id"), + server_executed=d.get("server_executed", False), + provider_metadata=d.get("provider_metadata"), + ) + if d["type"] == "tool_result": + return ToolResultPart( + name=d["name"], + output=d["output"], + tool_call_id=d.get("tool_call_id"), + server_executed=d.get("server_executed", False), + exception=d.get("exception"), + attachments=[ + _attachment_from_dict(a) for a in d.get("attachments", []) + ], + provider_metadata=d.get("provider_metadata"), + ) + if d["type"] == "attachment": + att_dict = d.get("attachment") + attachment = _attachment_from_dict(att_dict) if att_dict else None + return AttachmentPart( + attachment=attachment, + provider_metadata=d.get("provider_metadata"), + ) + raise ValueError(f"Unknown part type: {d['type']!r}") + + +@dataclass +class TextPart(Part): + text: str = "" + provider_metadata: dict[str, Any] | None = None + + def to_dict(self) -> TextPartDict: + d: dict[str, Any] = {"type": "text", "text": self.text} + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + +@dataclass +class ReasoningPart(Part): + """Reasoning/thinking tokens from the model. + + `redacted=True, text=""` is the marker for the opaque-reasoning + case (OpenAI GPT-5 series, Gemini without `includeThoughts`) where + the provider reports that reasoning happened but withholds the + content. The actual token total lives on response.token_details + (e.g. `reasoning_tokens`); the Part only records the structural + fact that reasoning occurred. + """ + + text: str = "" + redacted: bool = False + provider_metadata: dict[str, Any] | None = None + + def to_dict(self) -> ReasoningPartDict: + d: dict[str, Any] = {"type": "reasoning", "text": self.text} + if self.redacted: + d["redacted"] = True + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + +@dataclass +class ToolCallPart(Part): + """A request by the model to call a tool. + + `server_executed=True` marks calls the provider executed on the + server (Anthropic web search, Gemini code execution) rather than + the LLM tool framework. + """ + + name: str = "" + arguments: dict[str, Any] = field(default_factory=dict) + tool_call_id: str | None = None + server_executed: bool = False + provider_metadata: dict[str, Any] | None = None + + def to_dict(self) -> ToolCallPartDict: + d: dict[str, Any] = { + "type": "tool_call", + "name": self.name, + "arguments": self.arguments, + } + if self.tool_call_id is not None: + d["tool_call_id"] = self.tool_call_id + if self.server_executed: + d["server_executed"] = True + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + +@dataclass +class ToolResultPart(Part): + """The result of a tool call.""" + + name: str = "" + output: str = "" + tool_call_id: str | None = None + server_executed: bool = False + attachments: list[Any] = field(default_factory=list) + exception: str | None = None + provider_metadata: dict[str, Any] | None = None + + def to_dict(self) -> ToolResultPartDict: + d: dict[str, Any] = { + "type": "tool_result", + "name": self.name, + "output": self.output, + } + if self.tool_call_id is not None: + d["tool_call_id"] = self.tool_call_id + if self.server_executed: + d["server_executed"] = True + if self.exception is not None: + d["exception"] = self.exception + if self.attachments: + d["attachments"] = [_attachment_to_dict(a) for a in self.attachments] + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + +@dataclass +class AttachmentPart(Part): + """An inline attachment (image, audio, file).""" + + attachment: Attachment | None = None + provider_metadata: dict[str, Any] | None = None + + def to_dict(self) -> AttachmentPartDict: + d: dict[str, Any] = {"type": "attachment"} + if self.attachment: + d["attachment"] = _attachment_to_dict(self.attachment) + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + +@dataclass +class Message: + """A single turn in a conversation: role + list of parts. + + `parts` contains one or more Part objects. `provider_metadata` + carries opaque provider-specific data attached to the message as a + whole; part-level data lives on the individual Part's + `provider_metadata`. + """ + + role: str + parts: list[Part] = field(default_factory=list) + provider_metadata: dict[str, Any] | None = None + + def to_dict(self) -> MessageDict: + d: dict[str, Any] = { + "role": self.role, + "parts": [p.to_dict() for p in self.parts], + } + if self.provider_metadata: + d["provider_metadata"] = self.provider_metadata + return d # type: ignore[return-value] + + @staticmethod + def from_dict(d: MessageDict) -> "Message": + return Message( + role=d["role"], + parts=[Part.from_dict(p) for p in d.get("parts", [])], + provider_metadata=d.get("provider_metadata"), + ) + + +def normalize_parts(items: Any) -> list[Part]: + """Normalize helper inputs to a list of Part objects. + + Accepts str (→ TextPart), Attachment (→ AttachmentPart), Part + (passed through), or a list/tuple of those (flattened one level). + """ + out: list[Part] = [] + for item in items: + if isinstance(item, Part): + out.append(item) + elif isinstance(item, str): + out.append(TextPart(text=item)) + elif isinstance(item, Attachment): + out.append(AttachmentPart(attachment=item)) + elif isinstance(item, (list, tuple)): + out.extend(normalize_parts(item)) + else: + raise TypeError(f"Cannot convert {item!r} to an llm Part") + return out + + +def system(*items: Any, provider_metadata: dict[str, Any] | None = None) -> Message: + "Build a Message with role='system'." + return Message( + role="system", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +def user(*items: Any, provider_metadata: dict[str, Any] | None = None) -> Message: + "Build a Message with role='user'." + return Message( + role="user", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +def assistant(*items: Any, provider_metadata: dict[str, Any] | None = None) -> Message: + "Build a Message with role='assistant'." + return Message( + role="assistant", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +def tool_message( + *items: Any, provider_metadata: dict[str, Any] | None = None +) -> Message: + "Build a Message with role='tool' (typically wrapping ToolResultParts)." + return Message( + role="tool", + parts=normalize_parts(items), + provider_metadata=provider_metadata, + ) + + +@dataclass +class StreamEvent: + """A streaming event from a model response. + + `part_index` groups events into parts. When left at its default of + `None`, the framework allocates an index automatically: consecutive + same-family text/reasoning events concatenate, tool-call events + group by `tool_call_id`, and `tool_result` always starts its own + part. Pass an explicit integer only to override the default + grouping (e.g. forcing a single TextPart across non-adjacent text + bursts). + + `redacted=True` (only meaningful on `type="reasoning"` events with + an empty `chunk`) signals that opaque reasoning happened — content + withheld by the provider, token total on response.token_details. + The framework hoists redacted reasoning Parts to the start of the + assembled message regardless of when they were emitted in the + stream, so UIs can render them before the visible content. + + `provider_metadata` carries opaque provider data (Anthropic + `signature`, Gemini `thoughtSignature`, OpenAI `encrypted_content`) + that must be echoed back on the next request; the framework merges + it onto the finalized Part (last non-None wins per top-level key). + A reasoning event with an empty `chunk` and `redacted=False` still + assembles into a ReasoningPart when it carries provider_metadata — + the metadata-only form used for opaque reasoning state such as an + Anthropic omitted-thinking signature or `redacted_thinking` data. + Unlike `redacted=True` markers, metadata-only reasoning Parts are + not hoisted: they keep their emitted position, because providers + require opaque blocks replayed in their original order. + + `message_index` is for providers that emit multiple assistant + messages in a single response (OpenAI Responses server-side tool + execution interleaves multiple `message` output items with tool + calls); most plugins leave it at 0 and get a single assistant + Message. Events with distinct indexes assemble into distinct + Messages, in first-seen order. + """ + + type: str # "text" / "reasoning" / "tool_call_name" / + # "tool_call_args" / "tool_result" + chunk: str + part_index: int | None = None + tool_call_id: str | None = None + server_executed: bool = False + tool_name: str | None = None + redacted: bool = False + provider_metadata: dict[str, Any] | None = None + message_index: int = 0 diff --git a/llm/plugins.py b/llm/plugins.py index 0125ede04..49a134dac 100644 --- a/llm/plugins.py +++ b/llm/plugins.py @@ -1,8 +1,10 @@ import importlib -from importlib import metadata import os -import pluggy import sys +from importlib import metadata + +import pluggy + from . import hookspecs DEFAULT_PLUGINS = ( diff --git a/llm/serialization.py b/llm/serialization.py new file mode 100644 index 000000000..b2e5ae89c --- /dev/null +++ b/llm/serialization.py @@ -0,0 +1,182 @@ +"""TypedDict spec for the JSON-safe wire form of Part, Message, and Response. + +These are the exact shapes returned by ``Part.to_dict()``, +``Message.to_dict()``, and ``Response.to_dict()`` — and accepted by the +matching ``from_dict`` classmethods. They are the canonical wire format; +use them to annotate any code that reads or writes serialized llm data. + +Example:: + + from llm.serialization import MessageDict + + def save_messages(conn, messages: list[MessageDict]) -> None: + for m in messages: + conn.execute( + "INSERT INTO messages(role, parts_json) VALUES (?, ?)", + (m["role"], json.dumps(m["parts"])), + ) + +Or pair with Pydantic's TypeAdapter for runtime validation:: + + from pydantic import TypeAdapter + from llm.serialization import MessageDict + + msg = TypeAdapter(MessageDict).validate_python(incoming_dict) + +Or export JSON Schema for cross-language consumers:: + + schema = TypeAdapter(MessageDict).json_schema() + +The TypedDicts are erased at runtime — zero overhead. ``NotRequired`` +keys may be absent from a serialized payload; required keys must +always be present. +""" + +from typing import Any, Literal + +# NotRequired moved to typing in 3.11; use typing_extensions for 3.10 +# support. typing_extensions is a transitive dep via pydantic. +from typing_extensions import NotRequired, TypedDict + +__all__ = [ + "AttachmentDict", + "AttachmentPartDict", + "MessageDict", + "PartDict", + "PromptDict", + "ReasoningPartDict", + "ResponseDict", + "TextPartDict", + "ToolCallPartDict", + "ToolResultPartDict", + "UsageDict", +] + + +# ---- Attachment payload (nested inside AttachmentPartDict + tool results) ---- + + +class AttachmentDict(TypedDict, total=False): + """Nested attachment payload. All fields optional — an Attachment + may carry a type, a url, a path, and/or base64-encoded content. + """ + + type: str + url: str + path: str + # base64-encoded bytes when the attachment was constructed with raw + # content= bytes. + content: str + + +# ---- Per-Part TypedDicts (discriminated by the `type` field) ----------------- + + +class TextPartDict(TypedDict): + type: Literal["text"] + text: str + provider_metadata: NotRequired[dict[str, Any]] + + +class ReasoningPartDict(TypedDict): + type: Literal["reasoning"] + text: str + # `redacted=True` with `text=""` is the marker for opaque + # reasoning (OpenAI GPT-5, Gemini without thoughts). The token + # total lives on response usage, not on the Part. + redacted: NotRequired[bool] + provider_metadata: NotRequired[dict[str, Any]] + + +class ToolCallPartDict(TypedDict): + type: Literal["tool_call"] + name: str + arguments: dict[str, Any] + tool_call_id: NotRequired[str] + # True for provider-executed calls (Anthropic web search, Gemini code + # execution). Adapters use this to restore provider-side blocks on + # the next turn. + server_executed: NotRequired[bool] + provider_metadata: NotRequired[dict[str, Any]] + + +class ToolResultPartDict(TypedDict): + type: Literal["tool_result"] + name: str + output: str + tool_call_id: NotRequired[str] + server_executed: NotRequired[bool] + exception: NotRequired[str] + attachments: NotRequired[list[AttachmentDict]] + provider_metadata: NotRequired[dict[str, Any]] + + +class AttachmentPartDict(TypedDict): + type: Literal["attachment"] + attachment: NotRequired[AttachmentDict] + provider_metadata: NotRequired[dict[str, Any]] + + +PartDict = ( + TextPartDict + | ReasoningPartDict + | ToolCallPartDict + | ToolResultPartDict + | AttachmentPartDict +) +"""Discriminated union of Part dict shapes. Use with +``pydantic.TypeAdapter(PartDict)`` to validate / dispatch by ``type``. +""" + + +# ---- Message ---------------------------------------------------------------- + + +class MessageDict(TypedDict): + """JSON-safe form of ``llm.Message``. + + ``role`` is one of "user", "assistant", "system", "tool" in practice + — typed as ``str`` here to leave room for provider-specific values. + """ + + role: str + parts: list[PartDict] + provider_metadata: NotRequired[dict[str, Any]] + + +# ---- Response + nested shapes ----------------------------------------------- + + +class PromptDict(TypedDict): + """The ``prompt`` sub-dict of ``Response.to_dict()`` — captures the + full input chain that was sent for this turn plus any options that + apply.""" + + messages: list[MessageDict] + options: NotRequired[dict[str, Any]] + system: NotRequired[str] + + +class UsageDict(TypedDict, total=False): + """Optional usage block on ``ResponseDict``. All fields optional; + providers vary in which they report.""" + + input: int + output: int + details: dict[str, Any] + + +class ResponseDict(TypedDict): + """JSON-safe form of ``llm.Response`` — everything needed for + ``Response.from_dict`` to rehydrate and ``response.reply()`` to + continue a conversation across a process boundary. + """ + + model: str + prompt: PromptDict + messages: list[MessageDict] + # Audit fields — present on a freshly-serialized response, optional + # on hand-constructed ones. + id: NotRequired[str] + usage: NotRequired[UsageDict] + datetime_utc: NotRequired[str] diff --git a/llm/templates.py b/llm/templates.py index 657a47641..1919e0484 100644 --- a/llm/templates.py +++ b/llm/templates.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel, ConfigDict import string -from typing import Optional, Any, Dict, List, Tuple +from typing import Any + +from pydantic import BaseModel, ConfigDict class AttachmentType(BaseModel): @@ -9,21 +10,23 @@ class AttachmentType(BaseModel): class Template(BaseModel): + """A reusable prompt template.""" + name: str - prompt: Optional[str] = None - system: Optional[str] = None - attachments: Optional[List[str]] = None - attachment_types: Optional[List[AttachmentType]] = None - model: Optional[str] = None - defaults: Optional[Dict[str, Any]] = None - options: Optional[Dict[str, Any]] = None - extract: Optional[bool] = None # For extracting fenced code blocks - extract_last: Optional[bool] = None - schema_object: Optional[dict] = None - fragments: Optional[List[str]] = None - system_fragments: Optional[List[str]] = None - tools: Optional[List[str]] = None - functions: Optional[str] = None + prompt: str | None = None + system: str | None = None + attachments: list[str] | None = None + attachment_types: list[AttachmentType] | None = None + model: str | None = None + defaults: dict[str, Any] | None = None + options: dict[str, Any] | None = None + extract: bool | None = None # For extracting fenced code blocks + extract_last: bool | None = None + schema_object: dict | None = None + fragments: list[str] | None = None + system_fragments: list[str] | None = None + tools: list[str] | None = None + functions: str | None = None model_config = ConfigDict(extra="forbid") @@ -37,16 +40,17 @@ def __init__(self, **data): self._functions_is_trusted = False def evaluate( - self, input: str, params: Optional[Dict[str, Any]] = None - ) -> Tuple[Optional[str], Optional[str]]: + self, input: str, params: dict[str, Any] | None = None + ) -> tuple[str | None, str | None]: + """Evaluate the template with the given input and parameters, returning (prompt, system).""" params = params or {} params["input"] = input if self.defaults: for k, v in self.defaults.items(): if k not in params: params[k] = v - prompt: Optional[str] = None - system: Optional[str] = None + prompt: str | None = None + system: str | None = None if not self.prompt: system = self.interpolate(self.system, params) prompt = input @@ -56,6 +60,7 @@ def evaluate( return prompt, system def vars(self) -> set: + """Return the set of variable names used in the prompt and system templates.""" all_vars = set() for text in [self.prompt, self.system]: if not text: @@ -64,7 +69,8 @@ def vars(self) -> set: return all_vars @classmethod - def interpolate(cls, text: Optional[str], params: Dict[str, Any]) -> Optional[str]: + def interpolate(cls, text: str | None, params: dict[str, Any]) -> str | None: + """Substitute template variables in text with values from params, raising MissingVariables if any are absent.""" if not text: return text # Confirm all variables in text are provided @@ -78,7 +84,8 @@ def interpolate(cls, text: Optional[str], params: Dict[str, Any]) -> Optional[st return string_template.substitute(**params) @staticmethod - def extract_vars(string_template: string.Template) -> List[str]: + def extract_vars(string_template: string.Template) -> list[str]: + """Extract and return the list of named variable identifiers from a string.Template.""" return [ match.group("named") for match in string_template.pattern.finditer(string_template.template) diff --git a/llm/tools.py b/llm/tools.py index 5ac0a7dcb..0970205c7 100644 --- a/llm/tools.py +++ b/llm/tools.py @@ -1,6 +1,6 @@ +import time from datetime import datetime, timezone from importlib.metadata import version -import time def llm_version() -> str: @@ -12,7 +12,7 @@ def llm_time() -> dict: "Returns the current time, as local time and UTC" # Get current times utc_time = datetime.now(timezone.utc) - local_time = datetime.now() + local_time = datetime.now(timezone.utc).astimezone() # Get timezone information local_tz_name = time.tzname[time.localtime().tm_isdst] diff --git a/llm/utils.py b/llm/utils.py index 58194bd6a..5f6cfddb7 100644 --- a/llm/utils.py +++ b/llm/utils.py @@ -1,22 +1,20 @@ -import click import hashlib -import httpx import itertools import json +import os import pathlib -import puremagic import re -import sqlite_utils import textwrap -from typing import Any, List, Dict, Optional, Tuple, Type -import os import threading import time -from typing import Final +from typing import Any, Final +import click +import httpx +import puremagic +import sqlite_utils from ulid import ULID - MIME_TYPE_FIXES = { "audio/wave": "audio/wav", } @@ -35,7 +33,7 @@ def id(self): return hashlib.sha256(self.encode("utf-8")).hexdigest() -def mimetype_from_string(content) -> Optional[str]: +def mimetype_from_string(content) -> str | None: try: type_ = puremagic.from_string(content, mime=True) return MIME_TYPE_FIXES.get(type_, type_) @@ -43,7 +41,7 @@ def mimetype_from_string(content) -> Optional[str]: return None -def mimetype_from_path(path) -> Optional[str]: +def mimetype_from_path(path) -> str | None: try: type_ = puremagic.from_file(path, mime=True) return MIME_TYPE_FIXES.get(type_, type_) @@ -52,8 +50,8 @@ def mimetype_from_path(path) -> Optional[str]: def dicts_to_table_string( - headings: List[str], dicts: List[Dict[str, str]] -) -> List[str]: + headings: list[str], dicts: list[dict[str, str]] +) -> list[str]: max_lengths = [len(h) for h in headings] # Compute maximum length for each column @@ -180,7 +178,7 @@ def token_usage_string(input_tokens, output_tokens, token_details) -> str: return ", ".join(bits) -def extract_fenced_code_block(text: str, last: bool = False) -> Optional[str]: +def extract_fenced_code_block(text: str, last: bool = False) -> str | None: """ Extracts and returns Markdown fenced code block found in the given text. @@ -218,7 +216,7 @@ def extract_fenced_code_block(text: str, last: bool = False) -> Optional[str]: return None -def make_schema_id(schema: dict) -> Tuple[str, str]: +def make_schema_id(schema: dict) -> tuple[str, str]: schema_json = json.dumps(schema, separators=(",", ":")) schema_id = hashlib.blake2b(schema_json.encode(), digest_size=16).hexdigest() return schema_id, schema_json @@ -283,9 +281,9 @@ def resolve_schema_input(db, schema_input, load_template): template = load_template(name) schema_object = template.schema_object except ValueError: - raise click.ClickException("Invalid template: {}".format(name)) + raise click.ClickException(f"Invalid template: {name}") if not schema_object: - raise click.ClickException("Template '{}' has no schema".format(name)) + raise click.ClickException(f"Template '{name}' has no schema") return template.schema_object if schema_input.strip().startswith("{"): try: @@ -352,7 +350,7 @@ def schema_summary(schema: dict) -> str: return "" -def schema_dsl(schema_dsl: str, multi: bool = False) -> Dict[str, Any]: +def schema_dsl(schema_dsl: str, multi: bool = False) -> dict[str, Any]: """ Build a JSON schema from a concise schema string. @@ -373,7 +371,7 @@ def schema_dsl(schema_dsl: str, multi: bool = False) -> Dict[str, Any]: } # Initialize the schema dictionary with required elements - json_schema: Dict[str, Any] = {"type": "object", "properties": {}, "required": []} + json_schema: dict[str, Any] = {"type": "object", "properties": {}, "required": []} # Check if the schema is newline-separated or comma-separated if "\n" in schema_dsl: @@ -393,6 +391,10 @@ def schema_dsl(schema_dsl: str, multi: bool = False) -> Dict[str, Any]: # Process field name and type field_parts = field_info.strip().split() + if not field_parts: + raise ValueError( + f"Invalid schema DSL: field {field!r} is missing a name before ':'" + ) field_name = field_parts[0].strip() # Default type is string @@ -486,11 +488,10 @@ def ensure_fragment(db, content): source = None if isinstance(content, Fragment): source = content.source - with db.conn: - db.execute(sql, {"hash": hash_id, "content": content, "source": source}) - return list( - db.query("select id from fragments where hash = :hash", {"hash": hash_id}) - )[0]["id"] + db.execute(sql, {"hash": hash_id, "content": content, "source": source}) + return db.execute( + "select id from fragments where hash = :hash", {"hash": hash_id} + ).fetchone()[0] def ensure_tool(db, tool): @@ -499,20 +500,39 @@ def ensure_tool(db, tool): values (:hash, :name, :description, :input_schema, :plugin) on conflict(hash) do nothing """ - with db.conn: - db.execute( - sql, - { - "hash": tool.hash(), - "name": tool.name, - "description": tool.description, - "input_schema": json.dumps(tool.input_schema), - "plugin": tool.plugin, - }, - ) - return list( - db.query("select id from tools where hash = :hash", {"hash": tool.hash()}) - )[0]["id"] + # No `with db.conn:` here - its exit commit would also commit any + # open outer transaction, such as the one wrapping a turn write. + db.execute( + sql, + { + "hash": tool.hash(), + "name": tool.name, + "description": tool.description, + "input_schema": json.dumps(tool.input_schema), + "plugin": tool.plugin, + }, + ) + return db.execute( + "select id from tools where hash = :hash", {"hash": tool.hash()} + ).fetchone()[0] + + +def ensure_tool_instance(db, name, plugin, arguments) -> int: + """Row id in tool_instances for this configuration, storing each + distinct (plugin, name, arguments) once however many turns and + calls it serves.""" + match = db.execute( + "select id from tool_instances where name is ? " + "and plugin is ? and arguments is ?", + [name, plugin, arguments], + ).fetchone() + if match: + return match[0] + return ( + db["tool_instances"] + .insert({"name": name, "plugin": plugin, "arguments": arguments}) + .last_pk + ) def maybe_fenced_code(content: str) -> str: @@ -552,7 +572,7 @@ def has_plugin_prefix(value: str) -> bool: return bool(_plugin_prefix_re.match(value)) -def _parse_kwargs(arg_str: str) -> Dict[str, Any]: +def _parse_kwargs(arg_str: str) -> dict[str, Any]: """Parse key=value pairs where each value is valid JSON.""" tokens = [] buf = [] @@ -589,7 +609,7 @@ def _parse_kwargs(arg_str: str) -> Dict[str, Any]: if buf: tokens.append("".join(buf).strip()) - kwargs: Dict[str, Any] = {} + kwargs: dict[str, Any] = {} for token in tokens: if not token: continue @@ -606,7 +626,7 @@ def _parse_kwargs(arg_str: str) -> Dict[str, Any]: return kwargs -def instantiate_from_spec(class_map: Dict[str, Type], spec: str): +def instantiate_from_spec(class_map: dict[str, type], spec: str): """ Instantiate a class from a specification string with flexible argument formats. @@ -666,7 +686,7 @@ def instantiate_from_spec(class_map: Dict[str, Type], spec: str): return cls(**kw) # Starts with quote / number / [ / t f n for single positional JSON value - if re.match(r'\s*(["\[\d\-]|true|false|null)', arg_body, re.I): + if re.match(r'\s*(["\[\d\-]|true|false|null)', arg_body, re.IGNORECASE): try: positional_value = json.loads(arg_body) except json.JSONDecodeError as e: @@ -683,7 +703,7 @@ def instantiate_from_spec(class_map: Dict[str, Type], spec: str): RANDOMNESS_LEN = 10 _lock: Final = threading.Lock() -_last: Optional[bytes] = None # 16-byte last produced ULID +_last: bytes | None = None # 16-byte last produced ULID def monotonic_ulid() -> ULID: diff --git a/mypy.ini b/mypy.ini index a17287e45..cfc209480 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,3 @@ ignore_missing_imports = True [mypy-click_default_group.*] ignore_missing_imports = True - -[mypy-sqlite_migrate.*] -ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 6c3c5e70c..8ec3191a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" -version = "0.28" +version = "0.32" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -26,11 +26,10 @@ classifiers = [ dependencies = [ "click", - "condense-json>=0.1.3", - "openai>=1.55.3", + "condense-json>=1.1", + "openai>=2.32.0", "click-default-group>=1.2.3", - "sqlite-utils>=3.37", - "sqlite-migrate>=0.1a2", + "sqlite-utils>=4.0", "pydantic>=2.0.0", "PyYAML", "pluggy", @@ -44,16 +43,16 @@ dependencies = [ [dependency-groups] dev = [ "build", - "click<8.2.0", # https://github.com/simonw/llm/issues/1024 + "click>=8.2.0", "pytest", "numpy", "pytest-httpx>=0.33.0", "pytest-asyncio", "cogapp", "mypy>=1.10.0", - "black>=25.1.0", + "black>=26.3.1", "pytest-recording", - "ruff", + "ruff>=0.16.0", "syrupy", "types-click", "types-PyYAML", @@ -81,3 +80,6 @@ llm = "llm.cli:cli" [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["llm*"] diff --git a/ruff.toml b/ruff.toml index 7a3f6a3f8..567b4d53f 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1 +1,2 @@ line-length = 160 +target-version = "py310" diff --git a/tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml b/tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml new file mode 100644 index 000000000..1faaee3df --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_basic_non_streaming.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Reply + with exactly: pong"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '182' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAAA3RU247jIAx971dEPE9HSS9J2l8ZrZADTsuWAAumM9Wo/74KadJmpn1LfPDB9vHh + e5FlTEm2z5jH4HheS9mut0W1LcpNJcu8KHctNEVTQwWiLiCvi2aHq02eY121Tblmbz2Fbf6ioJHG + moBDXHgEQsmhx4qqqvN1VVRlwgIBxdDnCNs5jYRySGpAnA7eRtPX1YIOOISV1soc2D77XmRZljEH + F/R9vsQzauvQs0WWXYeLR8ofV1cJRe9tn2mi1inQevwX0YgLd2hA04Xts/w9T5gyIxmXSKB0eMxU + JpCPgpQ1s3gHX9xGcpE42RP+BslazQXoOV1nJeq+p4Oj5fZ9u1zlq3KZb5ar26T7Ax766x7ThpvY + PvtIoxkGNEnbhcNrZXf1qk7KNtiWuw0IWcntusjLdF9ioYvDxIMhwAHvwCsJEyisITT3oh4Lm9GO + Y8IvmrLTATDGEoyj/fgzA7U9OG+bJ0gi2mfMWXNgE3C9fU1nmTtCSAW0yoDmYMIn+nv93uqEQggq + EBgauHqexMEceNAa9VxK8nHYV+cxoBH4ZKWcx7OyMfDRLTwJNcnpvO0ccQHiiPyEl5eYx37EwzKw + 1eY47IhHCNbMvIJta32airaftx5ZiF0HfmSfvBOgRbpwJXvqVuHMKQH9WQnkpEbvtRD1IBwLZD3O + TEvYuX5fY4oXt/5vCt1qa63v4P7/sBnp3DD1W8ln9I0NKk2TdShV7O6mH3Q4WiUG4SJZNgH3RWFk + HX9Yn3wKuqTRrh4CPhoxOo1JFaDR4xMVkw+mDpSZ+bwo3n7HHx6Pqc8korwn5rNefz4f22fxZ7ST + /q+YyRLoh3rLaYQxzOXukEACQU9/XVz/AwAA//8DAEAnS6QwBgAA + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74ce61c87211fd-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 03:12:57 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '1036' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=UYuMLYCou_tA0.A8bkTTDs0INBmHlG2j7U9W9gdY8rQ-1778037177-1.0.1.1-5g.mxYNPGVDfBz2paukjljQbRq5PH1x6hIXpFFHvfnbkZxklDSJMF7lUnwNKNv48F1tAMpQxDGkwbCVoTbZJsbFM324N18q7IanrQ5hcaB8; + path=/; expires=Wed, 06-May-26 03:42:57 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=j9wkRiR3FXF4b0PnFv7DnktylnlM._0WSw8ZUpHPQmQ-1778037177704-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_c6713bbdac8d4639a5ea09cb6c5eb5a9 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml b/tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml new file mode 100644 index 000000000..2f7e820ea --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_basic_streaming.yaml @@ -0,0 +1,136 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Reply + with exactly: pong"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '181' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_00592e63e61b66660169fab1b9f8e481a2b321356198d7ac1b","object":"response","created_at":1778037177,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_00592e63e61b66660169fab1b9f8e481a2b321356198d7ac1b","object":"response","created_at":1778037177,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"pong","item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","logprobs":[],"obfuscation":"mZhSFjZXJr0X","output_index":0,"sequence_number":4} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","logprobs":[],"output_index":0,"sequence_number":5,"text":"pong"} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"},"sequence_number":6} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"}],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":7} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_00592e63e61b66660169fab1b9f8e481a2b321356198d7ac1b","object":"response","created_at":1778037177,"status":"completed","background":false,"completed_at":1778037178,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[{"id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"}],"phase":"final_answer","role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":11,"input_tokens_details":{"cached_tokens":0},"output_tokens":5,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":16},"user":null,"metadata":{}},"sequence_number":8} + + + ' + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74ce69fcf0cb75-DFW + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 06 May 2026 03:12:58 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '512' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=Xlm1i2d4hNcY9FYDrgbksx5Ak.InLICBWuWA7k.o5eU-1778037177.9137826-1.0.1.1-GD76qNB2IKnNrhlsjKWgXNF6BpybgPQmNoOTYCMh4ejIhEDWMMph.l_WZvek4hrh5qeLqHfJ28EH5H2c10KBdqGhET8jLthIUOa15fXX7oxYNG_MZ.CHZ9Fuj4bT9GU.; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:42:58 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '39999826' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_445d87d531af499daeb09f7826886b8c + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_openai_responses/test_responses_interleaved_reasoning_between_tool_calls.yaml b/tests/cassettes/test_openai_responses/test_responses_interleaved_reasoning_between_tool_calls.yaml new file mode 100644 index 000000000..ec692703c --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_interleaved_reasoning_between_tool_calls.yaml @@ -0,0 +1,540 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve + this puzzle: call db_lookup(''start''), then follow each instruction step by + step. Each lookup tells you the next key to use. Compute each step in your head. + State only the final integer."}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look + up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '551' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/31WWbOaShB+z6+wfA4pQNb7prKouCKuNylqgGGR1ZnBLXX++wXMUc9Jcn2woD+6 + p6e7Z77v55dWqx157X9abQRxYdMcK7uM7/q86/igw9OMIPvA8SHf4UVXYmSxehYAgILvcLzvO3L7 + ax0idw7QJe9h8gzDu91FEBDo2aDGGFGUaI6WaL7BMAGkxLWPm6dFAqvv7k4OcOMA5WVW5+WDBMO7 + OUqSKAsq28/qtTIU4ApR7e/BE0zyonqpgLf7wu8hPy0tNihEKK89szJJGoOP4LGEmXu1C5iBhFwr + kP5GN1iUvQezPUhAlOBXzyjDBJUuiapNv9pTcLHzkhQlsUkew99BkueJ7YLkY7g092BS7ykoCMV/ + 4ymWZgWK5ii20358gEC93KvbfaXK8m9TmnuBnq3Ff2+swAK5aazMcIIoQiADlmM5QDfLNUHItYD3 + 1gKcZ3ULHlBVMnQt6jK7eUZg1oxA0K1/vYJCdqaommIXY0+YqLd81telmQHjI6WepFPmzK14GUHO + zE99E2+Zsot2OKWvhnHtdtVdKBHztl8bKx8anjCwdupRR+cVHMpTTzQYrtQQve5LdlfVZlveNtPo + Fo6G2cGfjHv8BbHCMhg6y7i3K9Sxvrvl3IDKku5ofc5sK8giygD8aXoY7XHIUWf+vNouOlNgcAfa + 2Y52ypkYWrLgk/FtDjgjuMWbUSzQg4shceKe369jcYMH8tJQlPlyrGKXpNvLjj+Wo1jerpb6lI9N + 2pobOnaHF2tLbsp84JZeP+oePFM/0YFtDUfJcLUQcl/ZbhaJ5HjFLj8uho567RjhMezE2MiXushk + 9AqvzpTg0UPYmyxSmlu7etRfhcoAGKCjhtol5CfTIC3X6kBZFlzveobrsyHqnJ5FhidNTzEotas5 + W7Cz9JgdbGFigI3rDsFinfK7s7xZ0fZ8ONeUmXHFUHRTzJLkqhfaBSqBFSy7Jr6MwV7dT4YbN9pb + jAJuXXmMkMJ0QF/h4Hyw6Zsjp3BiZSBqk47FWshiBEqn435U2LNtKmv7/WV06gar7cTJ1ipmlmWw + 6fXZzOr7OzQ2z9vD9baPOzyMbydhPLyK+ygIaULZpjXboF48vXQp8XAK+nN7p/GMqzNZzoWWpg7C + 0C3Pu2U6WU4m3djdRLQSUtrWuYqxTA4rfWJYB8yYU3k6lIjLJstz2lW9cO/F9qhQs3XiF4qKD6Zm + DlXB9INQo5ZmxgZad+Wb7g0OjCJJyWZCQ8WIgCFsjYE1XwCC6fTG9BgBJOZopkLNOU/miy4zud5E + QAxH2vNDe4gSvRcyg1x2SrCzTc3vFzpM5+5MhibS8724GJ+Xi4FQnJkJUkq8twvCnYxdv3NUyg5S + 97YwG5y1XWbO1wd7UXIOkX0eIGNuGd2CDVMV74y+cgIb2CvYJFBnjiDgU4c7DoQjpvK8353pIkrJ + YTnur0tJo02GmUaaG51PaZKq/dnZkj0BpoFxyGSLs9l+D6zSXEaeGk39I+Wm4UQXASscdHDDw0Qe + 3qgClRcxXEuSyES7ZKqEmT1BkhDH/TTOjDHfIbpDsfLWNaiDtcqSsY4GaZj1Elro+7RPuaqILUFh + yQG4CgzluRCQkoZZz5875Wgg8nmX5ehBZKzEwOx5VLzLzkJ4CvsTYxUfHElaHXhmps0uPfqkjDON + s4WrhPD5usnnawhLc7IZjEhuQ4TNZVRustv6MOrmnGfIk7HfM8xwSEJho/RKwYRjlhf5hX9b7VTN + 0repPt92OrP1mabSIH1edbhMU4BqYvj3R2N7+/qnu9Z3/37XirTHSPVdK/kCBB4nsjIPKkZlf79r + /TJrOKVhiJck/sKZDQhQUKbVHdzgP7+3Y3j93v7ne+2EyPf22/PLOqh9z7d5HAoinXZvptDd4ZuX + H8FQFo4BHT89MpA2eXmOneR5XBbtew2q/x8NAxUAVZFg8pHXKmq8k3dR6YKKLOAf+LWCTlFeYvtd + Otwze3BbgaqtkiqkG0K72tNfMQRrBrozY5vlwjthPlnrIRyg7+eooaqwumN+bfKlwXX4h5LAwIfk + WuVUx/Yj+EE3YIhOUbUrEr0rER+UCWn/Ejg5gh8kDIFpUbN32diZXwUg8EKeyVWppeD5/jITzXev + o9c+QeTkOGrK2U6hF5XpUwLdGxHmVX41DEqStx8A/l0rfB69Z/M9iF0UFe+lHVcD0CqLFmidQFLC + lnNtVW1pRVmLhLBVlLdbAlseIMABGH77vxl6QPX0pFX7EH7Z+L2/VcGq6n60V8h9EF5NL1uoJFnd + 8Bfw7fH89vUZvlZ9EYLeoxgvwR/vP14c3hf4pXdfEOB5UV0gkMxfc256/+XTyk1+jVyuz8eng0Ty + oipPUO3cqQPQD2PRnBlZuhtQ1aV3Gdj2Igyc5F0/lxgE8DlQUfZBhErS19/tL8r25/OSqA6V93Sk + P4zeZ20r8H8C/hT3cSCfzhz/ITbJCUieKMN3HkNd4o8HsBoaUI9avcDbl7f/ADPaWJDQDAAA + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f7526f32c3c0044-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 04:13:27 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '2091' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=N7TAdwEKwrRihZ4HPzOd8tAdwT5JnZWp1Mj2.UfCC3g-1778040804.348554-1.0.1.1-NFuNpo0J7tlz8j4YDeoThOIY2CDX.4CkvnAcV4AwMaERfiFLSTPjDY8gcSPYiwIXP3TeplBxIS2vJK_VrxoTlp4TAVqwN8bgJ6CMnQFDfKuS7S0zRVqTDUsqYdgW6ccN; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 04:43:27 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_c48ea45fc57a407bb6b100db8c59af68 + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve + this puzzle: call db_lookup(''start''), then follow each instruction step by + step. Each lookup tells you the next key to use. Compute each step in your head. + State only the final integer."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe62a9c8197914677ea9a2424a0","encrypted_content":"gAAAAABp-r_nDEFD_pLd6MEzoOCG8OKekq-Ev8vnbPTkSie4RovCRsX1uArYsm0yKKyAAEYh8tRzZVKUfeKd6HTYEqGrwUeI9Nd7K14uFr0VC8_AEFOX5_RmizhJInjfMLB5xr26SgIbSkBYpELGYzo4H-nlAJVwn_Tgni-Ka5vNjJZsh4-w5wUXQ3NaK4j0bXJYDwtKFlQ5lLzPa4KgzkWJk60HxK847Z5ZVk7WsH9SKDDPSLEsctmXxY5quJk9XUSGN5kR0TPKGscIxTXtzDPHcudCiAjdRGv0g_TIJlIUQ6ofDXWQl8bdpYoqQIbEy3Khqh3ksKoSG71n0UsUw-6d0IeBMQm04VcGiCUhDHaKa3EhFxh5MNgmuVEHDSp4ByweVwK7G4GniKd8NvkauFyROQ2Omqnj_6MKaWccIaQVm5Yw9WU0_PIPFDOKyse7cms2tlyGpFxeDgTgSARsxLaZEZMIWciZT1DazA9LrrD13aCD4ePHWCRJbpbkDH7FM3T2TrT16-G0kCip_OXm9FZZxJvAgUXMbnVEs1SugWBC2nTCfYrLRwXjyzZk35ekzv6LIy7Zigh0t-_RTOWrBkNxA-7jvgCP_YF51cG1no4hTFEHhhcuwYSmMSMMAkcWi0Dh-FXby7k9tjUGMKTjs1RN9NI8tc2lSwmAEdhZdk_JpEnVlfpDEsjRFRIE6RfghF-SRn2gFAUfRczeHKplmtWM0eDKiaK6XKHTPQats0mz1B16alRJOEeFbwMPQA1Myz7atKb8Z5I_IrlGBh1Ho9buaY_RFfCpGemPcO9eRrGoZ7QLwSQH6pw1MrDusZ_pt4vKYC3qDu3rEZ_6OHwFYnRPVj_Qu4bt9f5arKPTKAp2hmEsYKCDvaWeBp2lgEOb66sv34qH6qs-ooCAOG7rmtjSLCVu8F0R11NiFciwvmlmECOwT9d6emgKjn9T4_2CBaUmo9rdEiNfq-cmhMG7a26jGazsIl9Iz-prux7hV8871iYlNDhn_Mr86kkCmknKL53tGb-29XcK-jTUnlLGrHmhnBl06Cf0f-cE7sT6D2tjacDeh9P6gtu0enBfPbuJH75oA240HiKU7gRBd-kYnw6hvhCMKUkjb88Uj51OFOxB0vDLnF4_6y8rswyWoPVeeuRMWHJto_ersRSiuWnzVjJAo4dK9MLfBKRhIth6WDBu6ReL2575QfzUYEFTGXmGPX33OVw0-mgm","summary":[]},{"type":"function_call","call_id":"call_I670mAzR6AYszdoqaI96qg0k","name":"db_lookup","arguments":"{\"key\": + \"start\"}"},{"type":"function_call_output","call_id":"call_I670mAzR6AYszdoqaI96qg0k","output":"Begin + with the value 7."}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look + up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2127' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/31XWbOiyBJ+719xwudmApR1IuZBERBRERG36QmigGKTtQABO85/v6B91NPL5cGg + 8qvMyq3Mj+9f3t4GgTP4+22AYJGZODnkbMK1Xcq2XDCicILmXGC5kOFYhmUJjuFI3GHhELcIAto4 + 6Qy+9iZSK4R2+WEmTQp4l9sIghI6JugxgmFYnMRZnLlhRQnKquh17DTOItjtuytZwD57KK2S3i8X + RAW8i4MoChKvk33vlp0gAy1Evb4DLzBKs27RAe/3gz9Mfj6awG8oRCjtNZMqim4CF8G8gondmhlM + QFS2HYj/dd8cJB/GTAeWIIiKV80gKUpU2WXQBf0qj0FjplWZVaVZpmf4K1imaWTaIPpsLk4dGPUx + eVmJUX9R2BAf0hhOYsPR4LEBgf64V7X7SZ3k31tq7gl6lrb4c2HZEcneCsv2DwDAtYbAItx7LW5G + yjaD99KCIk36EjygLmWozfo022lSwuTWAt64fyYZhswcKDNWwsY4yyrr2Bwr5517viLBrLlQCpxl + vvBH7iEFNSdb5VjnprVvmCqf+F4ocuV0pNB0E6Md6e0u0wMVu2FYiOSYRYgrnOO6XNbENrbrZVJO + dsplXpyVMAtPkC4op6iTK3+tjGsl0YaqZJssPWVJzMuwpvQYO6XNEV+H5eEYMubRuZ6vh9hil0Ri + iW0be3KUgilAMDoihhCxui1acyI367ErVpHN8HBrhtKRzrchdgAHIVzBSzY+bxqtGOVTsXCJy3w+ + 15cbo6IuM1HFzrFmXHiCZ5KVflawsdaIpOc3Q9m0pflKn1VFEko5JYMRx8xMnJ+oWrh1iNlcrkbG + XmK9aqmHMy4ewmixFI3MFZqLtSEDBYswWPP2PpNbd984iz2ajuR9HmuU7LCJIM9FhWHjkNzFi3XN + ztX1eYwSW7e9Q75V6KqLWzgMZ2WxEvhG3FA1AXcnnWiDRqBVvVOjrth0GjVNg2/TwBoTcy63C0pT + NhhuLBpzoZhrz+XGUGHN01DMZ6tE9KlE2e2dVj4emm1WoUwkciFRJrzO10BjDldjHs2xhDBE71Ti + xka0Cis/jg6Fz5MVQpe8qNc5sV5Mcl+zT83aKg75UrmKVb2l5zQvREMJGSkfzqzdNJd9IqQjm5pM + jwED1PExYSyh1mSz9N1YswJGQGB3WsaeJSSX47m5XOGaYcc6Tx6DY+3JV6kyNOVa5xtFsZoM0inc + CQwn8vDgC5ayHZ7wsz056UYaN67OB4tyteTxVcDtRddQK46MNuV27axnbLarMG0iwmvASm6jHmar + fFWJ+k42d1p3M+R0TQaAgMfGkod+Vo+j86hiHUPnL/NWdxaLmnPW+5XW2ClFXKcgbrWdAhMrEayQ + POWnjRJhw9JMoISP7QOVRwGfT8dknMcLhmwDhOJjSXNnWdRoXeLXGEuf4xGIvbGcqIU62emagmEz + oF4mhru1MX6/UNVV7oZDcDCCJa9c9OUl5TBarDySL+SDY3tNrmpHybUpNbMhG28NB+lh7FvEPliq + dHTGFzqa5bhsFLLPThPuQFl+V8WEdXiqjpgjYCap23pDYSJtU1K3mQBL3OHEFQ/MOd/GqrGRABmc + IOuvsiptS283jaywjVx81RqYJOBeXVmiF1v0NE8Yw5lDRzADcqrIdlllQZZBO64zT+V2RBJE2+iY + ro4hklAbbjcneiyEbArgZrgZSSRRG9lydqEroz1uZSUioqE9dmhbwj1qzAq8x1maKxv2eGxrq5Hg + uKmcqgdVyPklFvNCGK65WrMb8lquPNxMze2W5vOTt1am1+uepYzOda/cS75B0qS+Foz02MwRn+2D + JtdMdzteOSJZJOuRv5FIpgWVDBPRUUfEtQKuOkd2LkxbW3Gj6ewypo6GGihEoKZThQSW5p8mcB1n + +1YVXTaq3MJYSCks82Ahkbsl5TtBKOdnWd0HTqDok2VzYA8MZY3hlqr0UcLndKLj4WaxVebMYsHl + MxqPoYNxkntdNsohWc4CvIW1t4s11asDRKKCTYdVGRa+hVQ4vZLYCSMzvjjXp9wR8iOu/fPPcwQU + VRwD1A/Mf/+7yd6//m4GufafZxAYWjbZzyBAUCTrOC5OugRLctSvM8itktusvU3OFyf+wCVuIEBe + FXez6YZ//zY4w/bb4O9vA+bb4P25qzdo3n29vRohhdOBoIST0x7N1vmQH0nT0kNPjQTEN58cy4zS + 9Fxlg3v83e9/t6mcAdRZgtHnWd/RhTuhyTqu1A1Q+BvO0UGXIK0K84NO3T17zPsMdWGWnUnbh2YX + zx8xBPupfGcLgyHp30nEc5I/yBR03RTdxrcfeP6PIF+K25t/sKsCuLBsO596224AP3GpAqJL0EVV + Bh/szAVVVA5+kL4UwU+0roRx1jOa6iYnfiSghE35dK5zLQbP9Us/3Pa9tt3gApGVFsEtnYOu0YMq + ftLCeyH8tPOvh0FVpoMHUPzKn35uu2fxHVjYKMg+UrvoGuCtyt7A2wVEFXyz2reuLG9B8lb68C2r + rtcIvjmgBBYo4F//r4ceUN89cVc+VLwEfq9vl7Auu5/lHXJvhFfRSwgdTe0L/gK+P97fvz7N90w4 + QNB5JOPF+GP934vCxwE/vgFeEOA4QZ8gEK1ffb7V/stPJ9/8u31C9Pfjp4tUplmXHq+L3OoN4A9h + drszHHsXoK5KH9R44AQFsKKPb4qqAB58NlSQfCLmBEN8/RV4ofvfn/8S3a1ynpr4p977mfATBPs7 + 5HeGH1fyqc2xn4yXaQmiJzpkuUdbV8XnK9i1DeibrT/g/cv7/wBdVsXR5g0AAA== + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f7527072b8a0044-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 04:13:30 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '2766' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=DRZquxDLKhKw.kobZpdPpYdyAORTxb8d3bk6OV6L_3g-1778040807.5506942-1.0.1.1-C63sdNCp_e3AcsWa9mPoKJLTMlDCSogmc69aY3uzn3tTS0hpqMcaswl4Vlx9jLN79I4aGoL9PKji0tX10HxiuZxf4oEWmMjN_QopXLRhPxlCyZuOq26Bg0WQvU0arTlt; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 04:43:30 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_af2b96dc465344fbaecb33d287edd1c1 + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve + this puzzle: call db_lookup(''start''), then follow each instruction step by + step. Each lookup tells you the next key to use. Compute each step in your head. + State only the final integer."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe62a9c8197914677ea9a2424a0","encrypted_content":"gAAAAABp-r_nDEFD_pLd6MEzoOCG8OKekq-Ev8vnbPTkSie4RovCRsX1uArYsm0yKKyAAEYh8tRzZVKUfeKd6HTYEqGrwUeI9Nd7K14uFr0VC8_AEFOX5_RmizhJInjfMLB5xr26SgIbSkBYpELGYzo4H-nlAJVwn_Tgni-Ka5vNjJZsh4-w5wUXQ3NaK4j0bXJYDwtKFlQ5lLzPa4KgzkWJk60HxK847Z5ZVk7WsH9SKDDPSLEsctmXxY5quJk9XUSGN5kR0TPKGscIxTXtzDPHcudCiAjdRGv0g_TIJlIUQ6ofDXWQl8bdpYoqQIbEy3Khqh3ksKoSG71n0UsUw-6d0IeBMQm04VcGiCUhDHaKa3EhFxh5MNgmuVEHDSp4ByweVwK7G4GniKd8NvkauFyROQ2Omqnj_6MKaWccIaQVm5Yw9WU0_PIPFDOKyse7cms2tlyGpFxeDgTgSARsxLaZEZMIWciZT1DazA9LrrD13aCD4ePHWCRJbpbkDH7FM3T2TrT16-G0kCip_OXm9FZZxJvAgUXMbnVEs1SugWBC2nTCfYrLRwXjyzZk35ekzv6LIy7Zigh0t-_RTOWrBkNxA-7jvgCP_YF51cG1no4hTFEHhhcuwYSmMSMMAkcWi0Dh-FXby7k9tjUGMKTjs1RN9NI8tc2lSwmAEdhZdk_JpEnVlfpDEsjRFRIE6RfghF-SRn2gFAUfRczeHKplmtWM0eDKiaK6XKHTPQats0mz1B16alRJOEeFbwMPQA1Myz7atKb8Z5I_IrlGBh1Ho9buaY_RFfCpGemPcO9eRrGoZ7QLwSQH6pw1MrDusZ_pt4vKYC3qDu3rEZ_6OHwFYnRPVj_Qu4bt9f5arKPTKAp2hmEsYKCDvaWeBp2lgEOb66sv34qH6qs-ooCAOG7rmtjSLCVu8F0R11NiFciwvmlmECOwT9d6emgKjn9T4_2CBaUmo9rdEiNfq-cmhMG7a26jGazsIl9Iz-prux7hV8871iYlNDhn_Mr86kkCmknKL53tGb-29XcK-jTUnlLGrHmhnBl06Cf0f-cE7sT6D2tjacDeh9P6gtu0enBfPbuJH75oA240HiKU7gRBd-kYnw6hvhCMKUkjb88Uj51OFOxB0vDLnF4_6y8rswyWoPVeeuRMWHJto_ersRSiuWnzVjJAo4dK9MLfBKRhIth6WDBu6ReL2575QfzUYEFTGXmGPX33OVw0-mgm","summary":[]},{"type":"function_call","call_id":"call_I670mAzR6AYszdoqaI96qg0k","name":"db_lookup","arguments":"{\"key\": + \"start\"}"},{"type":"function_call_output","call_id":"call_I670mAzR6AYszdoqaI96qg0k","output":"Begin + with the value 7."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe83488819788888aaafb2ab1fd","encrypted_content":"gAAAAABp-r_qaKH8G-A088KPm_AKkVfkzrE_w9jGidMqLh3fXoaw9IbtAS9DwhU_OCnhgjF9tD3K66xmrV4gVvDX5mfjjsF4A8rr9sdYPtMw1TmcwMntBVKvJskKjpjZe6s5dswnzCzuUzuG6UOKpRpoZpnmCIew5Sm-ZoxY0PjtXYj7_YdzkzXmb8M1nbFyymgIloaDarelYr71F-wysy_BIxPAfFulc7CeT_jGY6qTj-XaXEjNevpAkRxQs3qDFsf1vJJJSMRUu5vHFO-kmQUvC1C7nNSkK-AQxF4ghx2I_cGJNSHusnjGq5Ia397H_0CBOQjTd1HJIu3UWG8guMSjH9m2elLMFUpfExvbR4iK-l-ewCcWpIyfWxdLWrD3IWqmQ5Id8nEIJFK78mj4VmLPw8JOPkArncScgXqTK6uzkzEX2HtsNECxFR5w1eVZS1yixE6OSj4V5z-DDlxxx0ToibA1J9qcs5QKR-0ULx_LK_Pgf9AeK8_Z2FqHNnFh5nKVWdyIYXxTpurpF1qEnKBCSCwaQ7XzUJlJ-n1UFgZt0URFbsbqY3XshC4urrvqswPq1PLBqhQcZxPbsXqMKzFuwT6J6CEl2GrUoCjHbVDqIh1j6lc5BDYi7aOAYn7bEwQI_thfmQbi7EraVZMmgbEnvYkxvzeP78ASC4YiYwgIzGuUQKzwqRKKbxpe6oeVE79FCeXhEbKT2Z0kcBZSUomxfSCiLtNMC0Ni9WFfUOu94lRtTPdPH8pVu-QBFezi8GfxOXHNqNuFSVI_VQtenIoP4ia1eYxbI2hpwAlk3u8dUSCvJySdLLw9dPWNQxco51zDamyQVKenbnEbj4ZqZRKl-2t_neG0AcX5qliCqDA4mqmL74yirrmYt69kIFQ6SGCP-86km3amgAInOsOBVSQK--HaOvBUfTc-CWLOONqfj2aXUiMCKvSMvo9-6Fug4CsIXdcgxqOQYGfc5Opce8mTUdrSjmhb1WiMO6lk0LSrHq0IUsIh8Dn9X5bhXshn8dC5wl7Ya7Bofyg2EBGTo4Sc7i-nf2BfFX7kqTmOURGa4iZe8hNpuoytgVDlbjylf0NyU-GE0gwubFgmb6Dqn7UdJedE_i4DKIctupippecmwpgO9V1nilTlYoNYjrGryjTRZ6AEj8oaeR2R3G41wUpMHv6uUyYTIKl1l2cAd6cG0g5A8ECg9bQfIUcAAcQN3EdfoIoOXOEqCM-mCEjjP9wQcx4ztNg0_o_TT6CqZgPKDzzW85UgVDgtWGhU464SPEUoYxJrCpWixqQ_fTANdF4snP3hRG47yauIenFdO31zuafOJrcqEDycKflDHvA5YUOiK1iOoDK4abQhZBePmpWyOFf8lufsULGoetqiLG4VM5hdijIqkIOWidiKSBMxX8X75bAeT5uS3nCq6nS0jRLTKJ7LL9qH60med-9GfzMxKXnMHi0yewgVmQOgwir4rs8o2utjshbrOeDz4-Z-4pCskwZqdEqY0Q==","summary":[]},{"type":"function_call","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","name":"db_lookup","arguments":"{\"key\": + \"7\"}"},{"type":"function_call_output","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","output":"unknown + key"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look + up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '3967' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/31XWbOiyBJ+n19x4jwPE4CsE3EfVAQERRQQ8M4EwVIg+1Yg0NH//aLO2bp7rg8G + lV9lVm4FX3777eXlNQ5e/3x5bUBbOSiBsz4W+iHpe6G7IFGMYkPXC4HLhB7BYCztoQuMoFHCoxkG + xQPq9fe7idJLgA/fzJRFC55yvwEuBIHj3jGMphmUQBkMfWAtdGHX3nX8Mq8yMO97Knmun0ZN2RV3 + v0I3a8FTHGdZXESz7Nu8nAWVO4Lmrh+AHmRlNS9m4Pvz4DeTPxxNPlDQNOVds+iy7CEIG1B3oPBH + pwKFm8FxBtE/nn7GxZsxJwDQjbP2s2ZctLDpfBjPQX+W5+7glB2sOujAMgU/g7AsM8d3s6/m8jIA + 2T2mqIII+QeJ4ChOISiB4IvX9w2Nez/us9rzpFny30dqngn6KG3774X1SIp9FJah0YUbMiS5cFF6 + 4bGP4x5G4FiBZ2ndtizuJXiH5pQ1Y3VPs18WEBSPFoiW99+qQhqn5wVfM48cyiwmUmpril7nMss6 + l7oubmumoBIiPYfmVRFJia2kGwno7owduyYIEtHGbvQq8M6E2IR6OhGaIcYl5fJA3BuNdzx4XHy6 + qUwqWrik2Yt8cbNca7kqSKeWYobcJfSaoyx6kUx1cKw2QDgi5LRbSO1wQFPS6C4eikFy002dMN0c + dHfC9CxGDelGLBo5Wm9qtxH1rl1k1CLAmMrqZPK0X52QxcUZtuy8B9yGJtVdEeNW0QCzAXLkhQiR + AEVOXihghNbXyFFb3NowIq2mYEhZEm8hM0okuadJfDmslk3vxHV7xuxOvixvIz/oqBjvGu5w2zqC + cxztkENQAxHbswA22lpDIiK5cSpntRB3JVU/jFeodyjnVdZoOEbqNTC7tcThcOmaDDHpXd9bmSkp + AmqaCZVXa19szP3UeUJ61ESG9hHB2mhiuQqG4QQsocy5lojqwlvRgV1FyknpgT3tCul0DDg7DWxR + l7Z0tNqcFX51I282Olmnqw3j/uoIhNLtcN72dbVhJr4d6q1hjTtTwhGWVNCk1xy3Ik6o6GSkbTPw + Jvc60UJ7uWB1XJUhOndHxqLU2SpdKJmJyY7hLimwcAvwfGE2xNm6pqaR0z29JAfySBsHL77CZYGQ + yrU02JWM8f5+Yzuoket+Ziz3iMafm92CgyErwNIudyKjGh6hBdWVHVGO8tyzphSragzOGJOxFoya + ThPJCoWsD4bZsWnSl/neY7uEn5eeU5sXZ8VrfCGVNJ3Z6zKWTaCAzT5b2KVJQKvOyzao6kzjK1Fu + 3AGxBbDnsMKNlPrIbwttByJ/o9VbJkOEsZ5LZS3OwmA6iSpIUXJ1bpeledTlfSz70F8iS5WUDpU0 + uOp0ia6nvVAFW3pRHA762UmO7OGcyrIJEx7su6Djr+YOxrF/bU1TqxnrJnGGrJ9G29nZq2FSeX7Y + Xoe0PuK05xSBpO76kdPZpd4rDEnlgybZIJPRTkqGOsEWB/JibtcVWgWGEG92EYoeMdLYruos4449 + jTk7WKROqW2qxjplXhzy7Y1ndOXMe714JS+84ybAUjwt2oetU2uQFbsLh8PRa7XWFus8hpXboF3g + R4rcj7Eqq5brLdJbrNrbxKGL3s/8en0xjCkI9iMZLDEmaaeNb5MKQQHj7CjwIhzo6xG7kFfNbHJV + A7oh8nLE4McwNxkHJB46snWolsPR10PGms4BHbkKlyZauV8jJN97k+NEgKvSk6fUmtqygN+QJ5nW + 69Rf0aqC14qC5HRSDJHu7kNy6vwlaiW4JiWl7nNrRcPxwKnjdCqRZDp0scTR2mUKNZ3oIikLSW9v + CNv4sIUcAZzZ/3i4XkRzwwiiGCR8UWVu221x3bglhib0kdlQ1GZn5tXt6HKjS3kGDp1rIx7Naxq7 + 0JLsKZ7G7SIzhClEKAVzYtAVm03d7Zrq7AbEjtsVtrAlS48/GpLrNRwCjltECuWTLGjZaQUja23B + WJcimiBSqSfPUhv58pgSQ1KCcXkzlzssv6HKfCWtA5B2gSlss5J2zOZc8QrUisgu0iVHj9ZotqXq + +dWqSp1kgds66V4oRQejTiaiY3F+h1Q1lfJsuw1U6jSitSag1mac5tdGlRveDu3L2zQqplrKVtCa + 9nlnLX1iyyMbZ9k4+dTupXTlLI5u5uGtjge5SRupax2cTYKr1ZJ1FROp/TNRky1ksNo7FqYjib0b + o+Ykb/g4rzYhjZYuw1zo5QkMauzH01LoEGwQ9PVlzwh1AG9ds4uFQ8GYBVsmS62gJ2PDO4N+Yw0/ + WbFEcrz5jiqLNrVSVxmQZdXWL+awlG7TDrILWrqByZ8EUQ43XLB3uP2Z2zq9RIlILpFFx1yXkkwZ + IOcQYUOcNaCpc+sKF8uKC9AHOZ4yokIjskgh03X+oo376UYoq/kLuQ6GvhtSPbjCCqHYzGYYH3Fy + ahj7tKm41OBge+RS6pCs7BpQHktAuThe2S3Ll+eDiiEVcVMkhRtkfJms9XZz2U4cEmG+r+3wCwcV + dp231jIEfkac5CkYpagXLS/kJ2x14t0uaLKUZtdkeq17FzF3DR8OaUaNrLFBMHzNq5Cm2arLqhJo + lneiVBO1nfxiVXwPVZFSCK0Z40Vsz99qOExRcA2otRI0VtkfW2gau6lbaTS1sUdePf7ng1a0XZ67 + zZ2E/ffvh+z777/iNaH/77wG+PjCv/MalgnDmc9QIYr7LoszP/OasCse/O3Bxj458S/89AG6TdTl + M9954N/+ek3B+Nfrn3/NSqDC/nr9/rHzbtR5+vt4PMcpm6dKFjNyW2ieeNCubhXkwodG4eYPvwLP + ycoy7arXZw7m/78fbG9+a86WQPaVQ8409EmUq5mDz8QM/ILLzlAfl13rvNH0p2fvPLJq5lDhbNK/ + AmeO6V+xBtzZ3pOFvuLE9UlOPxjiO0kHYVg2D1p4jaPrP0F+KvDd/Dtrb90QwHH26W47jMEXjt6C + po/nqGD8xvpDt8vg6z/DRNmAL+MCBHl1Z8rdQ479kwAIBvjh3Oxa7n6sP/XEY9/n1nvtQeOVbfxI + 52sOgrjLP8aNZyGu5ezfHXY7WL6+A+3PvPzH1vsofgBav4mrt9Tu5gZ46aoX96V3sw68eOPLXJaX + uHiBV/BSddOUgZfAha7ntuCP/9dD79C9e/K5fE37KfBnfeeEzdn9Kp+RZyN8Fn0KYR5/7gX/BH5/ + f/7++4f5+4QVNyB4T8Yn4+/rvz8pvB3wz2z5CXGDIL4nyM3Uzz4/av/bDyc//HuMpvf78cNFgmU1 + pyeaI/fuBtB3YfW4MyzzFDRzld5Grtcgbl0ve5tVu9aNwEdDxcWXgW+B4r//DHwaI799vCXmWxV8 + aKJfeu/HQRLH6F8hvzL8fiU/tDGW+mIdltDNPmASY9/7umu/3sG5b9x7t91P+P7b9/8Bf9t5rz8Q + AAA= + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f752719988393b1-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 04:13:35 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '4711' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=ADJyoHaKUvEeC3nK2_sSJ7dr9CEJ2NNMqc9USLXclT4-1778040810.495933-1.0.1.1-uG8A.s4fFg.igFwTcj1WtTLUqjC4PfZvnuYaIcoE7CBZKb_G.0rMV93DAq7s2AeJruAKwYTEjGzqEdIZMlC3NnwS6wHlmWG9sXbbAroBBCjQg2JoSGoYK1dMCFoUIdsl; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 04:43:35 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_c6ffb3ca01ef49e7b02c5b09f6dacbd1 + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve + this puzzle: call db_lookup(''start''), then follow each instruction step by + step. Each lookup tells you the next key to use. Compute each step in your head. + State only the final integer."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe62a9c8197914677ea9a2424a0","encrypted_content":"gAAAAABp-r_nDEFD_pLd6MEzoOCG8OKekq-Ev8vnbPTkSie4RovCRsX1uArYsm0yKKyAAEYh8tRzZVKUfeKd6HTYEqGrwUeI9Nd7K14uFr0VC8_AEFOX5_RmizhJInjfMLB5xr26SgIbSkBYpELGYzo4H-nlAJVwn_Tgni-Ka5vNjJZsh4-w5wUXQ3NaK4j0bXJYDwtKFlQ5lLzPa4KgzkWJk60HxK847Z5ZVk7WsH9SKDDPSLEsctmXxY5quJk9XUSGN5kR0TPKGscIxTXtzDPHcudCiAjdRGv0g_TIJlIUQ6ofDXWQl8bdpYoqQIbEy3Khqh3ksKoSG71n0UsUw-6d0IeBMQm04VcGiCUhDHaKa3EhFxh5MNgmuVEHDSp4ByweVwK7G4GniKd8NvkauFyROQ2Omqnj_6MKaWccIaQVm5Yw9WU0_PIPFDOKyse7cms2tlyGpFxeDgTgSARsxLaZEZMIWciZT1DazA9LrrD13aCD4ePHWCRJbpbkDH7FM3T2TrT16-G0kCip_OXm9FZZxJvAgUXMbnVEs1SugWBC2nTCfYrLRwXjyzZk35ekzv6LIy7Zigh0t-_RTOWrBkNxA-7jvgCP_YF51cG1no4hTFEHhhcuwYSmMSMMAkcWi0Dh-FXby7k9tjUGMKTjs1RN9NI8tc2lSwmAEdhZdk_JpEnVlfpDEsjRFRIE6RfghF-SRn2gFAUfRczeHKplmtWM0eDKiaK6XKHTPQats0mz1B16alRJOEeFbwMPQA1Myz7atKb8Z5I_IrlGBh1Ho9buaY_RFfCpGemPcO9eRrGoZ7QLwSQH6pw1MrDusZ_pt4vKYC3qDu3rEZ_6OHwFYnRPVj_Qu4bt9f5arKPTKAp2hmEsYKCDvaWeBp2lgEOb66sv34qH6qs-ooCAOG7rmtjSLCVu8F0R11NiFciwvmlmECOwT9d6emgKjn9T4_2CBaUmo9rdEiNfq-cmhMG7a26jGazsIl9Iz-prux7hV8871iYlNDhn_Mr86kkCmknKL53tGb-29XcK-jTUnlLGrHmhnBl06Cf0f-cE7sT6D2tjacDeh9P6gtu0enBfPbuJH75oA240HiKU7gRBd-kYnw6hvhCMKUkjb88Uj51OFOxB0vDLnF4_6y8rswyWoPVeeuRMWHJto_ersRSiuWnzVjJAo4dK9MLfBKRhIth6WDBu6ReL2575QfzUYEFTGXmGPX33OVw0-mgm","summary":[]},{"type":"function_call","call_id":"call_I670mAzR6AYszdoqaI96qg0k","name":"db_lookup","arguments":"{\"key\": + \"start\"}"},{"type":"function_call_output","call_id":"call_I670mAzR6AYszdoqaI96qg0k","output":"Begin + with the value 7."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe83488819788888aaafb2ab1fd","encrypted_content":"gAAAAABp-r_qaKH8G-A088KPm_AKkVfkzrE_w9jGidMqLh3fXoaw9IbtAS9DwhU_OCnhgjF9tD3K66xmrV4gVvDX5mfjjsF4A8rr9sdYPtMw1TmcwMntBVKvJskKjpjZe6s5dswnzCzuUzuG6UOKpRpoZpnmCIew5Sm-ZoxY0PjtXYj7_YdzkzXmb8M1nbFyymgIloaDarelYr71F-wysy_BIxPAfFulc7CeT_jGY6qTj-XaXEjNevpAkRxQs3qDFsf1vJJJSMRUu5vHFO-kmQUvC1C7nNSkK-AQxF4ghx2I_cGJNSHusnjGq5Ia397H_0CBOQjTd1HJIu3UWG8guMSjH9m2elLMFUpfExvbR4iK-l-ewCcWpIyfWxdLWrD3IWqmQ5Id8nEIJFK78mj4VmLPw8JOPkArncScgXqTK6uzkzEX2HtsNECxFR5w1eVZS1yixE6OSj4V5z-DDlxxx0ToibA1J9qcs5QKR-0ULx_LK_Pgf9AeK8_Z2FqHNnFh5nKVWdyIYXxTpurpF1qEnKBCSCwaQ7XzUJlJ-n1UFgZt0URFbsbqY3XshC4urrvqswPq1PLBqhQcZxPbsXqMKzFuwT6J6CEl2GrUoCjHbVDqIh1j6lc5BDYi7aOAYn7bEwQI_thfmQbi7EraVZMmgbEnvYkxvzeP78ASC4YiYwgIzGuUQKzwqRKKbxpe6oeVE79FCeXhEbKT2Z0kcBZSUomxfSCiLtNMC0Ni9WFfUOu94lRtTPdPH8pVu-QBFezi8GfxOXHNqNuFSVI_VQtenIoP4ia1eYxbI2hpwAlk3u8dUSCvJySdLLw9dPWNQxco51zDamyQVKenbnEbj4ZqZRKl-2t_neG0AcX5qliCqDA4mqmL74yirrmYt69kIFQ6SGCP-86km3amgAInOsOBVSQK--HaOvBUfTc-CWLOONqfj2aXUiMCKvSMvo9-6Fug4CsIXdcgxqOQYGfc5Opce8mTUdrSjmhb1WiMO6lk0LSrHq0IUsIh8Dn9X5bhXshn8dC5wl7Ya7Bofyg2EBGTo4Sc7i-nf2BfFX7kqTmOURGa4iZe8hNpuoytgVDlbjylf0NyU-GE0gwubFgmb6Dqn7UdJedE_i4DKIctupippecmwpgO9V1nilTlYoNYjrGryjTRZ6AEj8oaeR2R3G41wUpMHv6uUyYTIKl1l2cAd6cG0g5A8ECg9bQfIUcAAcQN3EdfoIoOXOEqCM-mCEjjP9wQcx4ztNg0_o_TT6CqZgPKDzzW85UgVDgtWGhU464SPEUoYxJrCpWixqQ_fTANdF4snP3hRG47yauIenFdO31zuafOJrcqEDycKflDHvA5YUOiK1iOoDK4abQhZBePmpWyOFf8lufsULGoetqiLG4VM5hdijIqkIOWidiKSBMxX8X75bAeT5uS3nCq6nS0jRLTKJ7LL9qH60med-9GfzMxKXnMHi0yewgVmQOgwir4rs8o2utjshbrOeDz4-Z-4pCskwZqdEqY0Q==","summary":[]},{"type":"function_call","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","name":"db_lookup","arguments":"{\"key\": + \"7\"}"},{"type":"function_call_output","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","output":"unknown + key"},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfeb569481978703af8553a073b9","encrypted_content":"gAAAAABp-r_vFGcSWQD083z5Jsq67CmK99_ZqqnwC8n6j4kVfWhNH5J9pJw5e7uV1QurddjHY1w7BdbV4HrfTkz4SUHio6aFeHMUrbQObDiRwP8kHX2JSY3m3wXaXABn5_qJi85Lj7CD6X73jzqdQpEeGQ-5zL3JsxO0k5UuZb01t5EuzuGzw_0LR1Tli0UJw43rKgCEqarHTus3l63d18pXuK5RMBR-3Z_xI93rKewxrkTaH1DBgxtlxtD5Z4f-d0-RbfG14Svq-QS3wsfg5Xrn85KJHwf8yJ55M752AxBArv_iqsV1YuKZAwyFxT0HiLrDOwI_G_QyYfD-0U-HsVGeESCS-g4jwDPDXst2aJPTOyhtTu0DbpXyU_Ukbrtlws4OOZurl-W7LvvXlWJNG0WWj6mpCcHrWMzubGkQSH87c-GXESHoBdxxReXGomDs4gqnbB7dYpgNRNveYzLnJRQdDYkdYHTJI7gBEVNFBw5wY0zXRhYtivh_G4NuL2FYcTPr8zFsxqIUXyLWJ2-95N0jvS_ap4R0H_l5YY8twKvT4stYA39T2PKt067Cl906VXoatJWjW9yfLjn1fIe2m3Wr4VXhkWUm7v7A5x5Q7UObihtAn-5NhoU9BK1FcMEY_0UmTclUAM-SFVrL3Dtf9GtoYoLH8PUb4Sdph9y0D6baVSNnBpydV18l9XtgruSH5p0t9cexYA3zzTAmMb9ujFxYAb_qWZ_BFSFnJo77lYCoiKWeNeEMl3YoW4tXqmosdpqlSFpHKrax-YGeMD1nagNqQFInSLegcESqI8l-Gyqws4X3VGxW_jPGJgjh_wZAWQTKMiKctcA-AP5JOpJxaPzZghRMGpdI73nOOTV_jQ9OVkKKWtjFeMuduFhWLtiichsWWSq8XwJDUKTRyY_LYBxzPFFxIhxkqQ27b_ndJPLvyDT9ATvN856mxSJYelK0uJjxqj13O5ZWICp0pdUGiELg00Q15UIBqllDQv71_Ltnk_oSEprXRlbifFswF8TNVFbvHh5ZF_ajeXNbSgMfs_qSt9HuZD2tybsSsYHqmitpar0udcgNKvyiPKPXab3kwiPYIj_7nvclcqCZUUzddMy5dA18jszEcY5N46eUV_NtZGO7hQ1Z5hSWrmPSeTUHFKg82QfmW8_ejb0y9qfPoxQcTf8XzVd7gaNDkjSoMC-5Fvbz__geDpkRbNqSPs9eFE5RK7TqkcB7PN2qNN-m7jnxgTaMf5zucA0Xj2SJjoTcDCNS22d_qikzo-jzOuiJD7SZzfST4ugJlf5bMUGIiOItD4e_jszixhZHWE8GHHdjFnplasuI2TUwjUSGvgWr66ELWmpwQaDya6bU2t_hrHQWhkiatXJYzizyI3lUGzf-6N1_ieunEEquLrpVad4LDLnYGI5obFQUJabrD-eQI-JfKRKGSlRBtgXCXtiTJg744kJv5VJsgcKyk4xjoeyAwWAL1mw0NKt0XOeJLdWGIlo7_WrVpFNtSngYnkAD7yXyWsoPbcpBpk_j32YT5aZ6NTeyT5jH_XDcu-pq6kF9sIdP6Ry0qSG0XEyz0zXpmUbL0vowzyNWPoKXdsWYVLXAc4IF-E_Ar_mzsMJkB_3Qalb2sT2dmW7UkaXO_Ej2PpA9aNW-qcV4q5st81qbQnW_JHvai0WzKEFimpEf70oa88Z7ARexPicizAGu-1xGTCZM8GqdtwurLiGOn8Wn9ojASn7zUEF_xTw9UcjB94jQwc_PKHY6BPBleKKPYTZWxAJwzLt937JwezczGHKfEDdM_DMVDI_vJ6H-mJ5nu8hAJK6UemD-GE4VSeSP_NtGZXXinevdm2k8HN7-KH6-zhwP8yMzw4NBnwCCdxvuxkTdhtp-69lY88c-_m6xyvkrpDkUDtsQDk6OjBYqe6b94tKnQh9I9FoVOP1-p4wNJNDxK2AjCTsEZIzD-g1ccSL2ZDtN9CmsXAfecl4RKzdyJgvHXbfFz1BRFaudrlk79C5khqva-WLrFfxkl6y9UE-12CFPt779pulpoeSXbR6PW0Y_mZXpFvtPH6N4Sryi3iYObDtxzgdhd6CNdrXovQstWULzuBS76EYyFPQ=","summary":[]},{"type":"function_call","call_id":"call_Vik9mkNli8KsnSbHOShapdmG","name":"db_lookup","arguments":"{\"key\": + \"step1\"}"},{"type":"function_call_output","call_id":"call_Vik9mkNli8KsnSbHOShapdmG","output":"unknown + key"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look + up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '6407' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/7VXWY+byhJ+z6+w/BxH7MuR7oONd7BjG29wEqGmafbN7Caa/34BZ2xPltHVlWYe + RqaKqq7+auGrH596vb5j9P/p9ROUxhpGETzETWjSUDcBSWM4w5tAN5HJMDjG4TwLOMqkSBIwDNJx + CsL+59ZFpLsIZq9uojBFNzlMEMiQoYFWh7Msh1GNF7rTpRnI8rS1gVEQ+6h572akA+hZSZSHbVwm + 8FN0Ezu+74RWI/vRPDaCGFxR0tobqEB+FDcPjeLldvCry1+OZjstSpKotQxz3+8EZoIuOQrhVYtR + CPzs2iixL1inc8JXZ5qBMuD46bOlE6ZZksPMaS79LA9ApUV5FueZlkUe+l2ZRZGvQeC/dRdEBvLb + O1lxNqC/0AMCI5gBRg0Isn9/IQHtcc9mt5Mayb8dNDeAHqlN30msybG3xPIkADrCCJo1MAbQ3XGd + k+wao1tqQRqFbQruqgay5Bq3MMMozFDYlYA1bP9G8SDRqmCIHSplk5W25lKYvwLbQBAuziWdNDmj + vXg1on0RwAk6bVezwep4Hg/jhLVGMuPO1dV0PoiTNXvNRTgKx5zmE9VptZueRkeQVIPSkfFJ8HW9 + s5ZRKa+Xu5QUrQ03xwJ5dZlqijG18+mR2YOpcbamJm2iyW5EoulyI2yDFbPEw+RgR5PhCtF4eE2C + mObkmosJcWhuZ6Gqzom5V1nn0UUAZcYsM5uChSmnx42lXtyRGJKqlpbw6gWKEegbejY6wYXnRNHc + kJfqWYsDGRvUvspvKn2yGB/WsrQnYnG8Z3CqRvlFGcCcjmrdY1VKOCx5t9rUB+WcmwUQ1l6lMMyU + 5mCoZdh5W8acJx2yo30aDddidNnALbaXY++g6xeQz3OZzm3PmB3kwtIz63w4qfYQDGOg18l0NKOk + 6VlCcyxW1qeKitNks6cPXLCeHpWcXR4UHJVHiXd1e5f6Kn1ZDeURWU0p5utYwdwolFYckgZT+QLx + C1Geh5qxGe1ktNZ4MjxuvWNQm2LuKIEYFVitnA6B6IhSgfnzen0dOrCYhYUzPJMGPpsnTq5FohCv + dbkSJNbRgOBV+m6QoEKZzpOgWFUlT9hcAt1FMtsdancoT7jscNgPHaygLz7NHufBaO4UFEXklOib + 9jkWJ/MQSu5xuCk346OtZzhb4yJuX/e26ij78AICT+FOpDjYLCR5qs2AbJJHfOCWy9FmNxLgboFd + pXx+srV9GlkupSNF59fFhrC8oqykpWdNdqY0XM3MAbnFF6sa5ppURO5lRwrEAZKSRM9qM9Uoa2Kp + B9/kLtXBsOcInKpgyw3m/rYQ9v7MHawLd5IzYYIUcgJlLJZ0Fk0FPjB4b7rNFJUe1y7ciOaZZvF0 + eD3EWlAHy8lkKm3nllidwvJUjq76/CCpZooBIO7H2KR2YP7VqFUPKSZeHQTXdD3JH9i7VVktzDW1 + UIxtUMbCUZgGPlosBmKd5Bk/3qmhBxxv5wzT83aO02pQUu5xcXHxGb8/sV45qqvYTN1MbSbC5Dw4 + uXt/d6UENr5iuzGbqNnpxEjjNVAxvlj4yn8eYyHNgwAk7RD993sne/n8p7lkwr/OJROnWZ5r5xJP + cibOAQ4ShgkgwH+fS2YedvO3m6ZPQfzl+9IpQWLlQTOvOv2Pb30PXb/1//nW15HlhN/6L483W6fa + Ld7u50k/lWJJLZnJecvKIU+aVpCriHxYhCDo4jJ0zY8iL4/7/z8GALYY6BgweArAVtx8PumPxaAA + fo7exQBlpJ2uIe7ErmDmo6sfDMuLN7l+DAY61WEAyLYMaBLqBKnDD8Ygs1Gvw6HHvovEkkrijYxt + JzUlLIz0OBMn4ozxPqga9K4aeJpjSZMmWd6EkGbMj0Vi1HZEr3Qyu/cEypd3UbEMAioKg4veQpCt + EXNZG6wqMMzHoAK7+uChTnK0qeuESeE8gX0sKo1RkhHvghAuQnEYGOpXXYkvkPy6I09CA8n/UBrN + /+8drYtB0nhC/luy2PDNGyOOG7LdMDD0B9LaqAonylPtlY/fIrsTxjhp7po1LqGNtOZSf9UlqKV1 + N7rZJyj7xkIfVPDOxhsmGSUd/7Mdy/55yacvQev+Ts9TYKLs2sTU+jYd9IaMpygpnOZWmfNK702Q + +1n/59YQJejNXpChIG4pcd7J8Z8AZKjKHsE1oQXg8fxUFN17z7XXL1CiR6nTwdkPkOHkwWOvuCXC + jpr4WjXIs6h/V6S/E/Bfa++RfAOlMHHiV2ilpgB6edwDPztMv/aatPSa1mu7Ls7r2kc9A2RAByn6 + 8l4N3VVt9QRN+pL06eK3/DaANei+lTeaWyE8i56u0Ow5bcKflC/33y+fH+7bVcpJkHEH48n5/fn7 + k8HrAT+XyCcNMAynBQj4m+eYu9x/+uXkLr5uB23745dGyqK4gcdqbq63DrC7MO56hudugqTJ0utu + 1TecFOj+61Kap8BCj4JywjebHU0Sn39XPO2LPx5Toukq42GJvam9XzdGHOf/pPmT43tLPlmzb5xn + UQb8h5ah8XtZ5+nbFmzKBrTF1h7w8unlvzbc0z8nEAAA + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f752737fd3c0044-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 04:13:37 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '2162' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=_EXrqe7IjNWn6uSESYNRV6be_iYv5vfdiSH8V3vdNgs-1778040815.3529754-1.0.1.1-qv4Kk81buc06G3o0AUYL254nHY_tKoM.r4ToKt1atzCBnU7vDOrMSgZt87H9a__SsX0twv_Dqbwnv35sedBqkSVJ5tcTI3rPQI4rOXZ2o7bFUr43uDG7zWB_uR4Rv0xw; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 04:43:37 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_83b594dde5454daa8a186b3a26e8df84 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_openai_responses/test_responses_round_trips_encrypted_reasoning.yaml b/tests/cassettes/test_openai_responses/test_responses_round_trips_encrypted_reasoning.yaml new file mode 100644 index 000000000..023c034f1 --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_round_trips_encrypted_reasoning.yaml @@ -0,0 +1,356 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Pick + a clever country name, look up its population, then check whether it can have + dragons. Be brief."}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"lookup_population","description":"Returns + the current population of the specified fictional country.","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}},{"type":"function","name":"can_have_dragons","description":"Returns + True if the specified population can have dragons.","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '722' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/71XWZOiyhJ+n1/R4fNwApD1vLmhICIqrmcmiAIKRLZiKRUm+r9fwG61e3oi7hJx + fTAks3Kp/NL8kl/fXl46vtP5+6WTwRyZpMsIpGjRPMORXZZySIoTXWDRFO9YXUagRM4SBbLbpSyW + Z1yGtbnO98ZFYp2gXby7SeIc3uR2BkEBHRM0Oorna1ue5qlWlxegwHljYycRCmF97mZkATvwsgTH + TV4uCHN4E/th6MdeLftVP9YCBEqYNfYOPMMwQfVDrXi9BX53+Sk002phliWNZYzDsBW4GUwxjO3S + RDAGYVHWSvIvstX58bsz04EF8MP82dKP8yLDduHXl36WR+BqJrhAuDCLJIC/K4skCU0bhB/dRYkD + w+ZOHioI9i+WoEmaI0iGoLud+4EMNOGezW6Rask/bWluBXpAm/8ZWEHkHbsBVuRIKHAC6Vo8DQTh + hkXrpCgRvEEL8iRuILir6pJlJWrKbCdxAeO2Bbxe8+kjIpOBBrltEBuzWB8YW06R9+t5ni0sbX02 + gpEZbcg1QKP1hdFJlkulrrgBVzQ4GWMJDa04PRwZbFu5gayRvL86Z4IgWG8ZBZUHT0LXYUdld1EE + qrQnSo4ieR0rZ6wOh/mih1fq3jP8rjII0zxBWlAxhaAMVuVAFzyDG9mZWAWrfDTTJolSTg8CvZwY + ispr5arAqevK/mouKT6xEbeStcgnV2QtSWuUrOH4pKygI3CcaOpbKVIFo/R6R0nDVzI7L3ejSCMT + cb2wgZkJfkokRmH39+RMt/gySgvqgCfunqeWWo514miJs3nXlLtyts1ExdDYfLmQVYK9+Ny+HCn0 + DBnVGYPteoO7YD4OxghWrBsEG2dJc0FxusrustiQZ6WSNgItCodetNEjOln23TzzTAvtmaWo2Zy6 + mSJigLfm8mrsaLHf3RCTqZoMp9HZj7dLyByG4s53Np7pLqI9POdaJqiL7kTDO1kdokWyE2Lblvub + FR6a9my4WJUWkwuUvhghJw4de7qiMNCwtpH2/aB/1aeYy6mxxpKJdLog3fLMgKTigb5zLrsZmSpL + ja+jKMZQVK14zc7OkDxq+lwxB2Sw5emUkg/eKZ76cRisnD2pDFfkvqf0qjklnibqfOcUVUAX/Q07 + 73Nj06MDow8inouW2mYwG1aD3Yq4CvZoA9lpNOyFmryohHBb9WVH2gUkWfbpmW1soozmablUgqWm + pjTAWJLmYbDexfR6xubdPvLn+X4Es52Ap5fJsOfLe9mE3rXq2ds5F7NuNZISdLACZ76YHqzrwoWX + /gEcJ0MmWJ8YYcLBKEkm5algRHarbk0C9wqXmCribNdjexsc9qsNbywoiSt3l9UIYEtaoQpNQaFH + PdHr9tM4To18wkCR7LOKTKuXPOPZ9TCaKWhywaPpXBHOVSz6UARzW3Tko3GaHt3JKPFywvL7Vjo1 + lHhZDbB2PBxT54DFrehmg+mxIhmtOqPo4DuiOMp54cRN5olzFgZ6PAvKmDXjA9EbTqTzAfqAzRNp + oxKMsL+sp0XFBCk3JSR5jC7d5YUcRr1uFYKRM+aOy3kGrEDXS0/qn2bEXPZFchGd8DUsNoeejPJq + e1pn6Ulxk5gliMw38DwS0xOp48swJR01NQp/tIl5wmGAkgnmVFxkzhipO3xSK10NUDaGe2kMpGPB + cpQ2WZbcmgtjIXaP3bMvkAfE296unpj97nAUT2RtRB/Y6JypYhBg+xJoFwsqRDqX9cE+jHS+2oVD + 0Ys4czO7hhREU1Zbn8qBHYmz8XhGq7RCQdeezi6LvqtpHEJHTE3plXdxPVJCepTq3o4I+2eKTmfO + cjCbBnNjT/bUuSyfWE0Xd9UBu+yWG85PScmp14Ww6suDbjq8rIEgY1afCqMg8LRgi6Gh6BUy9scT + WKZ4b/aFx6jNcRSBrCGmf362stfvX8161/7zrAddxiVbEqcYm7YcngI0bTsc+H3WuzhuOa1lqKck + /sDZrRJkHo5qDmj1v37UR3BcZOWPzt8/OnrN5kkGfnReH+cb1+Yt6/YnLvnC1TZJIGu8dpIkLuDn + hVo+hY9B1GYXJkmAkYkShMMbF94qUn//bPkQgaz2CMOPLFsT9W2VQPWWUlMX/ILta9XZT3Buvi8y + twzvTIuy+uJF7dI+QjOA5R91GWz48MbTHZo53uj7waH3NQa6bpK1xHn0vePbZZ/gbtzf95ocuLAo + 65wa364PP2wxOczOfn2rwn/fi1yAw6Lztm4lGfywUBUwQs0ugVs59VaAAl6LR3J1ahF4PD91SHvu + uRE7Z5hZSe635exE0PFx9FjIbkAckzq/Rg1wkXTuivz3zeVzIz6awIG5nfnovbRLWF8gzl+KI3yx + cZbVhXl5NMZL4raaHEG7qZfz4vqtPxC+vPXnX/9Of92PNJ0V1dBm+VNRbtjXxawr/1HerqJtmE/i + pyvWC2TTEE/K1/vv1++PEM2O6mfQuRfrU4C77OeT0XuQtw39SQMcx79VQn/Ove2Pb5+itzm2C37z + H/py/PzHgBm1pxf/MzxP0NkgfjmCM3xxMuDVf8UvgKqPmM0R8+3I/4LTE9p/hMqvl1zv9p7x32H1 + eWT9/+C6z8YiQWaYeHUhrMYBeReidgyKwk2Q1Ti+V6Pj+DmwwvcXNJwDDz5mhB9/eMsRme+/y59e + nX495n89J52HIflhmnx+eRLYrxRf+b3P2Icx1/3gu0gKED60FC/e5xTOP87UuoeAAwrQBHj99vov + v/3TYzEPAAA= + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74d0b45806549d-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 03:14:34 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '2715' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=JaAjlN.SsTHssMfEA38GX9HitAbY0eJ2RFPOYXHE.dM-1778037271.7367537-1.0.1.1-ITxNEO9g5Dnpe0xysdsQx6Pb4UfPCLUH47.8gv2jcppI9YSzDXqxNV1wVH1S0DOjPhkjvEoameT18PZ3jubWSiQeRsWVcFTBf8rKD.riIQ5F_txQnQ76olVhtIu0ZumI; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:44:34 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_83a358e390da43f0af03fb3186d6e121 + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Pick + a clever country name, look up its population, then check whether it can have + dragons. Be brief."},{"type":"reasoning","id":"rs_0f4809b27460351d0169fab21897dc8196960e8680fb72a88d","encrypted_content":"gAAAAABp-rIaNe6WknTMnPCTW6JIYUOsrQbNUvTkE_mV0UapEUw4P056qF39VaxpCjTGFpDbnqZh4ucbsTpbEIYxdv---5gRmkzgej83d5Ey3QtkLFY-y6107PuJvuLDDsQAuSLYgTi3JClqsopNkz4t8JCSyCP8gT6Ecr9zkSsEMNHoJyKZ82RHTJL7NyStuqffIiSOFJi-V9WFbQsHxpbR0bEoUeGjJSed8669_PWFmL8TygAhFNux0rvRXEmN0o9UQca_r8iq-oTtcBY0MPb7ymqt1ZuHfY71RNsuP-hb9MO3_I3IrWr9JTN5sRQIL-5wi6YyEJ2MpTzvuaWUVu3aOGkGpez5fkkVdR26ktjxIfRtV0vJzFV8298ZAmVPm2oRBfsrg_bpY4R9Nc6LVKp-CuW_RxTX29B3V-HKLoDKmvinWRe4ZD9XidVg_fQmYevsNr8LQ3HNuXILDpQoX8nccIBVSuD_cMDQSyb4s81PQEpdnldcKS1uaNuNVFYBkBxPKu6s1GN50oFjwpPbg_k01nCPXdwXM0qJRN7evsJTD9LbnU5Mve0hNPOJ_C0kW72q1IZgjnKinlkSdY0JDS0YAJAzO19jHLOXdtzk2tBV5OB6G_g2kTBam76mRNVCMDzCXS-x8cEVe5KmDAlNIQz8lWzBIdFXk00yB2McTVmr272IyJkRNLq2auuFFOlkUXn2UM5s3BpiOsYEerX8uKwHDAiIYI_egxzAcWO6n5fzEFopZbkdOQKZbxQfewBZahHD4kUj48H6emooHyjt495WLW_-uAtf-KJ9MXA5AVulBzV7TQ1F6yXwSEaubFSpzpKatPmA9g3BqnnqTsH4e90B5JI2Lwsr75UDmMJpHwuEKOJ8vzn9ie9aOc9dIhTjKhfHEogs-biBbqKTJnRzCuNhZhqdZu9W9frCKhz04NzvpmZid99Es78j6HOodv8CPnMkyn5_nZ-ADHFvZeia5soFVL-48YwUKtz4kq6K-FIGpw3Rw0DmA3zlaEdG6hROrabkPPygFBjM-OIi90QmjuxltVZAIpszWjUrqjJfon5--riTuOm9qj0PuwDq0dLqTtiEVn7-d4aJr8_K9QrdGpLXujLzPLkprGeYFGaFht561NHRy6U6ln8nfh3vi80Zp7cgX4-2B3DEnHINE2Z5mvrL9kkucwkNwbeJ-qOIPCYlmP7zXlD9gm6_VMxl1epK5NUjyCcm9MGGM2L2J1efcKMwQBfNN6pphu1K2Sgwfg0FpPmqPgX-lBv12qMdRCMKkOTY0ALOIIj5NP9XzZuf5W6DOjoy6LxQ8SBIC3qDwUa8Iu5PK8EkkgNkWueTJPzpTYhjaRquY_B8","summary":[]},{"type":"function_call","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","name":"lookup_population","arguments":"{\"country\": + \"Pundora\"}"},{"type":"function_call_output","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","output":"123124"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"lookup_population","description":"Returns + the current population of the specified fictional country.","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}},{"type":"function","name":"can_have_dragons","description":"Returns + True if the specified population can have dragons.","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2423' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/71VS4/jNgy+z68wfJ4Z2M7Te+t5L9ttgW7bXRiyRTuayJKqR7rBIP+9lBS/MhOg + 2AK9BDE/8SPFj6ReH5IkZTT9kKQajKqydr3PyrrYrbfZapPTLN+WLamLnLTbItvn5bZe53S/o7SE + ksKmLdNHTyHrF2jsQCOFgWhvNBALtCIey3e7fbbaIXnAjCXWGe/TyF5xwHPRqSbNsdPSCZ9XS7iB + aGacM9Gh7RU/0aDIGbT3p3ACLhV+IHCJgQfKm9CbgILW0nsKx3kwtBr+ciCac6VAEG7PCGbPWcCY + GMgqCpYwbuaeTBirXWMZXnpu78n3SjqrnK2sPMJb0ErJq4bwJV0vKXB/p07Zp83z5qnIiu1Ttn4q + Vul4QBMfbu4WI6Hlz1CaWKBR2ra5L2zdwH4fhM3qVVm3BdB8vVuvorCBxJ4VBBonwj1D1hN8T8cA + Et25HoQN+OvXVEnleEj/a/ohL1Z5sb5Mpz1xFXMOf1/+/n1/5Py3n4yQv3yxL1/++Hnz8fOWTR6C + 9BCPi+pATiiRJp2XIhy44O+3UCBFNBICX5YdlYu9pbBtUX54R36ETkw6Uw2dHRMcS6803toiZXOA + 6gjnu5jGyoircGmxPkQ9cUCMFIu+hraVOgzTgXWH611T4/qe6IF+bHRDWrBnzMlztwwWbW1Anxje + yrJhUFriuE2v8yc1LCbMQq98c7lgz68FsPDdTslhaj2ZvmftEc7Fsl9TPoGupWGhnGkPlLl+mtAo + xEFifh4mzsp0BMzbVr7twqkHKJhGMzWU9jPgBYRJ7AGSxmmNhUmmtktkGxCjoPH1oknLAh/hSYM7 + x+rz89v24lIenaommumI76wepdVmVpSoPRYTK7+0h90UwtyYZ1fEjeIbYgZexv+XxymEX1pMAx2L + dRNgtH2bOQ1Brit7hhBKWazEp3nuoT8ebqKHHMPG9zM0V/3HBfsVmRJ2K89MOhzyxA95ch3y53+x + B/6DTjO170rFhIUuPjw/ptUsyP8s17gbrVQVlx0WovYE2WhUYQ2W+2jQqONQjZQyQ2o+vNjOkA6m + HcHE4tnLy+LxLTB7TF+n/Y+Lkk6e2WKd3D6nRf4e8B7vuGTvUVtpCZ8zr8Y95cxyp2IPEUos8fyX + h8s/3J7RqkIJAAA= + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74d0c81c4d2b4b-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 03:14:36 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '1077' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=Ff7ZRSpcLAJPDQ.asF9RF4bqWtg97VpC0E51yPz81.U-1778037274.8993971-1.0.1.1-lFBHWH1sK41g3Of_MJ292u9sow8m9djH6vBQSla3pNCvWDtdVwVDdW.ko4QJQaLNA6RhV21md8copjjSWmsVnZJzyKsQgS.IPimoDzj3d9Lt6eQuV2eXHZenOh25fKxo; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:44:36 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_477366fddd32402daba00618a4c16efd + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Pick + a clever country name, look up its population, then check whether it can have + dragons. Be brief."},{"type":"reasoning","id":"rs_0f4809b27460351d0169fab21897dc8196960e8680fb72a88d","encrypted_content":"gAAAAABp-rIaNe6WknTMnPCTW6JIYUOsrQbNUvTkE_mV0UapEUw4P056qF39VaxpCjTGFpDbnqZh4ucbsTpbEIYxdv---5gRmkzgej83d5Ey3QtkLFY-y6107PuJvuLDDsQAuSLYgTi3JClqsopNkz4t8JCSyCP8gT6Ecr9zkSsEMNHoJyKZ82RHTJL7NyStuqffIiSOFJi-V9WFbQsHxpbR0bEoUeGjJSed8669_PWFmL8TygAhFNux0rvRXEmN0o9UQca_r8iq-oTtcBY0MPb7ymqt1ZuHfY71RNsuP-hb9MO3_I3IrWr9JTN5sRQIL-5wi6YyEJ2MpTzvuaWUVu3aOGkGpez5fkkVdR26ktjxIfRtV0vJzFV8298ZAmVPm2oRBfsrg_bpY4R9Nc6LVKp-CuW_RxTX29B3V-HKLoDKmvinWRe4ZD9XidVg_fQmYevsNr8LQ3HNuXILDpQoX8nccIBVSuD_cMDQSyb4s81PQEpdnldcKS1uaNuNVFYBkBxPKu6s1GN50oFjwpPbg_k01nCPXdwXM0qJRN7evsJTD9LbnU5Mve0hNPOJ_C0kW72q1IZgjnKinlkSdY0JDS0YAJAzO19jHLOXdtzk2tBV5OB6G_g2kTBam76mRNVCMDzCXS-x8cEVe5KmDAlNIQz8lWzBIdFXk00yB2McTVmr272IyJkRNLq2auuFFOlkUXn2UM5s3BpiOsYEerX8uKwHDAiIYI_egxzAcWO6n5fzEFopZbkdOQKZbxQfewBZahHD4kUj48H6emooHyjt495WLW_-uAtf-KJ9MXA5AVulBzV7TQ1F6yXwSEaubFSpzpKatPmA9g3BqnnqTsH4e90B5JI2Lwsr75UDmMJpHwuEKOJ8vzn9ie9aOc9dIhTjKhfHEogs-biBbqKTJnRzCuNhZhqdZu9W9frCKhz04NzvpmZid99Es78j6HOodv8CPnMkyn5_nZ-ADHFvZeia5soFVL-48YwUKtz4kq6K-FIGpw3Rw0DmA3zlaEdG6hROrabkPPygFBjM-OIi90QmjuxltVZAIpszWjUrqjJfon5--riTuOm9qj0PuwDq0dLqTtiEVn7-d4aJr8_K9QrdGpLXujLzPLkprGeYFGaFht561NHRy6U6ln8nfh3vi80Zp7cgX4-2B3DEnHINE2Z5mvrL9kkucwkNwbeJ-qOIPCYlmP7zXlD9gm6_VMxl1epK5NUjyCcm9MGGM2L2J1efcKMwQBfNN6pphu1K2Sgwfg0FpPmqPgX-lBv12qMdRCMKkOTY0ALOIIj5NP9XzZuf5W6DOjoy6LxQ8SBIC3qDwUa8Iu5PK8EkkgNkWueTJPzpTYhjaRquY_B8","summary":[]},{"type":"function_call","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","name":"lookup_population","arguments":"{\"country\": + \"Pundora\"}"},{"type":"function_call_output","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","output":"123124"},{"type":"function_call","call_id":"call_jwY8kllWAsnoSXtjXZQ5KR6i","name":"can_have_dragons","arguments":"{\"population\": + 123124}"},{"type":"function_call_output","call_id":"call_jwY8kllWAsnoSXtjXZQ5KR6i","output":"true"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"lookup_population","description":"Returns + the current population of the specified fictional country.","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}},{"type":"function","name":"can_have_dragons","description":"Returns + True if the specified population can have dragons.","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2645' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/71Wy47bOBC8z1cQOs8MJFm27P2KINhbNhAosWVzhyIZPpwYgf89TcqSKHtmDwmw + N6uLrCa7qpv++URIxln2F8kMWN3kfbXPD21ZV7t8sy1YXuwOPW3LoisZq/bFYXfYtvuS9mULXdHW + fZ49BwrV/gudm2iUtDDGOwPUAWtowIq63uebuqx3EbOOOm/Dnk4NWgCuGze1tHs7GuVlOFdPhYUx + zIXg8oixn/iJAU0vYMJ+BmcQSuMHAtcx8UR5l7qOKBijwk7phYiB3sA3D7K7NBokFe6CYP6aR4zL + iaxh4CgXNt3JpXXGd47jpdP4QH80yjvtXePUGzyCTinRdFSs6QbFQIQ7HbV72b5uX8q83L3k1Uu5 + yeYFhoZ06bYxE0a+xNKMBZqlHezxP5Rl9SYPyrZ1Ve9hAzsGHc1bGvNFFnfREHnAWnqEBfhIwgh2 + SjqQy6HSg61opzLBDzfvjguolMrRqbRfvq5AoY7aqPYdJBIh7yd0kDKUnKgllGilvYhkRPWkKDfP + RVmRf3yZFxWhkpEL2GfCHemoxC1nIMzQI6Z+zWby6+3XnC/TSB4v0XP0TUOl/Y42nGGjRESptRyL + hdV4mngiB3rYoAVArO2Ajho9r7Gd0Jbwji0ROnPlbTN1XBPFni2BxRm0Q8ruBM0bXD7EDASZRkNl + ZXUafYaNa5Vc9Rv0vTKxsid+PN0umVk/DNRM9HMDWtqDu+CZAnfPYdVuFsyZ460cnxq4p16M6qOp + lIFV5zsYdDC9j/HiVoCbzLfD4dEGunwn9orrxrLfjnwG0yrLYznR1Iz7YZkcoxAnheeLynmnshmw + jy02pem9jFNgEZ+B7QzXU2k/A15AWuJOQDpvDBbmzpQBsRq6UC9Geh75qCAdzkJnLq8LtaRDTCqU + evO6WWiWJcFZA0prbFKUUXssJlZ+HY8NG9PchZMr4qQLhkjA69IaSwtmYZhyA2zV+kmCOZb07TIN + xqckQShjfKzEp/Ts0R9Pd9njGeNLFHooVf33BfsbmQi/lyeR7nFiPAiFS5qwpLkt+ROdErU/lIrj + 7D2OD+LvaZUk+Z/lmmejU7pJpnw+B3Ucg4f9GDCo41SNjHFLWzH9k/DxuZpnBJer57gs6+dHIHnk + Z8vEQcmWnflqnNw/82X1HvAe7zxkP6J2+P6JhHlbzHPK2/VMRQ9RRh0N/Nen6y8ML4eS2gkAAA== + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74d0cfcf8b2b4b-ORD + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 03:14:37 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '1306' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=QSHgROUcoFXh6zyyvGCnOLGXX2c4b7S8nJowB4RJhtI-1778037276.1270735-1.0.1.1-m3I6gMTwPPLlpeHgduEBRKmVIQGDDZcoItTTaVkwIUQXMOTcCG5_fl8MQIwjyawnDJPGxUthuGFfjTpsPX1OSBOXGmAUziyOhJ04qWwyUzM7fMtNEWtTmOgvh4au55D0; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:44:37 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_e20848e56750417e9dd0940f7c4fc82e + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_openai_responses/test_responses_tool_use.yaml b/tests/cassettes/test_openai_responses/test_responses_tool_use.yaml new file mode 100644 index 000000000..4cd3af59b --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_tool_use.yaml @@ -0,0 +1,216 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What + is 1231 * 2331? Use the multiply tool."}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false,"tools":[{"type":"function","name":"multiply","description":"Multiply + two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '400' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/5VVS0/bQBC+8yuQz4BsB8cO115aaKVKPaGCrLF3HBbW3u0+gAjlv3d2Hb8CVGoO + UXa+nW/n8c3k7eT0NOIsujqNNBpVxlkdJyxjNVaX66TO4mS9aaBKqirDOi6SzbrKcVU0RRrnAJgy + Fp15Clk9Ym0HGtkZ7O21RrDISvBYkudFvMqTfBMwY8E6431q2SqBdK93qqB+2mrpOh9XA8Jgb+ZC + 8G5Ltjc6kkHBDrX3Z/iMQio6ELDvHx4ol08XcUBRa+k9OydEMDQa/zjs6l2psANhdwTGF/1l3g1k + JUMLXJi5J++M1a62nJKe21t4LaWzytnSyid8D1opRVmDWNK1kqHwOW2VPc8usvM0Ttfn8eV5uorG + Cxr8c3O3/iWy/A6l6Qs0trapP29svYY0NBaKJt9keZauWYLNZhOeCyR2pzDQuC7kGaKe4M/6GEDQ + W9diZwP+dhfBXXSVpKvk7C6q6Ge6WiX76bYnLvuYw8/uuv1yk/+6xhU8sttbJb5VX+Ob7ffJo4M2 + xNY6YbkSuygAe/q+D4VRoIkIxbLc1LFeU4rkSm3HD9pO0DOXzpSDovvAxpIrTdlaoqwfsHzC3aeY + pop0h4ZF6eVD30caDCO7hZ6xaaQOQyTkyyHFyLi2BT2wj/o20KDdUUieuuG4ULNB/cwpKcuH+WiA + 6hMdxk5qXAyWxVZ5TblgTw75W3y1U2wUWQvTeaaKcK+v+iHkZ9SVNDxUM2qRcddOg9n34UFSfB4G + Z2U0Aua9go/FN7Weoak1V0NlfxwUcGpfJJWirVCbi38IZUS8RFrqkTaz9PomUlmohku7F/WRYRYm + 7yxu+0U0fPZnc9/q/3xPPmCJ/LLiGtlYrTGuo6fG0/3Me3jusLNnCDDGfTlB/JznHpRychQG6Ujz + sPL9MB1NnZWqFHJLFaw8QTwaVRiwTdEbNPV0WGUR4wYqMfwHOANbnOTHu8UizYqz9/bZdn6bFgpN + IJsc44VQj/dzuvoI+Ih3nN7PqK20ICawSMYBcGY5rCQ9YGCDpvYn+79G2nD3kgcAAA== + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74ce7299e4cb75-DFW + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 03:13:00 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '1326' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=zXi8aTY7cDmdwne_.I.3Hf.KrkQjzxMBHFbO2WWZqFg-1778037179.2921715-1.0.1.1-l3RZkU_etG9t6aSFPSTnlBiIypr5vSKjMdMV9iEuWrDk59KcVyvnaNk5TDFILQjTF1bFbvNJ5NmInvZIg_tp2BdSmbNPLyyqdw3RXJ9W9MUEfl9QCXSFs4E6vlaRqupr; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:43:00 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_e6f3256822534dd38be1cf139234e596 + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What + is 1231 * 2331? Use the multiply tool."},{"type":"function_call","call_id":"call_nJmCK7SJe3ajdYYplIbH0KgL","name":"multiply","arguments":"{\"a\": + 1231, \"b\": 2331}"},{"type":"function_call_output","call_id":"call_nJmCK7SJe3ajdYYplIbH0KgL","output":"2869461"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false,"tools":[{"type":"function","name":"multiply","description":"Multiply + two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '619' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: !!binary | + H4sIAAAAAAAA/5VVTY/aMBC98ytQjhWskgAhVOpPqNT7qoomyQTcdWzXdthFK/57x3YSEhYOPUFm + PM/z8d74c7FcRqyOvi8jjUYV8b7aplBleZlWh+0miZPs0ECZlBVmeZwnsMkPmMEO4sMW8jrLMFo5 + CFn+wcoOMFKY3l5pBIt1Ac6X7Pd5vNkneex9xoLtjIupZKs40rkQVEL1dtSyEy6vBrjBYGacM3Ek + 2yd9kkHBBbWLr/GMXCr6IMc1XDxA3l2deC9qLV2k6Dj3hkbj3w5FdSkUCuD2Qs74JeTJxABW1GiB + cTONZMJY3VWWUdFTewsfheys6mxh5Rt+dVopeVEBn8O1skbuajoqu9697NZpnGbreLtON9F4QIO7 + bhoWbiLLq29NaNA42tYcn0+2dhY3WdjRSHebJq92+30FYYIexV4Uehw0Bo4Tx7MRemclhUVxS2qa + 2Ax2aBN+2DHaHwAhpIWhta+/Z04uj0rL8oHHAxFukm6S5bdluqGfH8t0lWeH1TZLovHstf83hkfq + BMbn1DCiQQHCvBOrRreW3HvBGEa1U3GLAcdjECU1TRT5fLpEkEBhReogluEDlpHrzGRnikFAhZ/d + OGGqtVWWIKsTFm94eerT6Loe+BGl21OgDenQSDGTDzaN1L5RXL73NUama1vQA/ooJwMN2gul5KAb + hjPxGNRnRkVZNsixgY6HWRJFpMaZji22ylG48/akr78fWp8bZdbC7XtCFn8udL1P+Yy6lIb5bhJF + a9a1tz0Q5nCSlJ8fXGdlNDrMV8EM1zSd8Jq+zb5GU2mmhs7+pAqZ4pelfZfUirZEbV5upwW0QTL9 + sZvHUaSlGWkzKS8MkdpCPZzbnQzuDJM0GUnsGPbeSOqZFsr/i108QIncbmQa65mSfV53V41fEz3e + VB6eiIkH6pq5dgL/Na3dM2VxlwbxSDP/wjgx3anOSlVM1kE8GpUX2CEPBk0zHTZnVDMDJR+enM7v + tZF+TMz29mG7+mqfPAYje7wC61tgPCPq/XOQ7B85HuGO6n0GbWlP8glykowK6MxcrcQ9qMF6Ul0X + 1381ZAUOAggAAA== + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74ce7c3b71cb75-DFW + content-encoding: + - gzip + content-type: + - application/json + date: + - Wed, 06 May 2026 03:13:02 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '1106' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=yqi2SzRPEQIj_22_ctCcbWfMKVpSPEKe83ZGD3EBaLY-1778037180.8329482-1.0.1.1-enl4RmONGIjXc08ZO75dW7K7g.J5p7W_GNXjJSYbhPlgeJK0.2h4kJfDALyQFDJUpAFXFOhfg.OC3VA.I7QfOELuUPnL1HR2iM7vf7Giaqal.KxFdQJDyFBy5H6GRQcu; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:43:02 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '40000000' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3a3a8ea3224a44ef8a1ccfa281e21c3a + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml b/tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml new file mode 100644 index 000000000..c20fa5ef7 --- /dev/null +++ b/tests/cassettes/test_openai_responses/test_responses_tool_use_streaming.yaml @@ -0,0 +1,393 @@ +interactions: +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What + is 1231 * 2331? Use the multiply tool."}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true,"tools":[{"type":"function","name":"multiply","description":"Multiply + two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '399' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_00d64fa806f333310169fab1be69d081a08f8285661855594c","object":"response","created_at":1778037182,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply + two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_00d64fa806f333310169fab1be69d081a08f8285661855594c","object":"response","created_at":1778037182,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply + two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","type":"function_call","status":"in_progress","arguments":"","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply"},"output_index":0,"sequence_number":2} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"ReLAoBL6oNRsXA","output_index":0,"sequence_number":3} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"a","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"oZA1PK3x65X0w51","output_index":0,"sequence_number":4} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"WFzXP9CAvnsSdE","output_index":0,"sequence_number":5} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"123","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"3bk9yszS1TEHx","output_index":0,"sequence_number":6} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"1","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"ckqcfb2NtXm34XT","output_index":0,"sequence_number":7} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":",\"","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"cspustnrzHoiSV","output_index":0,"sequence_number":8} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"b","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"HhO4w36skjtIxaU","output_index":0,"sequence_number":9} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"SnDoVlFiXfYxtl","output_index":0,"sequence_number":10} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"233","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"arziGMMxMM5qo","output_index":0,"sequence_number":11} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"1","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"obhmaTA5ZilaAQc","output_index":0,"sequence_number":12} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"}","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"Mezf0KdDroRLDUI","output_index":0,"sequence_number":13} + + + event: response.function_call_arguments.done + + data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":1231,\"b\":2331}","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","output_index":0,"sequence_number":14} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","type":"function_call","status":"completed","arguments":"{\"a\":1231,\"b\":2331}","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply"},"output_index":0,"sequence_number":15} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_00d64fa806f333310169fab1be69d081a08f8285661855594c","object":"response","created_at":1778037182,"status":"completed","background":false,"completed_at":1778037183,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[{"id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","type":"function_call","status":"completed","arguments":"{\"a\":1231,\"b\":2331}","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply + two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":58,"input_tokens_details":{"cached_tokens":0},"output_tokens":23,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":81},"user":null,"metadata":{}},"sequence_number":16} + + + ' + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74ce85bdf0cb75-DFW + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 06 May 2026 03:13:02 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '266' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=dOVwLeU7r7aShRQaxTKiYcu2Y1q8ZJiD8OFt516IQkk-1778037182.353376-1.0.1.1-kDgOFNW1kiTQunazntKDBwpmZh4ai7MGVZI2ZIDLzRXP4MRBfrPevoYSr41Ul_ozGR_VudKgENypl8pRCVaMoqfP66IEVu.Oz7Jz1Le7WzsMCL78NWXm.AbJCZy_KKVD; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:43:02 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '39999493' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_996657ef97514aeb93f449d57303514d + status: + code: 200 + message: OK +- request: + body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What + is 1231 * 2331? Use the multiply tool."},{"type":"function_call","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply","arguments":"{\"a\": + 1231, \"b\": 2331}"},{"type":"function_call_output","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","output":"2869461"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true,"tools":[{"type":"function","name":"multiply","description":"Multiply + two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '618' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.34.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - Linux + X-Stainless-Package-Version: + - 2.34.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.11.15 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "event: response.created\ndata: {\"type\":\"response.created\",\"response\"\ + :{\"id\":\"resp_0dacb603de1c9e6b0169fab1c2314081a3b1df3cc5c09e0c60\",\"object\"\ + :\"response\",\"created_at\":1778037186,\"status\":\"in_progress\",\"background\"\ + :false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\"\ + :null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\"\ + :null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"\ + parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\"\ + :null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\"\ + :{\"effort\":\"low\",\"summary\":null},\"safety_identifier\":null,\"service_tier\"\ + :\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\"\ + :\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"\ + type\":\"function\",\"description\":\"Multiply two numbers.\",\"name\":\"\ + multiply\",\"parameters\":{\"properties\":{\"a\":{\"type\":\"integer\"},\"\ + b\":{\"type\":\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\"\ + ,\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\"\ + :0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\"\ + :{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\"\ + :\"response.in_progress\",\"response\":{\"id\":\"resp_0dacb603de1c9e6b0169fab1c2314081a3b1df3cc5c09e0c60\"\ + ,\"object\":\"response\",\"created_at\":1778037186,\"status\":\"in_progress\"\ + ,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\"\ + :0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\"\ + :null,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\"\ + :null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"\ + previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\"\ + :\"24h\",\"reasoning\":{\"effort\":\"low\",\"summary\":null},\"safety_identifier\"\ + :null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\"\ + :{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\"\ + :\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Multiply two\ + \ numbers.\",\"name\":\"multiply\",\"parameters\":{\"properties\":{\"a\":{\"\ + type\":\"integer\"},\"b\":{\"type\":\"integer\"}},\"required\":[\"a\",\"b\"\ + ],\"type\":\"object\",\"additionalProperties\":false},\"strict\":true}],\"\ + top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"\ + user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\n\ + data: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\"\ + :\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\"\ + :2}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\"\ + ,\"content_index\":0,\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"\ + logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: response.output_text.delta\n\ + data: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\"\ + :\"123\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"yJXGKTBmb2UOz\",\"output_index\":0,\"sequence_number\"\ + :4}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"0bPnrPoF6d7HddV\",\"output_index\":0,\"\ + sequence_number\":5}\n\nevent: response.output_text.delta\ndata: {\"type\"\ + :\"response.output_text.delta\",\"content_index\":0,\"delta\":\" \xD7\",\"\ + item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\"\ + :[],\"obfuscation\":\"ft7PFRXyUN31Dr\",\"output_index\":0,\"sequence_number\"\ + :6}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\" \",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"HWsKMbb1JWpQBGo\",\"output_index\":0,\"\ + sequence_number\":7}\n\nevent: response.output_text.delta\ndata: {\"type\"\ + :\"response.output_text.delta\",\"content_index\":0,\"delta\":\"233\",\"item_id\"\ + :\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\ + obfuscation\":\"OyardqgvtiISP\",\"output_index\":0,\"sequence_number\":8}\n\ + \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"M5u2aOAAbIF9HaL\",\"output_index\":0,\"\ + sequence_number\":9}\n\nevent: response.output_text.delta\ndata: {\"type\"\ + :\"response.output_text.delta\",\"content_index\":0,\"delta\":\" =\",\"item_id\"\ + :\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\ + obfuscation\":\"DblLLi7wbRu7Yj\",\"output_index\":0,\"sequence_number\":10}\n\ + \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\" **\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"iiojCebBdYGkO\",\"output_index\":0,\"sequence_number\"\ + :11}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\"2\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"eJPGA9ScIqJTAvj\",\"output_index\":0,\"\ + sequence_number\":12}\n\nevent: response.output_text.delta\ndata: {\"type\"\ + :\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\"\ + :\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\ + obfuscation\":\"pkBo5DcWhNmQVlb\",\"output_index\":0,\"sequence_number\":13}\n\ + \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\"869\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"6Z8oUB1RMn3DD\",\"output_index\":0,\"sequence_number\"\ + :14}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"Bf7uSiaON5ThxbH\",\"output_index\":0,\"\ + sequence_number\":15}\n\nevent: response.output_text.delta\ndata: {\"type\"\ + :\"response.output_text.delta\",\"content_index\":0,\"delta\":\"461\",\"item_id\"\ + :\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\ + obfuscation\":\"qqwxt4JuKvQU6\",\"output_index\":0,\"sequence_number\":16}\n\ + \nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\ + ,\"content_index\":0,\"delta\":\"**\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"obfuscation\":\"XM0rK6J7j52uLE\",\"output_index\":0,\"\ + sequence_number\":17}\n\nevent: response.output_text.done\ndata: {\"type\"\ + :\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"logprobs\":[],\"output_index\":0,\"sequence_number\":18,\"text\":\"1231\ + \ \xD7 2331 = **2,869,461**\"}\n\nevent: response.content_part.done\ndata:\ + \ {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\"\ + :\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"output_index\"\ + :0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"\ + text\":\"1231 \xD7 2331 = **2,869,461**\"},\"sequence_number\":19}\n\nevent:\ + \ response.output_item.done\ndata: {\"type\":\"response.output_item.done\"\ + ,\"item\":{\"id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\"\ + ,\"annotations\":[],\"logprobs\":[],\"text\":\"1231 \xD7 2331 = **2,869,461**\"\ + }],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"\ + sequence_number\":20}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\"\ + ,\"response\":{\"id\":\"resp_0dacb603de1c9e6b0169fab1c2314081a3b1df3cc5c09e0c60\"\ + ,\"object\":\"response\",\"created_at\":1778037186,\"status\":\"completed\"\ + ,\"background\":false,\"completed_at\":1778037187,\"error\":null,\"frequency_penalty\"\ + :0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\"\ + :null,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\"\ + :null,\"output\":[{\"id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\ + ,\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\"\ + ,\"annotations\":[],\"logprobs\":[],\"text\":\"1231 \xD7 2331 = **2,869,461**\"\ + }],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\"\ + :true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\"\ + :null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"low\"\ + ,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\"\ + ,\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"\ + },\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\"\ + :\"function\",\"description\":\"Multiply two numbers.\",\"name\":\"multiply\"\ + ,\"parameters\":{\"properties\":{\"a\":{\"type\":\"integer\"},\"b\":{\"type\"\ + :\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\",\"additionalProperties\"\ + :false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\"\ + :\"disabled\",\"usage\":{\"input_tokens\":94,\"input_tokens_details\":{\"\ + cached_tokens\":0},\"output_tokens\":18,\"output_tokens_details\":{\"reasoning_tokens\"\ + :0},\"total_tokens\":112},\"user\":null,\"metadata\":{}},\"sequence_number\"\ + :21}\n\n" + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 9f74ce9d5cec11fd-ORD + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 06 May 2026 03:13:06 GMT + openai-organization: + - user-r3e61fpak04cbaokp5buoae4 + openai-processing-ms: + - '353' + openai-project: + - proj_2Y4H7T2XPShczTQx5fqMXdNp + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=YuSyQqrRg_ripxVBGX9ZnLTFASaREsd8RjcORfPvRSE-1778037186.1373405-1.0.1.1-OTt6GIIb.vqG7KdooMAgro_13UWD_y3xibuJnSKYjofIRRPgkkgl3ivm9KWDf85p3Bq6X.FzVkbHHpoqVRfiGNNvdSjs_gxjjGTIEXa8..L8k11d2g6DXPHMJLp1bXFs; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026 + 03:43:06 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '39999457' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_808d2a0863014e248d080b21301d5d58 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_d.yaml b/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_d.yaml new file mode 100644 index 000000000..01c54923b --- /dev/null +++ b/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_d.yaml @@ -0,0 +1,145 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"What is the current llm version?"}],"model":"muse-spark-1.1","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return + the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '315' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.78.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"gen-1753242299-DdArgsNullVariantD00","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-DdArgsNullVariantD00","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"llm_version","arguments":null}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-DdArgsNullVariantD00","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"tool_calls","native_finish_reason":"tool_calls","logprobs":null}],"usage":{"prompt_tokens":57,"completion_tokens":17,"total_tokens":74,"cost":0.00007159,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 23 Jul 2025 14:54:09 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"user","content":"What is the current llm version?"},{"role":"assistant","content":""},{"role":"assistant","tool_calls":[{"type":"function","id":"0","function":{"name":"llm_version","arguments":"{}"}}]},{"role":"tool","tool_call_id":"0","content":"0.fixed-version"}],"model":"muse-spark-1.1","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return + the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '517' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.78.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"The"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + current"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + of"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + *"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"ll"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"m"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"*"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + is"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + **"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"0"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"fixed-version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"**."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop","native_finish_reason":"stop","logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":107,"completion_tokens":15,"total_tokens":122,"cost":0.0001017,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 23 Jul 2025 14:54:10 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/conftest.py b/tests/conftest.py index 8b9a7f85c..a4b6add12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,15 @@ -import pytest -import sqlite_utils +import importlib.metadata import json -import llm +import sqlite3 + import llm_echo -from llm.plugins import pm +import pytest +import sqlite_utils from pydantic import Field from pytest_httpx import IteratorStream -from typing import Optional + +import llm +from llm.plugins import pm def pytest_configure(config): @@ -15,6 +18,17 @@ def pytest_configure(config): sys._called_from_test = True +def pytest_report_header(config): + conn = sqlite3.connect(":memory:") + version = conn.execute("select sqlite_version()").fetchone()[0] + conn.close() + sqlite_utils_version = importlib.metadata.version("sqlite-utils") + return [ + f"SQLite: {version}", + f"sqlite-utils: {sqlite_utils_version}", + ] + + @pytest.fixture def user_path(tmpdir): dir = tmpdir / "llm.datasette.io" @@ -50,12 +64,13 @@ def env_setup(monkeypatch, user_path): class MockModel(llm.Model): model_id = "mock" - attachment_types = {"image/png", "audio/wav"} + attachment_types = frozenset({"image/png", "audio/wav"}) + can_stream = True supports_schema = True supports_tools = True class Options(llm.Options): - max_tokens: Optional[int] = Field( + max_tokens: int | None = Field( description="Maximum number of tokens to generate.", default=None ) @@ -105,6 +120,7 @@ async def execute(self, prompt, stream, response, conversation, key): class AsyncMockModel(llm.AsyncModel): model_id = "mock" + can_stream = True supports_schema = True def __init__(self): @@ -246,6 +262,56 @@ def mocked_openai_chat(httpx_mock): return httpx_mock +@pytest.fixture +def mock_openai_responses(httpx_mock): + def add_response( + text="Bob, Alice, Eve", + model="gpt-5.6-luna", + response_id="resp_test", + message_id="msg_test", + ): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": response_id, + "object": "response", + "created_at": 1, + "model": model, + "output": [ + { + "type": "message", + "id": message_id, + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + + return add_response + + +@pytest.fixture +def mocked_openai_responses(httpx_mock, mock_openai_responses): + mock_openai_responses() + return httpx_mock + + @pytest.fixture def mocked_openai_chat_returning_fenced_code(httpx_mock): httpx_mock.add_response( @@ -287,7 +353,7 @@ def stream_events(): } ) ).encode("utf-8") - yield "data: [DONE]\n\n".encode("utf-8") + yield b"data: [DONE]\n\n" @pytest.fixture @@ -321,6 +387,7 @@ def mocked_openai_completion(httpx_mock): "usage": {"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12}, }, headers={"Content-Type": "application/json"}, + is_reusable=True, ) return httpx_mock @@ -393,7 +460,7 @@ def stream_completion_events(): } ) ).encode("utf-8") - yield "data: [DONE]\n\n".encode("utf-8") + yield b"data: [DONE]\n\n" @pytest.fixture diff --git a/tests/test_aliases.py b/tests/test_aliases.py index 661eb1583..1645bd9e2 100644 --- a/tests/test_aliases.py +++ b/tests/test_aliases.py @@ -1,10 +1,12 @@ -from click.testing import CliRunner -from llm.cli import cli -import llm import json -import pytest import re +import pytest +from click.testing import CliRunner + +import llm +from llm.cli import cli + @pytest.mark.parametrize("model_id_or_alias", ("gpt-3.5-turbo", "chatgpt")) def test_set_alias(model_id_or_alias): @@ -30,17 +32,17 @@ def test_cli_aliases_list(args): runner = CliRunner() result = runner.invoke(cli, args) assert result.exit_code == 0 - for line in ( - "3.5 : gpt-3.5-turbo\n" - "chatgpt : gpt-3.5-turbo\n" - "chatgpt-16k : gpt-3.5-turbo-16k\n" - "3.5-16k : gpt-3.5-turbo-16k\n" - "4 : gpt-4\n" - "gpt4 : gpt-4\n" - "4-32k : gpt-4-32k\n" - "e-demo : embed-demo (embedding)\n" - "ada : text-embedding-ada-002 (embedding)\n" - ).split("\n"): + for line in [ + "3.5 : gpt-3.5-turbo", + "chatgpt : gpt-3.5-turbo", + "chatgpt-16k : gpt-3.5-turbo-16k", + "3.5-16k : gpt-3.5-turbo-16k", + "4 : gpt-4", + "gpt4 : gpt-4", + "e-demo : embed-demo (embedding)", + "ada : text-embedding-ada-002 (embedding)", + "", + ]: line = line.strip() if not line: continue @@ -64,7 +66,6 @@ def test_cli_aliases_list_json(args): "3.5-16k": "gpt-3.5-turbo-16k", "4": "gpt-4", "gpt4": "gpt-4", - "4-32k": "gpt-4-32k", "ada": "text-embedding-ada-002", "e-demo": "embed-demo", }.items() diff --git a/tests/test_async.py b/tests/test_async.py index 30e36e3e8..5a96bfbfb 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -1,6 +1,7 @@ -import llm import pytest +import llm + @pytest.mark.asyncio async def test_async_model(async_mock_model): diff --git a/tests/test_async_parity.py b/tests/test_async_parity.py new file mode 100644 index 000000000..7ba073fe6 --- /dev/null +++ b/tests/test_async_parity.py @@ -0,0 +1,395 @@ +"""Async parity: every sync API must work the same +way on AsyncResponse and AsyncConversation. + +Uses the llm-echo plugin (sync ``Echo`` + async ``EchoAsync``) so both +paths exercise real registered models with identical behaviour. +""" + +import json + +import pytest + +import llm + +# ---- basic sanity: both variants are registered -------------------- + + +def test_echo_registered_for_both(): + assert isinstance(llm.get_model("echo"), llm.Model) + assert isinstance(llm.get_async_model("echo"), llm.AsyncModel) + + +# ---- AsyncResponse.to_dict / from_dict ----------------------------- + + +@pytest.mark.asyncio +async def test_async_to_dict_captures_chain_and_output(): + model = llm.get_async_model("echo") + r = model.prompt("hello") + await r.text() + + d = r.to_dict() + assert d["model"] == "echo" + assert d["prompt"]["messages"] == [llm.user("hello").to_dict()] + # Echo's output is JSON describing the input; it's the assistant's text. + assert len(d["messages"]) == 1 + assert d["messages"][0]["role"] == "assistant" + + +@pytest.mark.asyncio +async def test_async_to_dict_raises_before_awaited(): + model = llm.get_async_model("echo") + r = model.prompt("hello") + with pytest.raises(ValueError): + r.to_dict() + + +@pytest.mark.asyncio +async def test_async_from_dict_rehydrates(): + model = llm.get_async_model("echo") + r = model.prompt("hello") + await r.text() + + payload = json.dumps(r.to_dict()) + restored = llm.AsyncResponse.from_dict(json.loads(payload)) + + assert restored._done + # text_or_raise should match (same text as original) + assert restored.text_or_raise() == r.text_or_raise() + # messages structure preserved + assert await restored.messages() == await r.messages() + # prompt.messages (the chain that was sent) preserved + assert restored.prompt.messages == r.prompt.messages + + +@pytest.mark.asyncio +async def test_async_from_dict_then_reply_continues(): + """Persist an async response across process + boundary (via JSON), rehydrate, continue with reply().""" + model = llm.get_async_model("echo") + r1 = model.prompt("q1") + await r1.text() + + payload = json.dumps(r1.to_dict()) + restored = llm.AsyncResponse.from_dict(json.loads(payload)) + + r2 = await restored.reply("q2") + await r2.text() + + # r2 was sent the full chain including r1's output. + chain_roles = [m.role for m in r2.prompt.messages] + assert chain_roles == ["user", "assistant", "user"] + assert r2.prompt.messages[0].parts[0].text == "q1" + assert r2.prompt.messages[-1].parts[0].text == "q2" + + +# ---- AsyncResponse rehydrated via from_row (SQLite path) ----------- + + +@pytest.mark.asyncio +async def test_async_from_row_response_messages_synthesized(tmp_path): + """SQLite rehydrate for async responses must populate + response.messages from _chunks+_tool_calls so follow-up chains + don't silently drop the assistant turn.""" + import sqlite_utils + + from llm.migrations import migrate + + db = sqlite_utils.Database(str(tmp_path / "logs.db")) + migrate(db) + # log_to_db no longer writes the legacy tables, so seed a row the + # way an older version of llm would have recorded it - from_row is + # the reader for exactly that history. + db["responses"].insert( + { + "id": "01aaaaaaaaaaaaaaaaaaaaaaaa", + "model": "echo", + "prompt": "hello", + "system": None, + "prompt_json": None, + "options_json": "{}", + "response": "echoed text", + "response_json": None, + "conversation_id": None, + "duration_ms": 1, + "datetime_utc": "2025-01-01T00:00:00", + "schema_id": None, + }, + alter=True, + ) + + row = next(db["responses"].rows) + rehydrated = llm.AsyncResponse.from_row(db, row) + + assert rehydrated._stream_events == [] + # response.messages falls back to _chunks — must not be empty. + msgs = await rehydrated.messages() + assert len(msgs) == 1 + assert msgs[0].role == "assistant" + assert isinstance(msgs[0].parts[0], llm.parts.TextPart) + + +# ---- AsyncConversation follow-up via load_conversation ------------- + + +@pytest.mark.asyncio +async def test_async_load_conversation_follow_up_preserves_chain(tmp_path): + """Async equivalent of the llm -c regression: after log_to_db + + load_conversation, a follow-up turn's prompt.messages is the full + [user, assistant, user] chain — not missing the assistant.""" + import sqlite_utils + + from llm.cli import load_conversation + from llm.migrations import migrate + + model = llm.get_async_model("echo") + r1 = model.prompt("q1") + await r1.text() + + db_path = tmp_path / "logs.db" + db = sqlite_utils.Database(str(db_path)) + migrate(db) + (await r1.to_sync_response()).log_to_db(db) + + conv = load_conversation(None, async_=True, database=str(db_path)) + r2 = conv.prompt("q2") + await r2.text() + + chain = r2.prompt.messages + assert [m.role for m in chain] == ["user", "assistant", "user"] + assert chain[0].parts[0].text == "q1" + assert chain[-1].parts[0].text == "q2" + + +# ---- Sync/async semantic parity for reply()+to_dict() -------------- + + +def _capture_sync(model): + r1 = model.prompt("ping") + r1.text() + payload1 = json.dumps(r1.to_dict()) + restored = llm.Response.from_dict(json.loads(payload1)) + r2 = restored.reply("pong") + r2.text() + return r2.prompt.messages + + +async def _capture_async(model): + r1 = model.prompt("ping") + await r1.text() + payload1 = json.dumps(r1.to_dict()) + restored = llm.AsyncResponse.from_dict(json.loads(payload1)) + r2 = await restored.reply("pong") + await r2.text() + return r2.prompt.messages + + +@pytest.mark.asyncio +async def test_sync_and_async_produce_identical_chain(): + """Run the full save → restore → reply loop against sync Echo and + async EchoAsync. The chain sent on the second turn must be + structurally identical.""" + sync_chain = _capture_sync(llm.get_model("echo")) + async_chain = await _capture_async(llm.get_async_model("echo")) + + # Echo's assistant output differs between invocations only in + # the "previous" field — but for the first turn both see empty + # previous, so outputs match. + sync_dicts = [m.to_dict() for m in sync_chain] + async_dicts = [m.to_dict() for m in async_chain] + assert sync_dicts == async_dicts + + +# ---- AsyncChainResponse tool-result turn pre-bakes chain ----------- + + +@pytest.mark.asyncio +async def test_async_chain_tool_result_turn_has_full_chain(): + """AsyncChainResponse must pre-bake the full chain on tool-result + turns, same as sync ChainResponse.""" + + async def my_tool(x: int) -> int: + "Double the input." + return x * 2 + + model = llm.get_async_model("echo") + # Drive a one-iteration chain by asking echo to emit a tool call + # (echo's JSON-prompt syntax). + chain = model.chain( + json.dumps( + { + "tool_calls": [{"name": "my_tool", "arguments": {"x": 5}}], + "prompt": "prompt", + } + ), + tools=[llm.Tool.function(my_tool, name="my_tool")], + ) + + responses = [] + async for response in chain.responses(): + responses.append(response) + + # Two responses: the tool-call turn and the tool-result turn. + assert len(responses) == 2 + second = responses[1] + # Second turn's prompt.messages includes the prior turn (user + + # assistant with tool call) plus a tool-role message with the result. + chain_roles = [m.role for m in second.prompt.messages] + assert "tool" in chain_roles + assert chain_roles[0] == "user" + + +# ---- astream_events() parity with stream_events() ------------------ + + +@pytest.mark.asyncio +async def test_astream_events_matches_stream_events_for_text_only(): + """Echo yields plain str (legacy plugin). Both sync and async + paths should wrap those into StreamEvent(type='text') with the + same shape.""" + sync_model = llm.get_model("echo") + async_model = llm.get_async_model("echo") + + sync_r = sync_model.prompt("hello") + sync_events = list(sync_r.stream_events()) + + async_r = async_model.prompt("hello") + async_events = [] + async for ev in async_r.astream_events(): + async_events.append(ev) + + # Same event types, same payload. + assert [e.type for e in sync_events] == [e.type for e in async_events] + assert all(e.type == "text" for e in sync_events) + assert "".join(e.chunk for e in sync_events) == "".join( + e.chunk for e in async_events + ) + + +# ---- Async reply chaining -------------------------------------------- + + +# ---- Additional edge cases ---------------------------------------- + + +@pytest.mark.asyncio +async def test_async_from_dict_model_override(): + model = llm.get_async_model("echo") + r = model.prompt("hi") + await r.text() + payload = json.dumps(r.to_dict()) + + # Pass model explicitly to override whatever's in the payload. + alt = llm.get_async_model("echo") + restored = llm.AsyncResponse.from_dict(json.loads(payload), model=alt) + assert restored.model is alt + + +def test_sync_from_dict_model_override(): + model = llm.get_model("echo") + r = model.prompt("hi") + r.text() + payload = json.dumps(r.to_dict()) + + alt = llm.get_model("echo") + restored = llm.Response.from_dict(json.loads(payload), model=alt) + assert restored.model is alt + + +@pytest.mark.asyncio +async def test_async_to_dict_preserves_datetime(): + model = llm.get_async_model("echo") + r = model.prompt("hi") + await r.text() + d = r.to_dict() + assert "datetime_utc" in d + assert isinstance(d["datetime_utc"], str) + + +@pytest.mark.asyncio +async def test_async_to_dict_preserves_usage_when_set(async_mock_model): + """When a plugin calls response.set_usage, to_dict captures it. + async_mock_model does set usage; llm-echo's async variant doesn't.""" + async_mock_model.enqueue(["ok"]) + r = async_mock_model.prompt("hi") + await r.text() + d = r.to_dict() + assert "usage" in d + assert d["usage"]["input"] is not None + assert d["usage"]["output"] is not None + + # And it round-trips. + restored = llm.AsyncResponse.from_dict(d, model=async_mock_model) + assert restored.input_tokens == d["usage"]["input"] + assert restored.output_tokens == d["usage"]["output"] + + +@pytest.mark.asyncio +async def test_async_reply_messages_kwarg_appends(): + """AsyncResponse.reply(messages=[...]) appends extra messages onto + the chain in place of a trailing user string (mirrors sync test).""" + model = llm.get_async_model("echo") + r1 = model.prompt("q1") + await r1.text() + r2 = await r1.reply(messages=[llm.user("extra")]) + await r2.text() + assert [m.role for m in r2.prompt.messages] == ["user", "assistant", "user"] + assert r2.prompt.messages[-1].parts[0].text == "extra" + + +@pytest.mark.asyncio +async def test_async_full_chain_to_dict_round_trip_three_turns(): + """Serialize on turn 3 — chain must include q1, a1, q2, a2, q3 on + round-trip.""" + model = llm.get_async_model("echo") + r1 = model.prompt("q1") + await r1.text() + r2 = await r1.reply("q2") + await r2.text() + r3 = await r2.reply("q3") + await r3.text() + + payload = json.dumps(r3.to_dict()) + restored = llm.AsyncResponse.from_dict(json.loads(payload)) + assert [m.role for m in restored.prompt.messages] == [ + "user", + "assistant", + "user", + "assistant", + "user", + ] + texts = [m.parts[0].text for m in restored.prompt.messages if m.parts] + assert texts[0] == "q1" + assert texts[2] == "q2" + assert texts[4] == "q3" + + # And continuing from the restored response extends the chain. + r4 = await restored.reply("q4") + await r4.text() + assert [m.role for m in r4.prompt.messages] == [ + "user", + "assistant", + "user", + "assistant", + "user", + "assistant", + "user", + ] + + +@pytest.mark.asyncio +async def test_async_reply_chains_three_turns(): + model = llm.get_async_model("echo") + r1 = model.prompt("q1") + await r1.text() + r2 = await r1.reply("q2") + await r2.text() + r3 = await r2.reply("q3") + await r3.text() + + chain = r3.prompt.messages + assert [m.role for m in chain] == ["user", "assistant", "user", "assistant", "user"] + texts = [m.parts[0].text for m in chain if m.parts] + assert texts[0] == "q1" + assert texts[2] == "q2" + assert texts[4] == "q3" diff --git a/tests/test_attachments.py b/tests/test_attachments.py index 6e20dd7d0..751ec8019 100644 --- a/tests/test_attachments.py +++ b/tests/test_attachments.py @@ -1,10 +1,13 @@ -from click.testing import CliRunner import os import sys from unittest.mock import ANY + +import httpx +import pytest +from click.testing import CliRunner + import llm from llm import cli -import pytest TINY_PNG = ( b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\xa6\x00\x00\x01\x1a" @@ -42,13 +45,12 @@ def test_prompt_attachment(mock_model, logs_db, attachment_type, attachment_cont ) # Check it was logged correctly - conversations = list(logs_db["conversations"].rows) - assert len(conversations) == 1 - conversation = conversations[0] - assert conversation["model"] == "mock" - assert conversation["name"] == "describe file" - response = list(logs_db["responses"].rows)[0] - attachment = list(logs_db["attachments"].rows)[0] + threads = list(logs_db["threads"].rows) + assert len(threads) == 1 + assert threads[0]["name"] == "describe file" + turn = next(iter(logs_db["turns"].rows)) + assert turn["model"] == "mock" + attachment = next(iter(logs_db["attachments"].rows)) assert attachment == { "id": ANY, "type": attachment_type, @@ -56,9 +58,9 @@ def test_prompt_attachment(mock_model, logs_db, attachment_type, attachment_cont "url": None, "content": attachment_content, } - prompt_attachment = list(logs_db["prompt_attachments"].rows)[0] - assert prompt_attachment["attachment_id"] == attachment["id"] - assert prompt_attachment["response_id"] == response["id"] + # The attachment hangs off a part of the stored user message + part_attachment = next(iter(logs_db["part_attachments"].rows)) + assert part_attachment["attachment_id"] == attachment["id"] def _count_open_fds(): @@ -96,3 +98,64 @@ def test_attachment_no_file_descriptor_leak(tmp_path): # File descriptor count should not have grown significantly assert _count_open_fds() <= baseline + 5 + + +def test_attachment_content_bytes_follows_redirects(httpx_mock): + httpx_mock.add_response( + url="https://example.com/redirected.png", + status_code=301, + headers={"Location": "https://example.com/actual.png"}, + ) + httpx_mock.add_response( + url="https://example.com/actual.png", + content=TINY_PNG, + ) + attachment = llm.Attachment(url="https://example.com/redirected.png") + assert attachment.content_bytes() == TINY_PNG + + +def test_attachment_content_bytes_limits_redirects(httpx_mock): + for redirect in range(4): + httpx_mock.add_response( + url=f"https://example.com/redirect-{redirect}", + status_code=301, + headers={"Location": f"https://example.com/redirect-{redirect + 1}"}, + ) + + attachment = llm.Attachment(url="https://example.com/redirect-0") + with pytest.raises(httpx.TooManyRedirects): + attachment.content_bytes() + + assert len(httpx_mock.get_requests()) == 4 + + +def test_attachment_resolve_type_follows_redirects(httpx_mock): + httpx_mock.add_response( + method="HEAD", + url="https://example.com/redirected.png", + status_code=301, + headers={"Location": "https://example.com/actual.png"}, + ) + httpx_mock.add_response( + method="HEAD", + url="https://example.com/actual.png", + headers={"content-type": "image/png"}, + ) + attachment = llm.Attachment(url="https://example.com/redirected.png") + assert attachment.resolve_type() == "image/png" + + +def test_attachment_resolve_type_limits_redirects(httpx_mock): + for redirect in range(4): + httpx_mock.add_response( + method="HEAD", + url=f"https://example.com/redirect-{redirect}", + status_code=301, + headers={"Location": f"https://example.com/redirect-{redirect + 1}"}, + ) + + attachment = llm.Attachment(url="https://example.com/redirect-0") + with pytest.raises(httpx.TooManyRedirects): + attachment.resolve_type() + + assert len(httpx_mock.get_requests()) == 4 diff --git a/tests/test_chat.py b/tests/test_chat.py index a0d010c2a..120f757f7 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -1,11 +1,35 @@ -from click.testing import CliRunner -from unittest.mock import ANY import json -import llm.cli -import pytest -import sqlite_utils +import re import sys import textwrap +from unittest.mock import ANY + +import pytest +import sqlite_utils +from click.testing import CliRunner + +import llm.cli +from llm.logs import LogStore, merged_log_rows + + +def logged_rows(db): + """Chronological log rows from the store, reduced to the fields + these tests care about.""" + rows = merged_log_rows(LogStore(db)) + rows.reverse() + return [ + { + "model": row["model"], + "prompt": row["prompt"], + "system": row["system"], + "options_json": row["options_json"], + "response": row["response"], + "conversation_id": row["conversation_id"], + "input_tokens": row["input_tokens"], + "output_tokens": row["output_tokens"], + } + for row in rows + ] @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") @@ -34,50 +58,30 @@ def test_chat_basic(mock_model, logs_db): "\n" ) # Should have logged - conversations = list(logs_db["conversations"].rows) - assert conversations[0] == { - "id": ANY, - "name": "Hi", - "model": "mock", - } - conversation_id = conversations[0]["id"] - responses = list(logs_db["responses"].rows) + threads = list(logs_db["threads"].rows) + assert threads[0]["name"] == "Hi" + conversation_id = threads[0]["id"] + responses = logged_rows(logs_db) assert responses == [ { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Hi", "system": None, - "prompt_json": None, "options_json": "{}", "response": "one world", - "response_json": None, "conversation_id": conversation_id, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 1, "output_tokens": 1, - "token_details": None, - "schema_id": None, }, { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Hi two", "system": None, - "prompt_json": None, "options_json": "{}", "response": "one again", - "response_json": None, "conversation_id": conversation_id, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 2, "output_tokens": 1, - "token_details": None, - "schema_id": None, }, ] # Now continue that conversation @@ -100,32 +104,17 @@ def test_chat_basic(mock_model, logs_db): "\n> quit" "\n" ) - new_responses = list( - logs_db.query( - "select * from responses where id not in ({})".format( - ", ".join("?" for _ in responses) - ), - [r["id"] for r in responses], - ) - ) + new_responses = logged_rows(logs_db)[len(responses) :] assert new_responses == [ { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Continue", "system": None, - "prompt_json": None, "options_json": "{}", "response": "continued", - "response_json": None, "conversation_id": conversation_id, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 1, "output_tokens": 1, - "token_details": None, - "schema_id": None, } ] @@ -151,25 +140,17 @@ def test_chat_system(mock_model, logs_db): "\n> quit" "\n" ) - responses = list(logs_db["responses"].rows) + responses = logged_rows(logs_db) assert responses == [ { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Hi", "system": "You are mean", - "prompt_json": None, "options_json": "{}", "response": "I am mean", - "response_json": None, "conversation_id": ANY, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 1, "output_tokens": 1, - "token_details": None, - "schema_id": None, } ] @@ -194,43 +175,27 @@ def test_chat_options(mock_model, logs_db, user_path): input="Hi with override\nquit\n", ) assert result.exit_code == 0 - responses = list(logs_db["responses"].rows) + responses = logged_rows(logs_db) assert responses == [ { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Hi", "system": None, - "prompt_json": None, "options_json": '{"max_tokens": 5}', "response": "Default options response", - "response_json": None, "conversation_id": ANY, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 1, "output_tokens": 1, - "token_details": None, - "schema_id": None, }, { - "id": ANY, "model": "mock", - "resolved_model": None, "prompt": "Hi with override", "system": None, - "prompt_json": None, "options_json": '{"max_tokens": 10}', "response": "Override options response", - "response_json": None, "conversation_id": ANY, - "duration_ms": ANY, - "datetime_utc": ANY, "input_tokens": 3, "output_tokens": 1, - "token_details": None, - "schema_id": None, }, ] @@ -275,7 +240,10 @@ def test_chat_multi(mock_model, logs_db, input, expected): llm.cli.cli, ["chat", "-m", "mock", "--option", "max_tokens", "10"], input=input ) assert result.exit_code == 0 - rows = list(logs_db["responses"].rows_where(select="prompt, response")) + rows = [ + {"prompt": row["prompt"], "response": row["response"]} + for row in logged_rows(logs_db) + ] assert rows == expected @@ -302,19 +270,17 @@ def test_llm_chat_creates_log_database(tmpdir, monkeypatch, custom_database_path else: assert (user_path / "logs.db").exists() db_path = str(user_path / "logs.db") - assert sqlite_utils.Database(db_path)["responses"].count == 2 + assert sqlite_utils.Database(db_path)["turns"].count == 2 @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") def test_chat_tools(logs_db): runner = CliRunner() - functions = textwrap.dedent( - """ + functions = textwrap.dedent(""" def upper(text: str) -> str: "Convert text to upper case" return text.upper() - """ - ) + """) result = runner.invoke( llm.cli.cli, ["chat", "-m", "echo", "--functions", functions], @@ -334,7 +300,8 @@ def upper(text: str) -> str: catch_exceptions=False, ) assert result.exit_code == 0 - assert result.output == ( + normalized_output = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", result.output) + assert normalized_output == ( "Chatting with echo\n" "Type 'exit' or 'quit' to exit\n" "Type '!multi' to enter multiple lines, then '!end' to finish\n" @@ -348,7 +315,7 @@ def upper(text: str) -> str: ' "attachments": [],\n' ' "stream": true,\n' ' "previous": []\n' - "}{\n" + "} {\n" ' "prompt": "",\n' ' "system": "",\n' ' "attachments": [],\n' @@ -364,7 +331,7 @@ def upper(text: str) -> str: " {\n" ' "name": "upper",\n' ' "output": "HELLO",\n' - ' "tool_call_id": null\n' + ' "tool_call_id": "tc_TCID"\n' " }\n" " ]\n" "}\n" @@ -384,7 +351,7 @@ def test_chat_fragments(tmpdir): output = runner.invoke( llm.cli.cli, ["chat", "-m", "echo", "-f", path1], - input=("hi\n!fragment {}\nquit\n".format(path2)), + input=(f"hi\n!fragment {path2}\nquit\n"), ).output assert '"prompt": "one' in output assert '"prompt": "two"' in output diff --git a/tests/test_chat_templates.py b/tests/test_chat_templates.py index 7687d78d9..ec8c49122 100644 --- a/tests/test_chat_templates.py +++ b/tests/test_chat_templates.py @@ -1,7 +1,10 @@ -from click.testing import CliRunner import sys -import llm.cli + import pytest +from click.testing import CliRunner + +import llm.cli +from llm.logs import LogStore, merged_log_rows @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") @@ -24,7 +27,7 @@ def test_chat_template_system_only_no_duplicate_prompt( assert result.exit_code == 0 # Ensure the logged prompt is not duplicated (no "hi\nhi") - rows = list(logs_db["responses"].rows) + rows = merged_log_rows(LogStore(logs_db)) assert len(rows) == 1 assert rows[0]["prompt"] == "hi" assert rows[0]["system"] == "Speak in French" @@ -49,17 +52,17 @@ def test_chat_system_fragments_only_first_turn(tmpdir, mock_model, logs_db): ) assert result.exit_code == 0 - # Verify only the first response has the system fragment - responses = list(logs_db["responses"].rows) - assert len(responses) == 2 - first_id = responses[0]["id"] - second_id = responses[1]["id"] + # Verify only the first turn has the system fragment + turns = list(logs_db["turns"].rows_where(order_by="id")) + assert len(turns) == 2 + first_id = turns[0]["id"] + second_id = turns[1]["id"] - sys_frags = list(logs_db["system_fragments"].rows) - # Exactly one system fragment row, attached to the first response only + sys_frags = list(logs_db["turn_fragments"].rows_where("kind = 'system'")) + # Exactly one system fragment row, attached to the first turn only assert len(sys_frags) == 1 - assert sys_frags[0]["response_id"] == first_id - assert sys_frags[0]["response_id"] != second_id + assert sys_frags[0]["turn_id"] == first_id + assert sys_frags[0]["turn_id"] != second_id @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") @@ -79,22 +82,22 @@ def test_chat_template_loads_tools_into_logs(logs_db, templates_path): ) assert result.exit_code == 0 - # Verify a single response was logged for the conversation - responses = list(logs_db["responses"].rows) - assert len(responses) == 1 - assert responses[0]["prompt"] == "hi" - response_id = responses[0]["id"] + # Verify a single turn was logged for the conversation + log_rows = merged_log_rows(LogStore(logs_db)) + assert len(log_rows) == 1 + assert log_rows[0]["prompt"] == "hi" + turn_id = log_rows[0]["id"] - # Tools from the template should be recorded against that response + # Tools from the template should be recorded against that turn rows = list( logs_db.query( """ select tools.name from tools - join tool_responses tr on tr.tool_id = tools.id - where tr.response_id = ? + join turn_tools tt on tt.tool_id = tools.id + where tt.turn_id = ? order by tools.name """, - [response_id], + [turn_id], ) ) assert [r["name"] for r in rows] == ["llm_time", "llm_version"] diff --git a/tests/test_cli_openai_models.py b/tests/test_cli_openai_models.py index fbb382de7..b763e1bf7 100644 --- a/tests/test_cli_openai_models.py +++ b/tests/test_cli_openai_models.py @@ -1,7 +1,12 @@ -from click.testing import CliRunner -from llm.cli import cli +import json + import pytest import sqlite_utils +from click.testing import CliRunner + +import llm +from llm.cli import cli +from llm.logs import LogStore @pytest.fixture @@ -59,91 +64,619 @@ def test_openai_options_min_max(): assert f"less than or equal to {max_val}" in result2.output -@pytest.mark.parametrize("model", ("gpt-4o-mini", "gpt-4o-audio-preview")) -@pytest.mark.parametrize("filetype", ("mp3", "wav")) -def test_only_gpt4_audio_preview_allows_mp3_or_wav(httpx_mock, model, filetype): +@pytest.mark.parametrize( + "model_id", + ( + "gpt-5", + "gpt-5-mini", + "gpt-5.1", + "gpt-5.2", + "gpt-5.4", + "gpt-5.5", + ), +) +def test_gpt5_models_support_verbosity_option(model_id): + assert "verbosity" in llm.get_model(model_id).Options.model_fields + assert "verbosity" in llm.get_async_model(model_id).Options.model_fields + + +@pytest.mark.parametrize("model_id", ("gpt-4o", "o3", "o4-mini")) +def test_non_gpt5_openai_chat_models_do_not_support_verbosity_option(model_id): + assert "verbosity" not in llm.get_model(model_id).Options.model_fields + assert "verbosity" not in llm.get_async_model(model_id).Options.model_fields + + +@pytest.mark.parametrize( + "model_id", + ( + "chatgpt-4o-latest", + "gpt-4o-audio-preview", + "gpt-4o-audio-preview-2024-12-17", + "gpt-4o-audio-preview-2024-10-01", + "gpt-4o-mini-audio-preview", + "gpt-4o-mini-audio-preview-2024-12-17", + "gpt-4-32k", + "gpt-4-1106-preview", + "gpt-4-0125-preview", + "gpt-4.5-preview-2025-02-27", + "gpt-4.5-preview", + "o1-preview", + "o1-mini", + "gpt-5.1-chat-latest", + ), +) +def test_deprecated_models_are_not_registered(model_id): + with pytest.raises(llm.UnknownModelError): + llm.get_model(model_id) + with pytest.raises(llm.UnknownModelError): + llm.get_async_model(model_id) + + +def test_gpt5_verbosity_option_is_sent_to_openai_chat_completions(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-5", + "usage": {}, + "choices": [{"message": {"content": "Verbose enough"}}], + }, + headers={"Content-Type": "application/json"}, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "gpt-5", + "-o", + "chat_completions", + "1", + "-o", + "verbosity", + "high", + "--no-stream", + "--key", + "x", + "Say hi", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["verbosity"] == "high" + assert "text" not in request_body + + +def test_gpt5_verbosity_option_is_sent_to_openai_responses_by_default(httpx_mock): httpx_mock.add_response( - method="HEAD", - url=f"https://www.example.com/example.{filetype}", - content=b"binary-data", - headers={"Content-Type": "audio/mpeg" if filetype == "mp3" else "audio/wav"}, + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-5", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Verbose enough", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "gpt-5", + "-o", + "verbosity", + "high", + "--no-stream", + "--key", + "x", + "Say hi", + ], + catch_exceptions=False, ) - if model == "gpt-4o-audio-preview": + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["text"]["verbosity"] == "high" + assert request_body["include"] == ["reasoning.encrypted_content"] + assert "verbosity" not in request_body + + +def test_gpt5_verbosity_option_validates_allowed_values(): + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "gpt-5", "-o", "verbosity", "extreme", "Say hi"], + ) + assert result.exit_code == 1 + assert "Input should be 'low', 'medium' or 'high'" in result.output + + +def test_code_interpreter_cli_tool_is_resolved_from_model(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_code_interpreter_cli", + "object": "response", + "created_at": 1, + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": "msg_code_interpreter_cli", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Calculated", + "annotations": [], + } + ], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + result = CliRunner().invoke( + cli, + [ + "-m", + "gpt-5.6-luna", + "-T", + 'CodeInterpreter(memory_limit="4g")', + "--no-stream", + "--no-log", + "--key", + "x", + "Run this calculation", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.stdout == "Calculated\n" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [ + { + "type": "code_interpreter", + "container": {"type": "auto", "memory_limit": "4g"}, + } + ] + assert "code_interpreter_call.outputs" in request_body["include"] + + +def test_code_interpreter_cli_tool_is_reused_on_continue(httpx_mock, user_path): + def response_payload(response_id, text): + return { + "id": response_id, + "object": "response", + "created_at": 1, + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": f"msg_{response_id}", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + } + ], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + "status": "completed", + } + + first_payload = response_payload("resp_code_interpreter_first", "Calculated") + first_payload["output"] = [ + { + "type": "message", + "id": "msg_before_code", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Running Python", + "annotations": [], + } + ], + }, + { + "type": "code_interpreter_call", + "id": "ci_continue", + "status": "completed", + "container_id": "cntr_continue", + "code": "print(6 * 7)", + "outputs": [{"type": "logs", "logs": "42\n"}], + }, + { + "type": "message", + "id": "msg_after_code", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Calculated", + "annotations": [], + } + ], + }, + ] + for payload in ( + first_payload, + response_payload("resp_code_interpreter_second", "Continued"), + ): httpx_mock.add_response( method="POST", - # chat completion request - url="https://api.openai.com/v1/chat/completions", - json={ - "id": "chatcmpl-AQT9a30kxEaM1bqxRPepQsPlCyGJh", - "object": "chat.completion", - "created": 1730871958, - "model": "gpt-4o-audio-preview-2024-10-01", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Why did the pelican get kicked out of the restaurant?\n\nBecause he had a big bill and no way to pay it!", - "refusal": None, - }, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 55, - "completion_tokens": 25, - "total_tokens": 80, - "prompt_tokens_details": { - "cached_tokens": 0, - "audio_tokens": 44, - "text_tokens": 11, - "image_tokens": 0, - }, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0, - "text_tokens": 25, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0, - }, - }, - "system_fingerprint": "fp_49254d0e9b", - }, + url="https://api.openai.com/v1/responses", + json=payload, headers={"Content-Type": "application/json"}, ) - httpx_mock.add_response( - method="GET", - url=f"https://www.example.com/example.{filetype}", - content=b"binary-data", - headers={ - "Content-Type": "audio/mpeg" if filetype == "mp3" else "audio/wav" + + runner = CliRunner() + first = runner.invoke( + cli, + [ + "-m", + "gpt-5.6-luna", + "-T", + 'CodeInterpreter(memory_limit="4g")', + "--no-stream", + "--key", + "x", + "Run this calculation", + ], + catch_exceptions=False, + ) + second = runner.invoke( + cli, + ["Continue", "-c", "--no-stream", "--key", "x"], + catch_exceptions=False, + ) + + assert first.exit_code == 0 + assert second.exit_code == 0 + assert second.output == "Continued\n" + request_bodies = [ + json.loads(request.content) for request in httpx_mock.get_requests() + ] + expected_tool = { + "type": "code_interpreter", + "container": {"type": "auto", "memory_limit": "4g"}, + } + assert request_bodies[0]["tools"] == [expected_tool] + assert request_bodies[1]["tools"] == [expected_tool] + assert request_bodies[1]["input"] == [ + {"role": "user", "content": "Run this calculation"}, + {"role": "assistant", "content": "Running Python"}, + {"role": "assistant", "content": "Calculated"}, + {"role": "user", "content": "Continue"}, + ] + + db = sqlite_utils.Database(str(user_path / "logs.db")) + instance = next(iter(db["tool_instances"].rows)) + assert instance["name"] == "CodeInterpreter" + assert json.loads(instance["arguments"])["memory_limit"] == "4g" + assert {row["instance_id"] for row in db["turn_tools"].rows} == {instance["id"]} + + +def test_web_search_cli_tool_is_resolved_from_model(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_web_search_cli", + "object": "response", + "created_at": 1, + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": "msg_web_search_cli", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Search complete", + "annotations": [], + } + ], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + result = CliRunner().invoke( + cli, + [ + "-m", + "gpt-5.6-luna", + "-T", + 'WebSearch(allowed_domains=["openai.com"], search_context_size="low", include_sources=true)', + "--no-stream", + "--no-log", + "--key", + "x", + "Search for OpenAI news", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.stdout == "Search complete\n" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [ + { + "type": "web_search", + "filters": {"allowed_domains": ["openai.com"]}, + "search_context_size": "low", + } + ] + assert "web_search_call.action.sources" in request_body["include"] + + +def test_tools_list_for_model_includes_server_side_tools(): + runner = CliRunner() + result = runner.invoke(cli, ["tools", "-m", "gpt-5.6-luna"]) + + assert result.exit_code == 0 + assert ( + "Server-side tools for gpt-5.6-luna (executed by the provider):\n" + in result.output + ) + assert "CodeInterpreter(" in result.output + assert "WebSearch(" in result.output + assert "allowed_domains:" in result.output + assert "memory_limit:" in result.output + assert "Literal['1g', '4g', '16g', '64g']" in result.output + assert "Run Python in an OpenAI-managed container." in result.output + assert "ServerSideTool(spec: dict | None = None)" in result.output + + json_result = runner.invoke(cli, ["tools", "-m", "gpt-5.6-luna", "--json"]) + assert json_result.exit_code == 0 + server_side_tools = json.loads(json_result.output)["server_side_tools"] + assert [tool["name"] for tool in server_side_tools] == [ + "WebSearch", + "CodeInterpreter", + "ServerSideTool", + ] + assert all(tool["server_side"] is True for tool in server_side_tools) + assert server_side_tools[0]["signature"].startswith("(allowed_domains:") + assert server_side_tools[0]["description"].startswith( + "Search the web using OpenAI's hosted search tool." + ) + + +def test_tools_list_for_model_with_no_server_side_tools(): + runner = CliRunner() + result = runner.invoke(cli, ["tools", "-m", "chatgpt"]) + + assert result.exit_code == 0 + + json_result = runner.invoke(cli, ["tools", "-m", "chatgpt", "--json"]) + assert json_result.exit_code == 0 + assert json.loads(json_result.output)["server_side_tools"] == [] + + +def test_tools_list_rejects_unknown_model(): + result = CliRunner().invoke(cli, ["tools", "-m", "not-a-model"]) + + assert result.exit_code == 1 + assert "Unknown model: not-a-model" in result.output + + +@pytest.mark.parametrize( + "model_id,expected_description", + ( + ( + "gpt-4o", + "Controls the detail level for image attachments. Supported values are low, high, and auto.", + ), + ( + "gpt-5.4", + "Controls the detail level for image attachments. Supported values are low, high, original, and auto.", + ), + ( + "gpt-5.5", + "Controls the detail level for image attachments. Supported values are low, high, original, and auto.", + ), + ), +) +def test_openai_image_detail_option_description(model_id, expected_description): + field = llm.get_model(model_id).Options.model_fields["image_detail"] + assert field.description == expected_description + + +def test_openai_image_detail_option_is_sent_on_image_attachments(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-4o", + "usage": {}, + "choices": [{"message": {"content": "Looks detailed"}}], + }, + headers={"Content-Type": "application/json"}, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "gpt-4o", + "-o", + "image_detail", + "high", + "--at", + "https://example.com/image.jpg", + "image/jpeg", + "--no-stream", + "--key", + "x", + "Describe this", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[-1].content) + image_part = request_body["messages"][0]["content"][1] + assert image_part == { + "type": "image_url", + "image_url": { + "url": "https://example.com/image.jpg", + "detail": "high", + }, + } + assert "image_detail" not in request_body + + +def test_openai_image_detail_original_is_sent_for_gpt54(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-5.4", + "usage": {}, + "choices": [{"message": {"content": "Original detail"}}], + }, + headers={"Content-Type": "application/json"}, + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "gpt-5.4", + "-o", + "chat_completions", + "1", + "-o", + "image_detail", + "original", + "--at", + "https://example.com/image.jpg", + "image/jpeg", + "--no-stream", + "--key", + "x", + "Describe this", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[-1].content) + image_part = request_body["messages"][0]["content"][1] + assert image_part["image_url"]["detail"] == "original" + + +def test_openai_image_detail_original_is_sent_for_gpt54_responses_by_default( + httpx_mock, +): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-5.4", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Original detail", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, }, - ) + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) runner = CliRunner() result = runner.invoke( cli, [ "-m", - model, - "-a", - f"https://www.example.com/example.{filetype}", + "gpt-5.4", + "-o", + "image_detail", + "original", + "--at", + "https://example.com/image.jpg", + "image/jpeg", "--no-stream", "--key", "x", + "Describe this", ], + catch_exceptions=False, ) - if model == "gpt-4o-audio-preview": - assert result.exit_code == 0 - assert result.output == ( - "Why did the pelican get kicked out of the restaurant?\n\n" - "Because he had a big bill and no way to pay it!\n" - ) - else: - assert result.exit_code == 1 - long = "audio/mpeg" if filetype == "mp3" else "audio/wav" - assert ( - f"This model does not support attachments of type '{long}'" in result.output - ) + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[-1].content) + image_part = request_body["input"][0]["content"][1] + assert image_part == { + "type": "input_image", + "image_url": "https://example.com/image.jpg", + "detail": "original", + } + assert "image_detail" not in request_body + + +def test_openai_image_detail_original_is_rejected_for_other_models(): + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "gpt-5", "-o", "image_detail", "original", "Say hi"], + ) + assert result.exit_code == 1 + assert "Input should be 'low', 'high' or 'auto'" in result.output @pytest.mark.parametrize("async_", (False, True)) @@ -182,7 +715,7 @@ def test_gpt4o_mini_sync_and_async(monkeypatch, tmpdir, httpx_mock, async_, usag }, headers={"Content-Type": "application/json"}, ) - runner = CliRunner(mix_stderr=False) + runner = CliRunner() args = ["-m", "gpt-4o-mini", "--key", "x", "--no-stream"] if usage: args.append(usage) @@ -190,12 +723,14 @@ def test_gpt4o_mini_sync_and_async(monkeypatch, tmpdir, httpx_mock, async_, usag args.append("--async") result = runner.invoke(cli, args, catch_exceptions=False) assert result.exit_code == 0 - assert result.output == "Ho ho ho\n" + assert result.stdout == "Ho ho ho\n" if usage: assert result.stderr == "Token usage: 1,000 input, 2,000 output\n" # Confirm it was correctly logged assert log_db.exists() db = sqlite_utils.Database(str(log_db)) - assert db["responses"].count == 1 - row = next(db["responses"].rows) - assert row["response"] == "Ho ho ho" + assert db["turns"].count == 1 + turn = next(db["turns"].rows) + store = LogStore(db) + chain = store.load_chain(turn["tip_message_hash"]) + assert chain[-1].parts[0].text == "Ho ho ho" diff --git a/tests/test_cli_options.py b/tests/test_cli_options.py index ce8a704b4..901ba00e3 100644 --- a/tests/test_cli_options.py +++ b/tests/test_cli_options.py @@ -1,7 +1,9 @@ +import json + +import pytest from click.testing import CliRunner + from llm.cli import cli -import pytest -import json @pytest.mark.parametrize( diff --git a/tests/test_cli_streaming.py b/tests/test_cli_streaming.py new file mode 100644 index 000000000..4414706fc --- /dev/null +++ b/tests/test_cli_streaming.py @@ -0,0 +1,159 @@ +"""Tests for CLI streaming display: reasoning → stderr (dim), +text → stdout, -R / --hide-reasoning flag. +""" + +import click +from click.testing import CliRunner + +import llm +from llm.cli import cli + + +def test_text_goes_to_stdout_not_stderr(mock_model): + mock_model.enqueue(["Hello world"]) + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "Hello world" in result.stdout + # No reasoning was emitted — stderr should be empty. + assert result.stderr == "" + + +def test_reasoning_goes_to_stderr_not_stdout(mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", chunk="thinking hard", part_index=0 + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "thinking hard" in result.stderr + assert "thinking hard" not in result.stdout + assert "answer" in result.stdout + + +def test_reasoning_rendered_in_dim_style(mock_model): + """The click.style(..., dim=True) wrapper emits the ANSI dim code.""" + mock_model.enqueue( + [ + llm.parts.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.parts.StreamEvent(type="text", chunk="x", part_index=1), + ] + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log"], + catch_exceptions=False, + color=True, + ) + assert result.exit_code == 0 + # ANSI dim escape sequence is \x1b[2m + dim_start = click.style("x", dim=True).split("x", 1)[0] + assert dim_start in result.stderr + + +def test_hide_reasoning_flag_suppresses_reasoning(mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", chunk="hidden thinking", part_index=0 + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log", "--hide-reasoning"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "hidden thinking" not in result.stderr + assert "hidden thinking" not in result.stdout + assert "answer" in result.stdout + assert mock_model.history[0][0].hide_reasoning is True + + +def test_hide_reasoning_short_flag_R(mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent(type="reasoning", chunk="hidden", part_index=0), + llm.parts.StreamEvent(type="text", chunk="x", part_index=1), + ] + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log", "-R"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "hidden" not in result.stderr + + +def test_newline_between_reasoning_and_text(mock_model): + """When reasoning ends and text begins, stderr gets a newline so the + text on stdout starts on a fresh visual line.""" + mock_model.enqueue( + [ + llm.parts.StreamEvent(type="reasoning", chunk="think", part_index=0), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + # Reasoning ends, then a newline is emitted on stderr. + assert result.stderr.rstrip("\n").endswith("think") or "think\n" in result.stderr + + +def test_async_path_reasoning_to_stderr(async_mock_model): + async_mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", chunk="async thinking", part_index=0 + ), + llm.parts.StreamEvent(type="text", chunk="async answer", part_index=1), + ] + ) + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--async", "--no-log"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "async thinking" in result.stderr + assert "async answer" in result.stdout + + +def test_plain_str_plugin_still_works(mock_model): + """A plugin that yields plain strings (legacy) still displays + correctly — no reasoning branch, everything to stdout.""" + mock_model.enqueue(["plain ", "text"]) + runner = CliRunner() + result = runner.invoke( + cli, + ["-m", "mock", "hi", "--no-log"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "plain text" in result.stdout + assert result.stderr == "" diff --git a/tests/test_embed.py b/tests/test_embed.py index 9b9c809cb..292170c04 100644 --- a/tests/test_embed.py +++ b/tests/test_embed.py @@ -1,9 +1,11 @@ import json -import llm -from llm.embeddings import Entry +from unittest.mock import ANY + import pytest import sqlite_utils -from unittest.mock import ANY + +import llm +from llm.embeddings import Entry def test_demo_plugin(): @@ -20,7 +22,7 @@ def test_demo_plugin(): ) def test_embed_huge_list(batch_size, expected_batches): model = llm.get_embedding_model("embed-demo") - huge_list = ("hello {}".format(i) for i in range(1000)) + huge_list = (f"hello {i}" for i in range(1000)) kwargs = {} if batch_size: kwargs["batch_size"] = batch_size @@ -120,7 +122,7 @@ def test_embed_multi(with_metadata, batch_size, expected_batches): collection = llm.Collection("test", db, model_id="embed-demo") model = collection.model() assert getattr(model, "batch_count", 0) == 0 - ids_and_texts = ((str(i), "hello {}".format(i)) for i in range(1000)) + ids_and_texts = ((str(i), f"hello {i}") for i in range(1000)) kwargs = {} if batch_size is not None: kwargs["batch_size"] = batch_size diff --git a/tests/test_embed_cli.py b/tests/test_embed_cli.py index afee77122..693955610 100644 --- a/tests/test_embed_cli.py +++ b/tests/test_embed_cli.py @@ -1,13 +1,15 @@ -from click.testing import CliRunner -from llm.cli import cli -from llm import Collection import json import pathlib -import pytest -import sqlite_utils import sys from unittest.mock import ANY +import pytest +import sqlite_utils +from click.testing import CliRunner + +from llm import Collection +from llm.cli import cli + @pytest.mark.parametrize( "format_,expected", @@ -356,7 +358,7 @@ def test_embed_multi_files_binary_store(tmpdir): assert result.exit_code == 0 db = sqlite_utils.Database(str(db_path)) assert db["embeddings"].count == 1 - row = list(db["embeddings"].rows)[0] + row = next(iter(db["embeddings"].rows)) assert row == { "collection_id": 1, "id": "file.bin", @@ -582,7 +584,7 @@ def test_embed_multi_files_errors(multi_files, args, expected_error): ) def test_embed_multi_files_encoding(multi_files, extra_args, expected_error): db_path, files = multi_files - runner = CliRunner(mix_stderr=False) + runner = CliRunner() result = runner.invoke( cli, [ @@ -686,6 +688,21 @@ def test_default_embed_model_errors(user_path, default_is_set, command): assert db["embeddings"].count == 1 +def test_embed_multi_existing_collection_without_default(user_path): + db = sqlite_utils.Database(str(user_path / "embeddings.db")) + Collection("example", db, model_id="embed-demo") + + result = CliRunner().invoke( + cli, + ["embed-multi", "example", "-"], + input="id,name\n1,hello", + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert db["embeddings"].count == 1 + + def test_duplicate_content_embedded_only_once(embed_demo): # content_hash should avoid embedding the same content twice # per collection diff --git a/tests/test_encode_decode.py b/tests/test_encode_decode.py index 1310c6db2..5ff04470e 100644 --- a/tests/test_encode_decode.py +++ b/tests/test_encode_decode.py @@ -1,6 +1,7 @@ -import llm -import pytest import numpy as np +import pytest + +import llm @pytest.mark.parametrize( diff --git a/tests/test_fragments_cli.py b/tests/test_fragments_cli.py index 8606205bd..b3ad42aa7 100644 --- a/tests/test_fragments_cli.py +++ b/tests/test_fragments_cli.py @@ -1,11 +1,15 @@ -from click.testing import CliRunner +import json +import os +import textwrap from importlib.metadata import version -from llm.cli import cli from unittest import mock -import os -import yaml + import sqlite_utils -import textwrap +import yaml +from click.testing import CliRunner + +from llm.cli import cli +from llm.migrations import migrate def test_fragments_set_show_remove(user_path): @@ -69,65 +73,78 @@ def get_list(): def test_fragments_list(user_path): runner = CliRunner() - with runner.isolated_filesystem(): - # This is just to create the database schema - with open("fragment1.txt", "w") as f: - f.write("1") - assert ( - runner.invoke(cli, ["fragments", "set", "f1", "fragment1.txt"]).exit_code - == 0 - ) - # Now add the rest directly to the database - db = sqlite_utils.Database(str(user_path / "logs.db")) - db["fragments"].delete_where() - db["fragments"].insert( - { - "content": "1", - "datetime_utc": "2023-10-01T00:00:00Z", - "source": "file1.txt", - "hash": "hash1", - }, + db = sqlite_utils.Database(str(user_path / "logs.db")) + with db.conn: + migrate(db) + db["fragments"].insert_all( + [ + { + "id": 1, + "content": "1", + "datetime_utc": "2023-10-01T00:00:00Z", + "source": "file1.txt", + "hash": "hash1", + }, + { + "id": 2, + "content": "2", + "datetime_utc": "2022-10-01T00:00:00Z", + "source": "file2.txt", + "hash": "hash2", + }, + { + "id": 3, + "content": "3", + "datetime_utc": "2024-10-01T00:00:00Z", + "source": "file3.txt", + "hash": "hash3", + }, + ] ) - db["fragments"].insert( + db["fragment_aliases"].insert( { - "content": "2", - "datetime_utc": "2022-10-01T00:00:00Z", - "source": "file2.txt", - "hash": "hash2", - }, - ) - db["fragments"].insert( - { - "content": "3", - "datetime_utc": "2024-10-01T00:00:00Z", - "source": "file3.txt", - "hash": "hash3", - }, - ) - result = runner.invoke(cli, ["fragments", "list"]) - assert result.exit_code == 0 - assert result.output.strip() == ( - textwrap.dedent( - """ - - hash: hash2 - aliases: [] - datetime_utc: '2022-10-01T00:00:00Z' - source: file2.txt - content: '2' - - hash: hash1 - aliases: - - f1 - datetime_utc: '2023-10-01T00:00:00Z' - source: file1.txt - content: '1' - - hash: hash3 - aliases: [] - datetime_utc: '2024-10-01T00:00:00Z' - source: file3.txt - content: '3' - """ - ).strip() + "alias": "f1", + "fragment_id": 1, + } ) + result = runner.invoke(cli, ["fragments", "list"]) + assert result.exit_code == 0 + assert result.output.strip() == (textwrap.dedent(""" + - hash: hash2 + aliases: [] + datetime_utc: '2022-10-01T00:00:00Z' + source: file2.txt + content: '2' + - hash: hash1 + aliases: + - f1 + datetime_utc: '2023-10-01T00:00:00Z' + source: file1.txt + content: '1' + - hash: hash3 + aliases: [] + datetime_utc: '2024-10-01T00:00:00Z' + source: file3.txt + content: '3' + """).strip()) + + +def test_fragment_absolute_path(user_path, tmp_path): + path = tmp_path / "fragment.txt" + path.write_text("Hello from an absolute path") + + result = CliRunner().invoke( + cli, ["prompt", "-m", "echo", "-f", str(path)], catch_exceptions=False + ) + + assert result.exit_code == 0 + assert json.loads(result.output) == { + "prompt": "Hello from an absolute path", + "system": "", + "attachments": [], + "stream": True, + "previous": [], + } @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) @@ -137,12 +154,21 @@ def test_fragment_url_user_agent(mocked_openai_chat, user_path): text="Hello from URL", ) runner = CliRunner() - result = runner.invoke(cli, ["prompt", "-f", "https://example.com/fragment.txt"]) + result = runner.invoke( + cli, + [ + "prompt", + "-m", + "gpt-4o-mini", + "-f", + "https://example.com/fragment.txt", + ], + ) assert result.exit_code == 0 # Verify the User-Agent header was sent for the fragment URL request requests = mocked_openai_chat.get_requests() - fragment_request = [r for r in requests if "example.com" in str(r.url)][0] + fragment_request = next(r for r in requests if "example.com" in str(r.url)) llm_version = version("llm") expected_user_agent = f"llm/{llm_version} (https://llm.datasette.io/)" assert fragment_request.headers["User-Agent"] == expected_user_agent diff --git a/tests/test_keys.py b/tests/test_keys.py index ae142d002..7668479c7 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -1,10 +1,12 @@ -from click.testing import CliRunner import json -from llm.cli import cli import pathlib -import pytest import sys +import pytest +from click.testing import CliRunner + +from llm.cli import cli + @pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows") @pytest.mark.parametrize("env", ({}, {"LLM_USER_PATH": "/tmp/llm-keys-test"})) @@ -81,32 +83,44 @@ def test_uses_correct_key(mocked_openai_chat, monkeypatch, tmpdir): def assert_key(key): request = mocked_openai_chat.get_requests()[-1] - assert request.headers["Authorization"] == "Bearer {}".format(key) + assert request.headers["Authorization"] == f"Bearer {key}" runner = CliRunner() # Called without --key uses stored key - result = runner.invoke(cli, ["hello", "--no-stream"], catch_exceptions=False) + result = runner.invoke( + cli, + ["hello", "--no-stream", "-m", "gpt-4o-mini"], + catch_exceptions=False, + ) assert result.exit_code == 0 assert_key("from-keys-file") # Called without --key and without keys.json uses environment variable keys_path.write_text("{}", "utf-8") - result2 = runner.invoke(cli, ["hello", "--no-stream"], catch_exceptions=False) + result2 = runner.invoke( + cli, + ["hello", "--no-stream", "-m", "gpt-4o-mini"], + catch_exceptions=False, + ) assert result2.exit_code == 0 assert_key("from-env") keys_path.write_text(json.dumps(KEYS), "utf-8") # Called with --key name-in-keys.json uses that value result3 = runner.invoke( - cli, ["hello", "--key", "other", "--no-stream"], catch_exceptions=False + cli, + ["hello", "--key", "other", "--no-stream", "-m", "gpt-4o-mini"], + catch_exceptions=False, ) assert result3.exit_code == 0 assert_key("other-key") # Called with --key something-else uses exactly that result4 = runner.invoke( - cli, ["hello", "--key", "custom-key", "--no-stream"], catch_exceptions=False + cli, + ["hello", "--key", "custom-key", "--no-stream", "-m", "gpt-4o-mini"], + catch_exceptions=False, ) assert result4.exit_code == 0 assert_key("custom-key") diff --git a/tests/test_llm.py b/tests/test_llm.py index 334d40f6f..14407165b 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -1,14 +1,17 @@ -from click.testing import CliRunner -import llm -from llm.cli import cli -from llm.models import Usage import json import os import pathlib -from pydantic import BaseModel +from importlib.metadata import version +from unittest import mock + import pytest import sqlite_utils -from unittest import mock +from click.testing import CliRunner +from pydantic import BaseModel + +import llm +from llm.cli import cli +from llm.models import Usage def test_version(): @@ -21,7 +24,7 @@ def test_version(): @pytest.mark.parametrize("custom_database_path", (False, True)) def test_llm_prompt_creates_log_database( - mocked_openai_chat, tmpdir, monkeypatch, custom_database_path + mocked_openai_responses, tmpdir, monkeypatch, custom_database_path ): user_path = tmpdir / "user" custom_db_path = tmpdir / "custom_log.db" @@ -40,7 +43,7 @@ def test_llm_prompt_creates_log_database( else: assert (user_path / "logs.db").exists() db_path = str(user_path / "logs.db") - assert sqlite_utils.Database(db_path)["responses"].count == 1 + assert sqlite_utils.Database(db_path)["turns"].count == 1 @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) @@ -57,12 +60,14 @@ def test_llm_prompt_creates_log_database( ), ) def test_llm_default_prompt( - mocked_openai_chat, use_stdin, user_path, logs_off, logs_args, should_log + mocked_openai_responses, use_stdin, user_path, logs_off, logs_args, should_log ): # Reset the log_path database log_path = user_path / "logs.db" log_db = sqlite_utils.Database(str(log_path)) - log_db["responses"].delete_where() + if "turns" in log_db.table_names(): + with log_db.conn: + log_db.execute("delete from turns") logs_off_path = user_path / "logs-off" if logs_off: @@ -91,35 +96,23 @@ def test_llm_default_prompt( result = runner.invoke(cli, args, input=input, catch_exceptions=False) assert result.exit_code == 0 assert result.output == "Bob, Alice, Eve\n" - last_request = mocked_openai_chat.get_requests()[-1] + last_request = mocked_openai_responses.get_requests()[-1] assert last_request.headers["Authorization"] == "Bearer X" - # Was it logged? - rows = list(log_db["responses"].rows) + # Was it logged? The legacy tables are read-only now, so the turn + # is the record; its content is asserted below through `llm logs`. + rows = list(log_db["turns"].rows) if not should_log: assert len(rows) == 0 + assert log_db["responses"].count == 0 return assert len(rows) == 1 - expected = { - "model": "gpt-4o-mini", - "prompt": "three names \nfor a pet pelican", - "system": None, - "options_json": "{}", - "response": "Bob, Alice, Eve", - } row = rows[0] - assert expected.items() <= row.items() + assert row["model"] == "gpt-5.6-luna" assert isinstance(row["duration_ms"], int) assert isinstance(row["datetime_utc"], str) - assert json.loads(row["prompt_json"]) == { - "messages": [{"role": "user", "content": "three names \nfor a pet pelican"}] - } - assert json.loads(row["response_json"]) == { - "choices": [{"message": {"content": {"$": f"r:{row['id']}"}}}], - "model": "gpt-4o-mini", - } # Test "llm logs" log_result = runner.invoke( @@ -131,54 +124,40 @@ def test_llm_default_prompt( assert ( log_json[0].items() >= { - "model": "gpt-4o-mini", + "model": "gpt-5.6-luna", "prompt": "three names \nfor a pet pelican", "system": None, - "prompt_json": { - "messages": [ - {"role": "user", "content": "three names \nfor a pet pelican"} - ] - }, + # prompt_json and response_json are no longer recorded: the + # message chain holds the structure, and the raw provider + # payload was dropped as redundant with it. "options_json": {}, "response": "Bob, Alice, Eve", - "response_json": { - "model": "gpt-4o-mini", - "choices": [{"message": {"content": {"$": f"r:{row['id']}"}}}], - }, # This doesn't have the \n after three names: "conversation_name": "three names for a pet pelican", - "conversation_model": "gpt-4o-mini", + "conversation_model": "gpt-5.6-luna", }.items() ) @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) @pytest.mark.parametrize("async_", (False, True)) -def test_llm_prompt_continue(httpx_mock, user_path, async_): - httpx_mock.add_response( - method="POST", - url="https://api.openai.com/v1/chat/completions", - json={ - "model": "gpt-4o-mini", - "usage": {}, - "choices": [{"message": {"content": "Bob, Alice, Eve"}}], - }, - headers={"Content-Type": "application/json"}, +def test_llm_prompt_continue(httpx_mock, mock_openai_responses, user_path, async_): + mock_openai_responses( + text="Bob, Alice, Eve", + response_id="resp_first", + message_id="msg_first", ) - httpx_mock.add_response( - method="POST", - url="https://api.openai.com/v1/chat/completions", - json={ - "model": "gpt-4o-mini", - "usage": {}, - "choices": [{"message": {"content": "Terry"}}], - }, - headers={"Content-Type": "application/json"}, + mock_openai_responses( + text="Terry", + response_id="resp_second", + message_id="msg_second", ) log_path = user_path / "logs.db" log_db = sqlite_utils.Database(str(log_path)) - log_db["responses"].delete_where() + if "turns" in log_db.table_names(): + with log_db.conn: + log_db.execute("delete from turns") # First prompt runner = CliRunner() @@ -190,7 +169,7 @@ def test_llm_prompt_continue(httpx_mock, user_path, async_): assert result.output == "Bob, Alice, Eve\n" # Should be logged - rows = list(log_db["responses"].rows) + rows = list(log_db["turns"].rows) assert len(rows) == 1 # Now ask a follow-up @@ -199,7 +178,7 @@ def test_llm_prompt_continue(httpx_mock, user_path, async_): assert result2.exit_code == 0, result2.output assert result2.output == "Terry\n" - rows = list(log_db["responses"].rows) + rows = list(log_db["turns"].rows) assert len(rows) == 2 @@ -240,7 +219,9 @@ def test_openai_chat_stream(mocked_openai_chat_stream, user_path): def test_openai_completion(mocked_openai_completion, user_path): log_path = user_path / "logs.db" log_db = sqlite_utils.Database(str(log_path)) - log_db["responses"].delete_where() + if "turns" in log_db.table_names(): + with log_db.conn: + log_db.execute("delete from turns") runner = CliRunner() result = runner.invoke( cli, @@ -267,18 +248,40 @@ def test_openai_completion(mocked_openai_completion, user_path): } # Check it was logged - rows = list(log_db["responses"].rows) + rows = list(log_db["turns"].rows) assert len(rows) == 1 - expected = { - "model": "gpt-3.5-turbo-instruct", - "prompt": "Say this is a test", - "system": None, - "prompt_json": '{"messages": ["Say this is a test"]}', - "options_json": "{}", - "response": "\n\nThis is indeed a test", - } - row = rows[0] - assert expected.items() <= row.items() + assert rows[0]["model"] == "gpt-3.5-turbo-instruct" + log_result = runner.invoke( + cli, ["logs", "-n", "1", "--json"], catch_exceptions=False + ) + log_json = json.loads(log_result.output) + assert ( + log_json[0].items() + >= { + "model": "gpt-3.5-turbo-instruct", + "prompt": "Say this is a test", + "system": None, + "response": "\n\nThis is indeed a test", + }.items() + ) + + +def test_openai_completion_continue_includes_history( + mocked_openai_completion, user_path +): + # A continued conversation reloaded from storage must send the + # prior exchanges, not just the newest prompt - prompt.messages + # carries them; conversation.responses does not. + runner = CliRunner() + base = ["-m", "gpt-3.5-turbo-instruct", "--no-stream", "--key", "x"] + result = runner.invoke(cli, base + ["Say this is a test"], catch_exceptions=False) + assert result.exit_code == 0 + result2 = runner.invoke(cli, base + ["Say it again", "-c"], catch_exceptions=False) + assert result2.exit_code == 0 + body = json.loads(mocked_openai_completion.get_requests()[-1].content) + assert body["prompt"] == ( + "Say this is a test\n\n\nThis is indeed a test\nSay it again" + ) def test_openai_completion_system_prompt_error(): @@ -307,7 +310,9 @@ def test_openai_completion_logprobs_stream( ): log_path = user_path / "logs.db" log_db = sqlite_utils.Database(str(log_path)) - log_db["responses"].delete_where() + if "turns" in log_db.table_names(): + with log_db.conn: + log_db.execute("delete from turns") runner = CliRunner() args = [ "-m", @@ -322,22 +327,11 @@ def test_openai_completion_logprobs_stream( result = runner.invoke(cli, args, catch_exceptions=False) assert result.exit_code == 0 assert result.output == "\n\nHi.\n" - rows = list(log_db["responses"].rows) + # Raw provider payloads (which carried the logprobs) are no longer + # persisted - the message chain is the record of what happened. + rows = list(log_db["turns"].rows) assert len(rows) == 1 - row = rows[0] - assert json.loads(row["response_json"]) == { - "content": {"$": f'r:{row["id"]}'}, - "logprobs": [ - {"text": "\n\n", "top_logprobs": [{"\n\n": -0.6, "\n": -1.9}]}, - {"text": "Hi", "top_logprobs": [{"Hi": -1.1, "Hello": -0.7}]}, - {"text": ".", "top_logprobs": [{".": -1.1, "!": -0.9}]}, - {"text": "", "top_logprobs": []}, - ], - "id": "cmpl-80MdSaou7NnPuff5ZyRMysWBmgSPS", - "object": "text_completion", - "model": "gpt-3.5-turbo-instruct", - "created": 1695097702, - } + assert rows[0]["model"] == "gpt-3.5-turbo-instruct" def test_openai_completion_logprobs_nostream( @@ -345,7 +339,9 @@ def test_openai_completion_logprobs_nostream( ): log_path = user_path / "logs.db" log_db = sqlite_utils.Database(str(log_path)) - log_db["responses"].delete_where() + if "turns" in log_db.table_names(): + with log_db.conn: + log_db.execute("delete from turns") runner = CliRunner() args = [ "-m", @@ -361,33 +357,12 @@ def test_openai_completion_logprobs_nostream( result = runner.invoke(cli, args, catch_exceptions=False) assert result.exit_code == 0 assert result.output == "\n\nHi.\n" - rows = list(log_db["responses"].rows) + # Raw provider payloads (which carried the logprobs) are no longer + # persisted - the message chain is the record of what happened. + rows = list(log_db["turns"].rows) assert len(rows) == 1 row = rows[0] - assert json.loads(row["response_json"]) == { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "logprobs": { - "text_offset": [16, 18, 20], - "token_logprobs": [-0.6, -1.1, -0.9], - "tokens": ["\n\n", "Hi", "1"], - "top_logprobs": [ - {"\n": -1.9, "\n\n": -0.6}, - {"Hello": -0.7, "Hi": -1.1}, - {"!": -1.1, ".": -0.9}, - ], - }, - "text": {"$": f"r:{row['id']}"}, - } - ], - "created": 1695097747, - "id": "cmpl-80MeBfKJutM0uMNJkRrebJLeP3bxL", - "model": "gpt-3.5-turbo-instruct", - "object": "text_completion", - "usage": {"completion_tokens": 3, "prompt_tokens": 5, "total_tokens": 8}, - } + assert row["model"] == "gpt-3.5-turbo-instruct" EXTRA_MODELS_YAML = """ @@ -431,6 +406,22 @@ def test_openai_localai_configuration(mocked_localai, user_path): } +def test_extra_openai_models_async(user_path): + from llm.default_plugins.openai_models import AsyncChat + + config_path = user_path / "extra-openai-models.yaml" + config_path.write_text(EXTRA_MODELS_YAML, "utf-8") + async_model = llm.get_async_model("orca") + assert isinstance(async_model, AsyncChat) + assert async_model.model_id == "orca" + assert async_model.model_name == "orca-mini-3b" + assert async_model.api_base == "http://localai.localhost" + assert async_model.needs_key is None + # Completion models should not have an async variant + with pytest.raises(llm.UnknownModelError): + llm.get_async_model("completion-babbage") + + @pytest.mark.parametrize( "args,exit_code", ( @@ -504,6 +495,21 @@ def test_llm_models_options(user_path): assert "AsyncMockModel (async): mock" not in result.output +def test_prompt_options_shows_selected_model_options(user_path): + runner = CliRunner() + result = runner.invoke(cli, ["-m", "gpt-5.5", "--options"], catch_exceptions=False) + expected = runner.invoke( + cli, ["models", "-m", "gpt-5.5", "--options"], catch_exceptions=False + ) + assert result.exit_code == 0 + assert expected.exit_code == 0 + assert result.output == expected.output + assert "OpenAI Responses: gpt-5.5" in result.output + assert " Options:" in result.output + assert " reasoning_effort: str" in result.output + assert not (user_path / "logs.db").exists() + + def test_llm_models_async(user_path): runner = CliRunner() result = runner.invoke(cli, ["models", "--async"], catch_exceptions=False) @@ -511,6 +517,52 @@ def test_llm_models_async(user_path): assert "AsyncMockModel (async): mock" in result.output +def test_llm_models_json_includes_server_side_tools(): + runner = CliRunner() + result = runner.invoke( + cli, + [ + "models", + "--json", + "-m", + "gpt-5.6-luna", + "-m", + "chatgpt", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + models = {model["model_id"]: model for model in json.loads(result.output)} + assert set(models) == {"gpt-5.6-luna", "gpt-3.5-turbo"} + assert models["gpt-5.6-luna"]["server_side_tools"] == [ + {"name": "WebSearch", "plugin": None}, + {"name": "CodeInterpreter", "plugin": None}, + {"name": "ServerSideTool", "plugin": None}, + ] + assert models["gpt-3.5-turbo"]["server_side_tools"] == [] + assert models["gpt-5.6-luna"]["supports_schema"] is True + assert models["gpt-5.6-luna"]["supports_tools"] is True + assert models["gpt-5.6-luna"]["can_stream"] is True + assert models["gpt-5.6-luna"]["supports_async"] is True + assert "application/pdf" in models["gpt-5.6-luna"]["attachment_types"] + + +def test_llm_models_json_options_and_alias_filter(): + result = CliRunner().invoke( + cli, + ["models", "--json", "--options", "-m", "4.1"], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + models = json.loads(result.output) + assert len(models) == 1 + assert models[0]["model_id"] == "gpt-4.1" + assert "4.1" in models[0]["aliases"] + assert "temperature" in models[0]["options"] + + @pytest.mark.parametrize( "args,expected_model_ids,unexpected_model_ids", ( @@ -523,8 +575,8 @@ def test_llm_models_async(user_path): ["OpenAI Chat: gpt-4o "], ), ( - ["-m", "gpt-4o-mini", "-m", "gpt-4.5"], - ["OpenAI Chat: gpt-4o-mini", "OpenAI Chat: gpt-4.5"], + ["-m", "gpt-4o-mini", "-m", "4.1"], + ["OpenAI Chat: gpt-4o-mini", "OpenAI Chat: gpt-4.1"], ["OpenAI Chat: gpt-4o "], ), ), @@ -555,8 +607,8 @@ def test_model_defaults(tmpdir, monkeypatch): monkeypatch.setenv("LLM_USER_PATH", user_dir) config_path = pathlib.Path(user_dir) / "default_model.txt" assert not config_path.exists() - assert llm.get_default_model() == "gpt-4o-mini" - assert llm.get_model().model_id == "gpt-4o-mini" + assert llm.get_default_model() == "gpt-5.6-luna" + assert llm.get_model().model_id == "gpt-5.6-luna" llm.set_default_model("gpt-4o") assert config_path.exists() assert llm.get_default_model() == "gpt-4o" @@ -568,6 +620,8 @@ def test_get_models(): assert all(isinstance(model, (llm.Model, llm.KeyModel)) for model in models) model_ids = [model.model_id for model in models] assert "gpt-4o-mini" in model_ids + assert "gpt-5.4-mini" in model_ids + assert "gpt-5.4-nano" in model_ids # Ensure no model_ids are duplicated # https://github.com/simonw/llm/issues/667 assert len(model_ids) == len(set(model_ids)) @@ -580,6 +634,8 @@ def test_get_async_models(): ) model_ids = [model.model_id for model in models] assert "gpt-4o-mini" in model_ids + assert "gpt-5.4-mini" in model_ids + assert "gpt-5.4-nano" in model_ids def test_mock_model(mock_model): @@ -801,27 +857,22 @@ def test_schemas_dsl(): @mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"}) @pytest.mark.parametrize("custom_database_path", (False, True)) def test_llm_prompt_continue_with_database( - tmpdir, monkeypatch, httpx_mock, user_path, custom_database_path + tmpdir, + monkeypatch, + httpx_mock, + mock_openai_responses, + user_path, + custom_database_path, ): - httpx_mock.add_response( - method="POST", - url="https://api.openai.com/v1/chat/completions", - json={ - "model": "gpt-4o-mini", - "usage": {}, - "choices": [{"message": {"content": "Bob, Alice, Eve"}}], - }, - headers={"Content-Type": "application/json"}, + mock_openai_responses( + text="Bob, Alice, Eve", + response_id="resp_first", + message_id="msg_first", ) - httpx_mock.add_response( - method="POST", - url="https://api.openai.com/v1/chat/completions", - json={ - "model": "gpt-4o-mini", - "usage": {}, - "choices": [{"message": {"content": "Terry"}}], - }, - headers={"Content-Type": "application/json"}, + mock_openai_responses( + text="Terry", + response_id="resp_second", + message_id="msg_second", ) user_path = tmpdir / "user" @@ -851,7 +902,83 @@ def test_llm_prompt_continue_with_database( else: assert (user_path / "logs.db").exists() db_path = str(user_path / "logs.db") - assert sqlite_utils.Database(db_path)["responses"].count == 2 + assert sqlite_utils.Database(db_path)["turns"].count == 2 + + +@pytest.mark.parametrize("async_", (False, True)) +def test_llm_prompt_json(logs_db, async_): + "llm --json should output the same JSON as llm logs --json" + runner = CliRunner() + args = ["-m", "echo", "hello world", "--json"] + if async_: + args.append("--async") + result = runner.invoke(cli, args, catch_exceptions=False) + assert result.exit_code == 0, result.output + rows = json.loads(result.output) + assert len(rows) == 1 + row = rows[0] + assert row["model"] == "echo" + assert row["prompt"] == "hello world" + assert json.loads(row["response"])["prompt"] == "hello world" + assert row["attachments"] == [] + assert row["tools"] == [] + assert row["tool_calls"] == [] + assert row["tool_results"] == [] + # Should be identical to the output of llm logs --json + logs_result = runner.invoke( + cli, ["logs", "-n", "1", "--json"], catch_exceptions=False + ) + assert logs_result.exit_code == 0, logs_result.output + assert json.loads(logs_result.output) == rows + + +@pytest.mark.parametrize("logs_args", (["--no-log"], ["-n"], [])) +def test_llm_prompt_json_without_logging(logs_db, logs_args): + "--json should work even when the response is not logged to the database" + runner = CliRunner() + if not logs_args: + # Turn logging off entirely instead + runner.invoke(cli, ["logs", "off"], catch_exceptions=False) + result = runner.invoke( + cli, ["-m", "echo", "hello world", "--json"] + logs_args, catch_exceptions=False + ) + assert result.exit_code == 0, result.output + rows = json.loads(result.output) + assert len(rows) == 1 + assert rows[0]["prompt"] == "hello world" + # But nothing should have been logged + assert logs_db["responses"].count == 0 + + +def test_llm_prompt_json_with_tools(logs_db): + "Each response in a tool chain should be included in the JSON" + runner = CliRunner() + result = runner.invoke( + cli, + [ + "-m", + "echo", + "-T", + "llm_version", + json.dumps({"tool_calls": [{"name": "llm_version"}]}), + "--json", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output + rows = json.loads(result.output) + assert len(rows) == 2 + assert [tool["name"] for tool in rows[0]["tools"]] == ["llm_version"] + assert [call["name"] for call in rows[0]["tool_calls"]] == ["llm_version"] + assert rows[0]["tool_results"] == [] + assert rows[1]["tool_calls"] == [] + assert [result_["output"] for result_ in rows[1]["tool_results"]] == [ + version("llm") + ] + logs_result = runner.invoke( + cli, ["logs", "-n", "2", "--json"], catch_exceptions=False + ) + assert json.loads(logs_result.output) == rows def test_default_exports(): diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 79f705521..28eec3b9e 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -1,20 +1,21 @@ -from click.testing import CliRunner -from llm.cli import cli -from llm.migrations import migrate -from llm.utils import monotonic_ulid -from llm import Fragment import datetime import json import pathlib -import pytest import re -import sqlite_utils import sys import textwrap import time -from ulid import ULID + +import pytest +import sqlite_utils import yaml +from click.testing import CliRunner +from ulid import ULID +from llm import Fragment +from llm.cli import cli +from llm.migrations import migrate +from llm.utils import monotonic_ulid SINGLE_ID = "5843577700ba729bb14c327b30441885" MULTI_ID = "4860edd987df587d042a9eb2b299ce5c" @@ -172,6 +173,41 @@ def test_logs_text_with_options(user_path): assert "- media_resolution: low" in output +def test_logs_token_usage_details_are_markdown_code(user_path): + log_path = str(user_path / "logs_token_details.db") + db = sqlite_utils.Database(log_path) + migrate(db) + db["responses"].insert( + { + "id": str(monotonic_ulid()).lower(), + "system": None, + "prompt": "prompt", + "response": "response", + "model": "davinci", + "datetime_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "conversation_id": "abc123", + "input_tokens": 2, + "output_tokens": 5, + "token_details": json.dumps( + { + "output_tokens_details": { + "reasoning_tokens": 1, + "label": "`reasoning`", + } + } + ), + } + ) + + runner = CliRunner() + result = runner.invoke(cli, ["logs", "-p", log_path, "-u"], catch_exceptions=False) + assert result.exit_code == 0 + assert ( + '## Token usage\n\n2 input, 5 output, ``{"output_tokens_details": ' + '{"reasoning_tokens": 1, "label": "`reasoning`"}}``\n' + ) in result.output + + @pytest.mark.parametrize("n", (None, 0, 2)) def test_logs_json(n, log_path): "Test that logs command correctly returns requested -n records" @@ -325,19 +361,128 @@ def test_logs_filtered(user_path, model, path_option): assert all(record["model"] == model for record in records) +def test_logs_search_new_tables(mock_model, logs_db): + runner = CliRunner() + mock_model.enqueue(["A fine city"]) + runner.invoke( + cli, ["-m", "mock", "tell me about Ljubljana"], catch_exceptions=False + ) + mock_model.enqueue(["Try httpx-retries for that"]) + runner.invoke( + cli, ["-m", "mock", "what retry library should I use"], catch_exceptions=False + ) + + # Matches prompt text + result = runner.invoke( + cli, ["logs", "-q", "Ljubljana", "--json"], catch_exceptions=False + ) + assert result.exit_code == 0 + rows = json.loads(result.output) + assert [row["prompt"] for row in rows] == ["tell me about Ljubljana"] + + # Matches response text + result2 = runner.invoke( + cli, ["logs", "-q", "httpx", "--json"], catch_exceptions=False + ) + assert result2.exit_code == 0 + rows2 = json.loads(result2.output) + assert [row["response"] for row in rows2] == ["Try httpx-retries for that"] + + +def test_logs_search_prompt_outranks_response(mock_model, logs_db): + # The prompt column carries a much higher bm25 weight: what you + # typed says more about what a turn is about than what came back. + runner = CliRunner() + mock_model.enqueue(["I recommend a python one-liner"]) + runner.invoke(cli, ["-m", "mock", "how do I sort a list"], catch_exceptions=False) + mock_model.enqueue(["Use yield"]) + runner.invoke( + cli, ["-m", "mock", "explain python generators"], catch_exceptions=False + ) + result = runner.invoke( + cli, ["logs", "-q", "python", "--json"], catch_exceptions=False + ) + assert result.exit_code == 0 + rows = json.loads(result.output) + assert [row["prompt"] for row in rows] == [ + "explain python generators", + "how do I sort a list", + ] + + +def test_logs_search_excludes_fragment_content(mock_model, logs_db, tmpdir): + fragment_path = str(tmpdir / "notes.txt") + with open(fragment_path, "w", encoding="utf-8") as fp: + fp.write("Wombats are sturdy quadrupedal marsupials") + runner = CliRunner() + mock_model.enqueue(["They do indeed"]) + runner.invoke( + cli, + ["-m", "mock", "-f", fragment_path, "do zebras have stripes"], + catch_exceptions=False, + ) + # The typed question is searchable + found = runner.invoke( + cli, ["logs", "-q", "zebras", "--json"], catch_exceptions=False + ) + assert len(json.loads(found.output)) == 1 + # The fragment's content is not + not_found = runner.invoke( + cli, ["logs", "-q", "wombats", "--json"], catch_exceptions=False + ) + assert json.loads(not_found.output) == [] + + +def test_logs_search_merges_legacy_rows(mock_model, logs_db): + # A legacy-only row and a new turn matching the same query both + # come back from one search. + migrate(logs_db) + logs_db["responses"].insert( + { + "id": "01aaaaaaaaaaaaaaaaaaaaaaaa", + "system": None, + "prompt": "name a pet pelican", + "response": "Percy", + "model": "davinci", + "datetime_utc": "2025-01-01T00:00:00", + }, + alter=True, + ) + runner = CliRunner() + mock_model.enqueue(["Scoop"]) + runner.invoke( + cli, ["-m", "mock", "another pelican name please"], catch_exceptions=False + ) + result = runner.invoke( + cli, ["logs", "-q", "pelican", "--json"], catch_exceptions=False + ) + assert result.exit_code == 0 + prompts = {row["prompt"] for row in json.loads(result.output)} + assert prompts == {"name a pet pelican", "another pelican name please"} + + +def test_logs_search_bad_query_is_a_clean_error(logs_db): + runner = CliRunner() + result = runner.invoke(cli, ["logs", "-q", 'unbalanced"quote']) + assert result.exit_code == 1 + assert "Invalid search query" in result.output + + @pytest.mark.parametrize( "query,extra_args,expected", ( # With no search term order should be by datetime ("", [], ["doc1", "doc2", "doc3"]), - # With a search it's order by rank instead - ("llama", [], ["doc1", "doc3"]), + # With a search it's order by rank instead - best match first. + # doc3 says llama twice. (The old implementation ordered by + # `rank desc`, which with bm25's negative-is-better scores put + # the weakest matches first; that sign bug is fixed.) + ("llama", [], ["doc3", "doc1"]), ("alpaca", [], ["doc2"]), # Model filter should work too - ("llama", ["-m", "davinci"], ["doc1", "doc3"]), + ("llama", ["-m", "davinci"], ["doc3", "doc1"]), ("llama", ["-m", "davinci2"], []), # Adding -l/--latest should return latest first (order by id desc) - ("llama", [], ["doc1", "doc3"]), ("llama", ["-l"], ["doc3", "doc1"]), ("llama", ["--latest"], ["doc3", "doc1"]), ), @@ -935,12 +1080,10 @@ def test_expand_fragment_markdown(fragments_fixture): def test_logs_tools(logs_db): runner = CliRunner() - code = textwrap.dedent( - """ + code = textwrap.dedent(""" def demo(): return "one\\ntwo\\nthree" - """ - ) + """) result1 = runner.invoke( cli, [ @@ -953,15 +1096,18 @@ def demo(): ) assert result1.exit_code == 0 result2 = runner.invoke(cli, ["logs", "-c"]) + normalized_output = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", result2.output) assert ( "### Tool results\n" "\n" - "- **demo**: `None`
\n" + "- **demo**: `tc_TCID` \n" + " ```\n" " one\n" " two\n" " three\n" + " ```\n" "\n" - ) in result2.output + ) in normalized_output # Log one that did NOT use tools, check that `llm logs --tools` ignores it assert runner.invoke(cli, ["-m", "echo", "badger"]).exit_code == 0 assert "badger" in runner.invoke(cli, ["logs"]).output @@ -970,6 +1116,72 @@ def demo(): assert "three" in logs_tools_output +def test_logs_repeated_tools_use_short_hash(logs_db): + runner = CliRunner() + code = textwrap.dedent(""" + def demo(): + return "ok" + """) + args = [ + "-m", + "echo", + "--functions", + code, + json.dumps({"tool_calls": [{"name": "demo"}]}), + ] + result1 = runner.invoke(cli, args) + assert result1.exit_code == 0 + result2 = runner.invoke(cli, args) + assert result2.exit_code == 0 + + result3 = runner.invoke(cli, ["logs", "-n", "2"]) + assert result3.exit_code == 0 + tool_hashes = re.findall(r"- \*\*demo\*\*: `([0-9a-f]+)`", result3.output) + assert len(tool_hashes) == 2 + assert len(tool_hashes[0]) == 64 + assert tool_hashes[1] == tool_hashes[0][:7] + + +def test_logs_tool_call_argument_formatting(logs_db): + runner = CliRunner() + code = textwrap.dedent(""" + def demo(timeout: int, options: list): + return "ok" + """) + result1 = runner.invoke( + cli, + [ + "-m", + "echo", + "--functions", + code, + json.dumps( + { + "tool_calls": [ + { + "name": "demo", + "arguments": { + "timeout": 120, + "options": ["`tick`"], + }, + } + ] + } + ), + ], + ) + assert result1.exit_code == 0 + result2 = runner.invoke(cli, ["logs", "-c"]) + normalized_output = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", result2.output) + assert ( + "### Tool calls\n" + "\n" + "- **demo**: `tc_TCID` \n" + " timeout: `120`\n" + ' options: ``["`tick`"]``\n' + ) in normalized_output + + def test_logs_backup(logs_db): assert not logs_db.tables runner = CliRunner() @@ -986,6 +1198,23 @@ def test_logs_backup(logs_db): assert expected_path.exists() +def test_logs_status_counts_threads_and_turns(logs_db): + runner = CliRunner() + runner.invoke(cli, ["-m", "echo", "simple prompt"]) + result = runner.invoke(cli, ["logs", "status"]) + assert result.exit_code == 0 + assert "Number of threads logged:\t1" in result.output + assert "Number of turns logged:\t\t1" in result.output + # No legacy rows, so the legacy lines should be hidden + assert "legacy" not in result.output + # Legacy counts show up once legacy tables have rows + logs_db["conversations"].insert({"id": "abc", "name": "test", "model": "echo"}) + result2 = runner.invoke(cli, ["logs", "status"]) + assert result2.exit_code == 0 + assert "Number of legacy conversations:\t1" in result2.output + assert "Number of legacy responses:\t0" in result2.output + + @pytest.mark.parametrize("async_", (False, True)) def test_logs_resolved_model(logs_db, mock_model, async_mock_model, async_): mock_model.resolved_model_name = "resolved-mock" @@ -996,10 +1225,10 @@ def test_logs_resolved_model(logs_db, mock_model, async_mock_model, async_): ) assert result.exit_code == 0 # Should have logged the resolved model name - assert logs_db["responses"].count - response = list(logs_db["responses"].rows)[0] - assert response["model"] == "mock" - assert response["resolved_model"] == "resolved-mock" + assert logs_db["turns"].count + turn = next(iter(logs_db["turns"].rows)) + assert turn["model"] == "mock" + assert turn["resolved_model"] == "resolved-mock" # Should show up in the JSON logs result2 = runner.invoke(cli, ["logs", "--json"]) @@ -1012,3 +1241,75 @@ def test_logs_resolved_model(logs_db, mock_model, async_mock_model, async_): # And the rendered logs result3 = runner.invoke(cli, ["logs"]) assert "Model: **mock** (resolved: **resolved-mock**)" in result3.output + + +# ---- Reasoning persistence and markdown rendering ----------------- + + +def test_log_to_db_persists_visible_reasoning(logs_db, mock_model): + """A response that streams reasoning events should round-trip the + visible reasoning text via a ReasoningPart in the stored chain.""" + import llm + from llm.logs import LogStore, merged_log_rows + + mock_model.enqueue( + [ + llm.parts.StreamEvent(type="reasoning", chunk="thinking "), + llm.parts.StreamEvent(type="reasoning", chunk="hard"), + llm.parts.StreamEvent(type="text", chunk="hello"), + ] + ) + response = mock_model.prompt("hi") + response.text() + response.log_to_db(logs_db) + + row = merged_log_rows(LogStore(logs_db))[0] + assert row["response"] == "hello" + assert row["reasoning"] == "thinking hard" + + +def test_log_to_db_persists_empty_reasoning_when_absent(logs_db, mock_model): + """No reasoning emitted → null reasoning, never raises.""" + from llm.logs import LogStore, merged_log_rows + + mock_model.enqueue(["just text"]) + response = mock_model.prompt("hi") + response.text() + response.log_to_db(logs_db) + row = merged_log_rows(LogStore(logs_db))[0] + assert not row["reasoning"] + + +def test_logs_markdown_renders_reasoning_heading(user_path): + """When a row has reasoning text, `llm logs` renders a `## Reasoning` + heading between System and Response.""" + log_path = str(user_path / "logs_with_reasoning.db") + db = sqlite_utils.Database(log_path) + migrate(db) + db["responses"].insert( + { + "id": str(monotonic_ulid()).lower(), + "system": None, + "prompt": "hi", + "response": "answer", + "reasoning": "I thought hard about it.\n\n\n", + "model": "mock", + "datetime_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "conversation_id": "c1", + } + ) + runner = CliRunner() + result = runner.invoke(cli, ["logs", "-p", log_path], catch_exceptions=False) + assert result.exit_code == 0 + # rstrip() before rendering so trailing newlines from the + # provider output don't push `## Response` down the page. + assert "## Reasoning\n\nI thought hard about it.\n\n## Response" in result.output + + +def test_logs_markdown_omits_reasoning_heading_when_empty(log_path): + """When reasoning is empty/null, no heading appears (existing + fixture rows have no reasoning).""" + runner = CliRunner() + result = runner.invoke(cli, ["logs", "-p", str(log_path)], catch_exceptions=False) + assert result.exit_code == 0 + assert "## Reasoning" not in result.output diff --git a/tests/test_logs_store.py b/tests/test_logs_store.py new file mode 100644 index 000000000..cd19399ba --- /dev/null +++ b/tests/test_logs_store.py @@ -0,0 +1,2091 @@ +"""Tests for llm.logs — the content-addressed message store. + +The store keeps conversations as a parent-linked tree of messages, each +identified by a hash over its own content plus its parent's hash. Shared +prefixes are stored once, so forking a conversation and re-sending a +history from a stateless client both write only what is new. +""" + +import json +import sqlite3 + +import pytest +import sqlite_utils +from click.testing import CliRunner + +import llm +from llm.cli import cli +from llm.logs import ( + LogStore, + canonical_json, + log_row_extras, + merged_log_rows, + message_hash, +) +from llm.migrations import migrate +from llm.models import Attachment +from llm.parts import ( + AttachmentPart, + Message, + ReasoningPart, + TextPart, + ToolCallPart, + ToolResultPart, +) +from llm.utils import ensure_fragment + +NEW_TABLES = { + "messages", + "parts", + "part_attachments", + "part_fragments", + "turns", + "turn_tools", + "turn_fragments", + "threads", +} + +# Tables the pre-existing logging path writes to. The new store must +# leave every one of them alone, so old logs stay readable without a +# backfill. +LEGACY_TABLES = { + "conversations", + "responses", + "attachments", + "prompt_attachments", + "fragments", + "tools", + "tool_calls", + "tool_results", +} + + +@pytest.fixture +def store(): + return LogStore(sqlite_utils.Database(memory=True)) + + +# ---- canonical form + hashing ---------------------------------------- + + +class TestCanonicalJson: + def test_key_order_does_not_matter(self): + assert canonical_json({"b": 1, "a": 2}) == canonical_json({"a": 2, "b": 1}) + + def test_is_compact(self): + assert canonical_json({"a": 1, "b": 2}) == '{"a":1,"b":2}' + + def test_non_ascii_is_not_escaped(self): + assert canonical_json({"a": "é"}) == '{"a":"é"}' + + def test_nested_keys_are_sorted(self): + assert canonical_json({"a": {"z": 1, "y": 2}}) == '{"a":{"y":2,"z":1}}' + + +class TestMessageHash: + def test_is_deterministic(self): + message = llm.user("Hello") + assert message_hash(message, None) == message_hash(message, None) + + def test_carries_algorithm_prefix(self): + assert message_hash(llm.user("Hello"), None).startswith("b2:") + + def test_equal_content_hashes_equal(self): + assert message_hash(llm.user("Hello"), None) == message_hash( + llm.user("Hello"), None + ) + + def test_different_text_hashes_differently(self): + assert message_hash(llm.user("Hello"), None) != message_hash( + llm.user("Goodbye"), None + ) + + def test_role_participates(self): + assert message_hash(llm.user("Hello"), None) != message_hash( + llm.assistant("Hello"), None + ) + + def test_parent_participates(self): + message = llm.user("Hello") + root = message_hash(message, None) + assert message_hash(message, root) != root + + def test_part_order_participates(self): + one = Message(role="user", parts=[TextPart(text="a"), TextPart(text="b")]) + two = Message(role="user", parts=[TextPart(text="b"), TextPart(text="a")]) + assert message_hash(one, None) != message_hash(two, None) + + def test_provider_metadata_participates(self): + plain = Message(role="assistant", parts=[TextPart(text="Hi")]) + with_meta = Message( + role="assistant", + parts=[TextPart(text="Hi", provider_metadata={"openai": {"id": "rs_1"}})], + ) + assert message_hash(plain, None) != message_hash(with_meta, None) + + def test_provider_metadata_key_order_does_not_matter(self): + one = Message( + role="assistant", + parts=[TextPart(text="Hi", provider_metadata={"a": 1, "b": 2})], + ) + two = Message( + role="assistant", + parts=[TextPart(text="Hi", provider_metadata={"b": 2, "a": 1})], + ) + assert message_hash(one, None) == message_hash(two, None) + + def test_redacted_reasoning_differs_from_empty_reasoning(self): + redacted = Message(role="assistant", parts=[ReasoningPart(redacted=True)]) + empty = Message(role="assistant", parts=[ReasoningPart()]) + assert message_hash(redacted, None) != message_hash(empty, None) + + +# ---- schema ---------------------------------------------------------- + + +class TestSchema: + def test_creates_new_tables(self, store): + assert NEW_TABLES <= set(store.db.table_names()) + + def test_leaves_legacy_tables_in_place(self, store): + assert LEGACY_TABLES <= set(store.db.table_names()) + + def test_migrating_twice_is_a_noop(self, store): + before = store.db.schema + LogStore(store.db) + assert store.db.schema == before + + def test_messages_are_keyed_by_hash(self, store): + assert store.db["messages"].pks == ["hash"] + + def test_parts_are_ordered_within_a_message(self, store): + indexes = {tuple(index.columns) for index in store.db["parts"].indexes} + assert ("message_hash", "position") in indexes + + +# ---- chain round-trip ------------------------------------------------ + + +def round_trip(store, messages): + "Write a chain, read it straight back." + return store.load_chain(store.ensure_chain(messages)) + + +class TestChainRoundTrip: + def test_empty_chain_has_no_tip(self, store): + assert store.ensure_chain([]) is None + + def test_load_chain_of_none_is_empty(self, store): + assert store.load_chain(None) == [] + + def test_tip_is_the_hash_of_the_last_message(self, store): + messages = [llm.user("Hi"), llm.assistant("Hello")] + tip = store.ensure_chain(messages) + root = message_hash(messages[0], None) + assert tip == message_hash(messages[1], root) + + def test_single_message(self, store): + messages = [llm.user("Hi")] + assert round_trip(store, messages) == messages + + def test_multiple_turns(self, store): + messages = [ + llm.system("Be brief"), + llm.user("Hi"), + llm.assistant("Hello"), + llm.user("Again"), + llm.assistant("Hello again"), + ] + assert round_trip(store, messages) == messages + + def test_reasoning_including_redacted(self, store): + messages = [ + llm.user("Think"), + llm.assistant( + ReasoningPart(redacted=True), + ReasoningPart(text="Considering it"), + TextPart(text="Done"), + ), + ] + assert round_trip(store, messages) == messages + + def test_metadata_only_opaque_reasoning_round_trips(self, store): + # Anthropic omitted thinking (empty text + signature) and + # redacted_thinking (opaque data) as adjacent, distinct parts: + # exact metadata and ordering must survive the database so the + # blocks can be replayed byte-for-byte on continuation. + messages = [ + llm.user("Think"), + llm.assistant( + ReasoningPart( + text="", + provider_metadata={ + "anthropic": {"type": "thinking", "signature": "sig-a"} + }, + ), + ReasoningPart( + text="", + provider_metadata={ + "anthropic": {"type": "redacted_thinking", "data": "blob-b"} + }, + ), + ToolCallPart(name="search", arguments={"q": "x"}, tool_call_id="tc1"), + ), + ] + assert round_trip(store, messages) == messages + + def test_interleaved_parts_keep_their_order(self, store): + # The ordering the old schema could not express: reasoning, a + # tool call, more reasoning, then text, all in one message. + messages = [ + llm.user("Search"), + llm.assistant( + ReasoningPart(text="first"), + ToolCallPart(name="search", arguments={"q": "a"}, tool_call_id="tc1"), + ReasoningPart(text="second"), + TextPart(text="answer"), + ToolCallPart(name="search", arguments={"q": "b"}, tool_call_id="tc2"), + ), + ] + assert round_trip(store, messages) == messages + + def test_tool_call_with_server_executed(self, store): + messages = [ + llm.assistant( + ToolCallPart( + name="web_search", + arguments={"query": "pelicans"}, + tool_call_id="tc1", + server_executed=True, + ) + ) + ] + assert round_trip(store, messages) == messages + + def test_tool_result_with_exception(self, store): + messages = [ + llm.tool_message( + ToolResultPart( + name="lookup", + output="", + tool_call_id="tc1", + exception="ValueError: nope", + ) + ) + ] + assert round_trip(store, messages) == messages + + def test_provider_metadata_survives(self, store): + messages = [ + Message( + role="assistant", + parts=[ + ReasoningPart( + redacted=True, + provider_metadata={ + "openai": {"id": "rs_1", "encrypted_content": "xyz"} + }, + ) + ], + provider_metadata={"openai": {"response_id": "resp_1"}}, + ) + ] + assert round_trip(store, messages) == messages + + def test_attachment_part(self, store): + messages = [ + llm.user( + "Describe", + Attachment(type="image/png", content=b"fake-png-bytes"), + ) + ] + assert round_trip(store, messages) == messages + + def test_attachment_part_with_provider_metadata(self, store): + messages = [ + llm.user( + AttachmentPart( + attachment=Attachment(type="image/png", content=b"bytes"), + provider_metadata={"openai": {"file_id": "file_1"}}, + ) + ) + ] + assert round_trip(store, messages) == messages + + def test_attachment_part_without_an_attachment(self, store): + messages = [llm.user(AttachmentPart())] + assert round_trip(store, messages) == messages + + def test_tool_result_attachments_keep_their_order(self, store): + messages = [ + llm.tool_message( + ToolResultPart( + name="render", + output="two images", + tool_call_id="tc1", + attachments=[ + Attachment(type="image/png", content=b"first"), + Attachment(type="image/png", content=b"second"), + ], + ) + ) + ] + assert round_trip(store, messages) == messages + + def test_attachment_content_is_shared_with_legacy_table(self, store): + store.ensure_chain( + [llm.user("Describe", Attachment(type="image/png", content=b"bytes"))] + ) + assert store.db["attachments"].count == 1 + + def test_unknown_tip_raises(self, store): + with pytest.raises(KeyError): + store.load_chain("b2:does-not-exist") + + +# ---- dedup ----------------------------------------------------------- + + +class TestDedup: + def test_writing_the_same_chain_twice_adds_nothing(self, store): + messages = [llm.user("Hi"), llm.assistant("Hello")] + first = store.ensure_chain(messages) + second = store.ensure_chain(messages) + assert first == second + assert store.db["messages"].count == 2 + assert store.db["parts"].count == 2 + + def test_extending_a_chain_only_writes_the_new_message(self, store): + messages = [llm.user("Hi"), llm.assistant("Hello")] + store.ensure_chain(messages) + store.ensure_chain(messages + [llm.user("More")]) + assert store.db["messages"].count == 3 + + def test_stateless_client_resending_history_writes_only_the_tail(self, store): + # What an OpenAI Chat Completions style server sees: the client + # holds the conversation and posts the whole thing every turn. + history = [] + for turn in range(5): + history.append(llm.user(f"question {turn}")) + history.append(llm.assistant(f"answer {turn}")) + store.ensure_chain(history) + assert store.db["messages"].count == 10 + + def test_diverging_chains_share_their_prefix(self, store): + prefix = [llm.user("Hi"), llm.assistant("Hello")] + store.ensure_chain(prefix + [llm.user("left")]) + store.ensure_chain(prefix + [llm.user("right")]) + # Two shared messages plus one for each branch. + assert store.db["messages"].count == 4 + + def test_appending_to_a_known_tip_skips_the_prefix_entirely(self, store): + tip = store.ensure_chain([llm.user("Hi"), llm.assistant("Hello")]) + new_tip = store.ensure_chain([llm.user("More")], parent=tip) + assert store.db["messages"].count == 3 + assert [message.parts[0].text for message in store.load_chain(new_tip)] == [ + "Hi", + "Hello", + "More", + ] + + def test_same_content_under_a_different_parent_is_a_different_message(self, store): + store.ensure_chain([llm.user("Hi"), llm.assistant("Same")]) + store.ensure_chain([llm.user("Different"), llm.assistant("Same")]) + assert store.db["messages"].count == 4 + + +# ---- threads and forking --------------------------------------------- + + +class TestThreads: + def test_new_thread_has_no_tip(self, store): + thread_id = store.create_thread(name="Empty") + assert store.thread_messages(thread_id) == [] + + def test_appending_advances_the_tip(self, store): + thread_id = store.create_thread(name="Chat") + store.append(thread_id, [llm.user("Hi")]) + store.append(thread_id, [llm.assistant("Hello")]) + assert [message.role for message in store.thread_messages(thread_id)] == [ + "user", + "assistant", + ] + + def test_thread_records_its_name(self, store): + thread_id = store.create_thread(name="Named") + assert store.db["threads"].get(thread_id)["name"] == "Named" + + def test_unknown_thread_raises(self, store): + with pytest.raises(KeyError): + store.append("nope", [llm.user("Hi")]) + + +class TestForking: + def test_fork_shares_history_up_to_the_fork_point(self, store): + thread_id = store.create_thread(name="Original") + store.append(thread_id, [llm.user("Hi"), llm.assistant("Hello")]) + fork_point = store.append(thread_id, [llm.user("Original branch")]) + + forked = store.fork(fork_point, name="What if") + assert [message.parts[0].text for message in store.thread_messages(forked)] == [ + "Hi", + "Hello", + "Original branch", + ] + + def test_forking_writes_no_new_messages(self, store): + thread_id = store.create_thread() + tip = store.append(thread_id, [llm.user("Hi"), llm.assistant("Hello")]) + before = store.db["messages"].count + store.fork(tip, name="Copy") + assert store.db["messages"].count == before + + def test_forked_branches_diverge_without_copying_the_prefix(self, store): + original = store.create_thread(name="Original") + fork_point = store.append(original, [llm.user("Hi"), llm.assistant("Hello")]) + forked = store.fork(fork_point, name="Alternative") + + store.append(original, [llm.user("down one path")]) + store.append(forked, [llm.user("down another")]) + + # Two shared messages, plus one new message per branch. + assert store.db["messages"].count == 4 + assert len(store.thread_messages(original)) == 3 + assert len(store.thread_messages(forked)) == 3 + + def test_fork_records_where_it_came_from(self, store): + original = store.create_thread(name="Original") + tip = store.append(original, [llm.user("Hi")]) + forked = store.fork(tip, name="Copy", forked_from=original) + assert store.db["threads"].get(forked)["forked_from"] == original + + def test_fork_of_an_interior_message_drops_the_later_history(self, store): + thread_id = store.create_thread() + fork_point = store.append(thread_id, [llm.user("Hi")]) + store.append(thread_id, [llm.assistant("Hello"), llm.user("More")]) + forked = store.fork(fork_point) + assert len(store.thread_messages(forked)) == 1 + + +# ---- pending tool calls ---------------------------------------------- + + +class TestPendingToolCalls: + def test_trailing_tool_calls_are_pending(self, store): + tip = store.ensure_chain( + [ + llm.user("Search"), + llm.assistant( + ToolCallPart(name="search", arguments={}, tool_call_id="tc1") + ), + ] + ) + assert [call.tool_call_id for call in store.pending_tool_calls(tip)] == ["tc1"] + + def test_tool_calls_followed_by_results_are_not_pending(self, store): + tip = store.ensure_chain( + [ + llm.user("Search"), + llm.assistant( + ToolCallPart(name="search", arguments={}, tool_call_id="tc1") + ), + llm.tool_message( + ToolResultPart(name="search", output="done", tool_call_id="tc1") + ), + ] + ) + assert store.pending_tool_calls(tip) == [] + + def test_a_plain_reply_has_nothing_pending(self, store): + tip = store.ensure_chain([llm.user("Hi"), llm.assistant("Hello")]) + assert store.pending_tool_calls(tip) == [] + + def test_server_executed_tool_calls_are_not_pending(self, store): + tip = store.ensure_chain( + [ + llm.user("Search"), + llm.assistant( + ToolCallPart( + name="web_search", + arguments={"q": "pelicans"}, + tool_call_id="tc1", + server_executed=True, + ) + ), + ] + ) + assert store.pending_tool_calls(tip) == [] + + +# ---- logging a response ---------------------------------------------- + + +class TestLogResponse: + def test_writes_a_turn(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + turn_id = store.log(response) + assert store.db["turns"].get(turn_id)["model"] == "mock" + + def test_turn_records_usage(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi there") + response.text() + turn = store.db["turns"].get(store.log(response)) + assert turn["input_tokens"] == 2 + assert turn["output_tokens"] == 1 + + def test_turn_spans_from_its_parent_to_its_tip(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + turn = store.db["turns"].get(store.log(response)) + chain = store.load_chain(turn["tip_message_hash"]) + assert [message.role for message in chain] == ["user", "assistant"] + assert chain[-1].parts[0].text == "Hello" + + def test_logging_advances_the_thread(self, store, mock_model): + thread_id = store.create_thread(name="Chat") + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + store.log(response, thread_id=thread_id) + assert [ + message.parts[0].text for message in store.thread_messages(thread_id) + ] == [ + "Hi", + "Hello", + ] + + def test_logging_the_same_response_twice_is_idempotent(self, store, mock_model): + # A turn is identified by the response it records, so re-logging + # one updates it in place rather than duplicating the event. + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + assert store.log(response) == store.log(response) + assert store.db["messages"].count == 2 + assert store.db["turns"].count == 1 + + +# ---- conversations map onto threads ---------------------------------- + + +class TestConversationThreads: + def test_log_uses_the_conversation_id_as_the_thread_id(self, store, mock_model): + conversation = mock_model.conversation() + mock_model.enqueue(["Hello"]) + response = conversation.prompt("Hi") + response.text() + store.log(response) + assert store.db["threads"].get(conversation.id) is not None + + def test_successive_turns_extend_the_same_thread(self, store, mock_model): + conversation = mock_model.conversation() + for reply in ("Hello", "Hello again"): + mock_model.enqueue([reply]) + response = conversation.prompt("Hi") + response.text() + store.log(response) + assert store.db["threads"].count == 1 + assert [ + message.parts[0].text for message in store.thread_messages(conversation.id) + ] == ["Hi", "Hello", "Hi", "Hello again"] + + def test_a_response_without_a_conversation_gets_its_own_thread( + self, store, mock_model + ): + # Parity with the legacy tables, which recorded a conversation + # for every response - without this, a response logged through + # the library API could never be continued with `llm -c`. + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + store.log(response) + assert store.db["threads"].count == 1 + + def test_an_explicit_thread_id_wins(self, store, mock_model): + thread_id = store.create_thread(name="Mine") + conversation = mock_model.conversation() + mock_model.enqueue(["Hello"]) + response = conversation.prompt("Hi") + response.text() + store.log(response, thread_id=thread_id) + assert store.db["threads"].count == 1 + assert len(store.thread_messages(thread_id)) == 2 + + +# ---- CLI integration ------------------------------------------------- + + +@pytest.fixture +def cli_store(user_path): + "A LogStore over the same database the CLI logs to." + return LogStore(sqlite_utils.Database(str(user_path / "logs.db"))) + + +def run(*args): + result = CliRunner().invoke(cli, list(args), catch_exceptions=False) + assert result.exit_code == 0, result.output + return result + + +class TestCliWrites: + def test_a_prompt_writes_a_turn(self, cli_store): + run("-m", "echo", "Hi") + assert cli_store.db["turns"].count == 1 + + def test_a_prompt_does_not_write_the_legacy_tables(self, cli_store): + run("-m", "echo", "Hi") + assert cli_store.db["responses"].count == 0 + assert cli_store.db["conversations"].count == 0 + + def test_the_turn_points_at_the_stored_chain(self, cli_store): + run("-m", "echo", "Hi") + turn = next(iter(cli_store.db["turns"].rows)) + chain = cli_store.load_chain(turn["tip_message_hash"]) + assert [message.role for message in chain] == ["user", "assistant"] + + def test_the_thread_has_a_tip(self, cli_store): + run("-m", "echo", "Hi") + conversation_id = next(iter(cli_store.db["threads"].rows))["id"] + assert cli_store.thread_tip(conversation_id) is not None + + def test_no_log_writes_nothing(self, cli_store): + run("-m", "echo", "Hi", "--no-log") + assert cli_store.db["turns"].count == 0 + assert cli_store.db["messages"].count == 0 + + +class TestCliContinuation: + def test_continuing_extends_the_same_thread(self, cli_store): + run("-m", "echo", "First") + run("-m", "echo", "Second", "-c") + assert cli_store.db["threads"].count == 1 + conversation_id = next(iter(cli_store.db["threads"].rows))["id"] + assert len(cli_store.thread_messages(conversation_id)) == 4 + + def test_history_comes_from_the_new_tables(self, user_path): + run("-m", "echo", "First") + + db = sqlite_utils.Database(str(user_path / "logs.db")) + conversation_id = next(iter(db["threads"].rows))["id"] + db.close() + + run("-m", "echo", "Second", "-c") + + # Four messages only if the second turn was built on top of the + # first. Had the history been lost, the second turn would have + # started a fresh root and the thread would hold just two. + store = LogStore(sqlite_utils.Database(str(user_path / "logs.db"))) + chain = store.thread_messages(conversation_id) + assert len(chain) == 4 + assert chain[0].parts[0].text == "First" + + def test_continuing_writes_only_the_new_messages(self, cli_store): + run("-m", "echo", "First") + before = cli_store.db["messages"].count + run("-m", "echo", "Second", "-c") + assert cli_store.db["messages"].count == before + 2 + + +# ---- history loaded from storage ------------------------------------- + + +class TestLoadedMessages: + def test_loaded_messages_supply_the_history(self, mock_model): + conversation = mock_model.conversation() + conversation.loaded_messages = [llm.user("Earlier"), llm.assistant("Reply")] + mock_model.enqueue(["Next"]) + response = conversation.prompt("Now") + response.text() + assert [message.parts[0].text for message in response.prompt.messages] == [ + "Earlier", + "Reply", + "Now", + ] + + def test_a_completed_response_supersedes_them(self, mock_model): + conversation = mock_model.conversation() + conversation.loaded_messages = [llm.user("Earlier"), llm.assistant("Reply")] + mock_model.enqueue(["First"]) + conversation.prompt("One").text() + mock_model.enqueue(["Second"]) + response = conversation.prompt("Two") + response.text() + # The live response takes over; the loaded history is not + # replayed a second time. + assert [message.parts[0].text for message in response.prompt.messages] == [ + "Earlier", + "Reply", + "One", + "First", + "Two", + ] + + +class TestLegacyConversations: + def test_continuing_a_conversation_with_no_thread_still_works(self, user_path): + # Conversations logged before this schema existed have no thread, + # so `-c` has to fall back to rebuilding from the legacy rows. + # Nothing writes those rows any more - seed them the way an + # older version of llm would have. + path = str(user_path / "logs.db") + db = sqlite_utils.Database(path) + migrate(db) + db["conversations"].insert( + {"id": "01aaaaaaaaaaaaaaaaaaaaaaaa", "name": "First", "model": "echo"} + ) + db["responses"].insert( + { + "id": "01aaaaaaaaaaaaaaaaaaaaaaab", + "model": "echo", + "prompt": "First", + "system": None, + "prompt_json": None, + "options_json": "{}", + "response": "First response", + "response_json": None, + "conversation_id": "01aaaaaaaaaaaaaaaaaaaaaaaa", + "duration_ms": 1, + "datetime_utc": "2025-01-01T00:00:00", + "schema_id": None, + }, + alter=True, + ) + db.close() + + result = run("-m", "echo", "Second", "-c") + assert "First" in result.output + + +# ---- logging through the library API --------------------------------- + + +class TestLibraryLogging: + """`log_to_db` is what plugins call, so the store write belongs there + rather than in the CLI - otherwise anything that is not `llm` itself + writes only the legacy tables.""" + + def test_log_to_db_writes_the_store_too(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + response.log_to_db(store.db) + assert store.db["turns"].count == 1 + assert store.db["messages"].count == 2 + + def test_log_to_db_leaves_the_legacy_tables_alone(self, store, mock_model): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + response.log_to_db(store.db) + assert store.db["responses"].count == 0 + assert store.db["conversations"].count == 0 + + def test_log_to_db_without_conversation_still_gets_a_thread( + self, store, mock_model + ): + # The legacy path recorded a conversation for every response; + # the store keeps that guarantee with a thread, so `llm -c` can + # continue a response logged through the library API. + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + response.log_to_db(store.db) + assert store.db["threads"].count == 1 + turn = next(iter(store.db["turns"].rows)) + assert turn["thread_id"] is not None + + def test_a_chain_writes_the_store_too(self, store, mock_model): + conversation = mock_model.conversation() + mock_model.enqueue(["Hello"]) + chain = conversation.chain("Hi") + chain.text() + chain.log_to_db(store.db) + assert store.db["turns"].count == 1 + assert len(store.thread_messages(conversation.id)) == 2 + + def test_messages_plus_prompt_both_reach_the_chain(self, store, mock_model): + """prompt= alongside messages= used to vanish from the logged + chain: Prompt.messages returned the explicit list verbatim, so + the text the model answered was absent from the store.""" + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("follow-up", messages=[llm.user("original")]) + response.text() + response.log_to_db(store.db) + turn = next(iter(store.db["turns"].rows)) + chain = store.load_chain(turn["tip_message_hash"]) + texts = [part.text for message in chain for part in message.parts] + assert texts == ["original", "follow-up", "Hello"] + assert store.verify() == [] + + def test_log_to_db_records_tool_instantiations(self, store, mock_model): + class Notes(llm.Toolbox): + def __init__(self, path: str): + self.path = path + + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "next", + tool_results=[ + llm.ToolResult( + name="Notes_read", + output="hello", + tool_call_id="tc_1", + instance=Notes("/tmp/notes"), + ) + ], + ) + response.text() + response.log_to_db(store.db) + turn_id = next(iter(store.db["turns"].rows))["id"] + link = next(iter(store.db["tool_instantiations"].rows)) + assert link["turn_id"] == turn_id + assert link["tool_call_id"] == "tc_1" + instance = store.db["tool_instances"].get(link["instance_id"]) + assert instance["name"] == "Notes" + assert instance["plugin"] is None + assert instance["arguments"] == '{"path": "/tmp/notes"}' + + def test_turn_tools_reference_the_configured_instance(self, store, mock_model): + class Notes(llm.Toolbox): + def __init__(self, path: str): + self.path = path + + def read(self) -> str: + "Read the notes" + return "hi" + + for prompt in ("first", "second"): + mock_model.enqueue(["ok"]) + response = mock_model.prompt(prompt, tools=[Notes("/tmp/x")]) + response.text() + response.log_to_db(store.db) + + # One instance row however many turns it serves + assert store.db["tool_instances"].count == 1 + instance_ids = {row["instance_id"] for row in store.db["turn_tools"].rows} + assert len(instance_ids) == 1 + + # And the tools list in the display carries it + rows = merged_log_rows(store) + tools = log_row_extras(store, rows[0])["tools"] + assert tools[0]["instance"] == { + "name": "Notes", + "arguments": '{"path": "/tmp/x"}', + } + + def test_tool_instantiations_are_scoped_by_turn(self, store, mock_model): + # Providers with per-request counters can reuse a tool_call_id + # across turns - each turn keeps its own provenance row. + class Notes(llm.Toolbox): + def __init__(self, path: str): + self.path = path + + for path in ("/tmp/one", "/tmp/two"): + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "next", + tool_results=[ + llm.ToolResult( + name="Notes_read", + output="hello", + tool_call_id="call_0", + instance=Notes(path), + ) + ], + ) + response.text() + response.log_to_db(store.db) + arguments = sorted(row["arguments"] for row in store.db.query(""" + select tool_instances.arguments from tool_instantiations + join tool_instances + on tool_instances.id = tool_instantiations.instance_id + """)) + assert arguments == ['{"path": "/tmp/one"}', '{"path": "/tmp/two"}'] + + def test_successive_library_turns_extend_the_thread(self, store, mock_model): + conversation = mock_model.conversation() + for reply in ("One", "Two"): + mock_model.enqueue([reply]) + response = conversation.prompt("Ask") + response.text() + response.log_to_db(store.db) + assert len(store.thread_messages(conversation.id)) == 4 + + +# ---- storage by reference -------------------------------------------- + + +class TestPerTurnToolResolution: + def test_same_name_different_definitions_resolve_per_turn(self, store, mock_model): + # Two turns using tools that share a name but differ in + # definition - each turn's extras must report its own tool_id. + def make_tool(description): + return llm.Tool(name="lookup", description=description, input_schema={}) + + for description in ("first definition", "second definition"): + mock_model.enqueue(["ok"]) + response = mock_model.prompt("hi", tools=[make_tool(description)]) + response.text() + response.log_to_db(store.db) + + rows = merged_log_rows(store) + rows.reverse() + for row in rows: + extras = log_row_extras(store, row) + assert len(extras["tools"]) == 1 + descriptions_to_ids = { + log_row_extras(store, row)["tools"][0]["description"]: log_row_extras( + store, row + )["tools"][0]["id"] + for row in rows + } + assert len(descriptions_to_ids) == 2 + assert len(set(descriptions_to_ids.values())) == 2 + + +class TestRepeatedFragments: + def test_passing_the_same_fragment_twice_keeps_both_rows(self, store, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt("hi", fragments=["CONTEXT", "CONTEXT"]) + response.text() + response.log_to_db(store.db) + rows = list( + store.db["turn_fragments"].rows_where("kind = 'prompt'", order_by='"order"') + ) + assert [row["order"] for row in rows] == [0, 1] + assert rows[0]["fragment_id"] == rows[1]["fragment_id"] + + +class TestAtomicWrites: + """sqlite-utils runs in autocommit, so `with db.conn` was never a + transaction - a crash mid-write could strand a message without its + parts, and the dedup check would then skip it forever.""" + + def test_failed_part_write_rolls_back_the_message(self, store, monkeypatch): + message = Message(role="user", parts=[TextPart(text="a"), TextPart(text="b")]) + original = LogStore._write_part + + def flaky(self, hash_, position, part, fragment_map): + if position == 1: + raise RuntimeError("disk full") + return original(self, hash_, position, part, fragment_map) + + monkeypatch.setattr(LogStore, "_write_part", flaky) + with pytest.raises(RuntimeError): + store.ensure_chain([message]) + monkeypatch.undo() + assert store.db["messages"].count == 0 + assert store.db["parts"].count == 0 + # A retry can now write the whole message + tip = store.ensure_chain([message]) + assert store.load_chain(tip) == [message] + assert store.verify() == [] + + def test_failed_turn_write_rolls_back_the_whole_turn( + self, store, mock_model, monkeypatch + ): + mock_model.enqueue(["ok"]) + response = mock_model.prompt("Hi") + response.text() + # Fail at the search refresh, one of the last steps of log() + monkeypatch.setattr("llm.logs.TURN_SEARCH_INSERT_SQL", "this is not sql") + with pytest.raises(sqlite3.OperationalError): + store.log(response) + monkeypatch.undo() + assert store.db["turns"].count == 0 + assert store.db["messages"].count == 0 + assert store.db["threads"].count == 0 + # And the retry writes everything + store.log(response) + assert store.db["turns"].count == 1 + assert store.verify() == [] + + +class TestConcurrentWriters: + def test_losing_the_insert_race_neither_raises_nor_duplicates( + self, tmp_path, monkeypatch + ): + # Two connections to the same database. B checks for the hash + # while it is absent - simulated by disabling its fast-path + # check - then A wins the insert. B's own insert must quietly + # lose: no UNIQUE error, no second set of parts. + path = str(tmp_path / "logs.db") + store_a = LogStore(sqlite_utils.Database(path)) + store_b = LogStore(sqlite_utils.Database(path)) + message = llm.user("Hi") + tip = store_a.ensure_chain([message]) + monkeypatch.setattr( + sqlite_utils.db.Table, "count_where", lambda *args, **kwargs: 0 + ) + assert store_b.ensure_chain([message]) == tip + monkeypatch.undo() + assert store_a.db["messages"].count == 1 + assert store_a.db["parts"].count == 1 + assert store_a.verify() == [] + + +class TestTurnInputBoundary: + """A turn whose input ends [tool results, user prompt] owns both - + the tool results must not vanish from display or the -T filter just + because a user message follows them.""" + + def _log_turn_with_results_and_prompt(self, store, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "next question", + messages=[llm.user("orig"), llm.assistant("first answer")], + tool_results=[llm.ToolResult(name="t", output="RESULT", tool_call_id="c9")], + ) + response.text() + response.log_to_db(store.db) + + def test_tool_results_and_prompt_both_display(self, store, mock_model): + self._log_turn_with_results_and_prompt(store, mock_model) + row = merged_log_rows(store)[0] + assert row["prompt"] == "next question" + extras = log_row_extras(store, row) + assert [result["output"] for result in extras["tool_results"]] == ["RESULT"] + # The parts row id resolves even though the result sits one + # message above the parent. + assert extras["tool_results"][0]["id"] is not None + + def test_tool_filters_match(self, store, mock_model): + self._log_turn_with_results_and_prompt(store, mock_model) + assert len(merged_log_rows(store, any_tools=True)) == 1 + assert len(merged_log_rows(store, tool_names=["t"])) == 1 + assert merged_log_rows(store, tool_names=["other"]) == [] + + +class TestUnsupportedBranchDatabases: + def test_existing_message_store_tables_fail_loudly(self, tmp_path): + # The message store migration creates its tables in final form + # and assumes they do not exist - a database carrying tables + # from unreleased development revisions is an unsupported + # state, and the migration fails loudly rather than dropping + # or adapting whatever is there. + db = sqlite_utils.Database(str(tmp_path / "old-branch.db")) + db["messages"].create({"hash": str}, pk="hash") + with pytest.raises(sqlite3.OperationalError): + migrate(db) + + +class TestAttachmentHashing: + """Message identity covers attachment content, never the filesystem + path the bytes were loaded from.""" + + def test_same_bytes_at_two_paths_are_one_identity(self, tmp_path): + path_a = tmp_path / "a.png" + path_b = tmp_path / "b.png" + path_a.write_bytes(b"SAME BYTES") + path_b.write_bytes(b"SAME BYTES") + hashes = [ + message_hash( + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(p)) + ) + ], + ), + None, + ) + for p in (path_a, path_b) + ] + assert hashes[0] == hashes[1] + + def test_changed_bytes_at_the_same_path_change_the_hash(self, tmp_path): + path = tmp_path / "x.png" + + def hash_now(): + return message_hash( + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ), + None, + ) + + path.write_bytes(b"first version") + first = hash_now() + path.write_bytes(b"second version") + assert hash_now() != first + + def test_same_bytes_at_two_paths_share_the_stored_row(self, store, tmp_path): + for name in ("a.png", "b.png"): + path = tmp_path / name + path.write_bytes(b"SAME BYTES") + store.ensure_chain( + [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ) + ] + ) + assert store.db["messages"].count == 1 + + def test_attachment_chain_verifies(self, store, tmp_path): + path = tmp_path / "x.png" + path.write_bytes(b"PNG BYTES") + store.ensure_chain( + [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ), + llm.assistant("A fine image"), + ] + ) + assert store.verify() == [] + + def test_media_type_participates_in_identity(self, tmp_path): + path = tmp_path / "x.bin" + path.write_bytes(b"SAME BYTES") + + def hash_as(type_): + return message_hash( + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type=type_, path=str(path)) + ) + ], + ), + None, + ) + + # The model sees the media type: identical bytes sent as + # different types are different requests. + assert hash_as("image/png") != hash_as("text/plain") + + def test_cached_attachment_id_is_not_trusted(self, tmp_path): + path = tmp_path / "x.png" + path.write_bytes(b"first") + attachment = Attachment(type="image/png", path=str(path)) + attachment.id() # caches _id from the current bytes + message = Message(role="user", parts=[AttachmentPart(attachment=attachment)]) + first = message_hash(message, None) + path.write_bytes(b"second") + assert message_hash(message, None) != first + + def test_editing_the_file_after_logging_breaks_verify(self, store, tmp_path): + path = tmp_path / "x.png" + path.write_bytes(b"ORIGINAL") + tip = store.ensure_chain( + [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ) + ] + ) + assert store.verify() == [] + path.write_bytes(b"TAMPERED") + assert store.verify() == [tip] + + def test_deleting_the_file_is_detected_not_fatal(self, store, tmp_path): + path = tmp_path / "x.png" + path.write_bytes(b"ORIGINAL") + tip = store.ensure_chain( + [ + Message( + role="user", + parts=[ + AttachmentPart( + attachment=Attachment(type="image/png", path=str(path)) + ) + ], + ) + ] + ) + path.unlink() + assert store.verify() == [tip] + + +class TestRepeatedAttachments: + def test_a_tool_result_can_carry_the_same_attachment_twice(self, store): + attachment = Attachment(type="image/png", content=b"PNG BYTES") + message = Message( + role="tool", + parts=[ + ToolResultPart( + name="t", + output="ok", + tool_call_id="c1", + attachments=[attachment, attachment], + ) + ], + ) + tip = store.ensure_chain([message]) + assert store.db["part_attachments"].count == 2 + loaded = store.load_chain(tip) + assert len(loaded[0].parts[0].attachments) == 2 + assert store.verify() == [] + + +class TestPartStorageFormat: + """Literal text is stored raw in its own column - never escaped, + never parsed - and the JSON payload holds only structure, with the + type key left to the type column.""" + + def test_plain_text_stores_raw_text_and_no_payload(self, store): + text = '{"looks": "like json", "but": "is text"}' + store.ensure_chain([llm.user(text)]) + row = next(iter(store.db["parts"].rows)) + assert row["type"] == "text" + assert row["text"] == text + assert row["payload"] is None + + def test_redacted_reasoning_splits_text_from_structure(self, store): + message = Message( + role="assistant", + parts=[ReasoningPart(text="thinking", redacted=True)], + ) + tip = store.ensure_chain([message]) + row = next(iter(store.db["parts"].rows)) + assert row["text"] == "thinking" + assert json.loads(row["payload"]) == {"redacted": True} + assert store.load_chain(tip) == [message] + + def test_no_stored_payload_contains_a_type_key(self, store): + messages = [ + llm.user("hi"), + Message( + role="assistant", + parts=[ + ReasoningPart(text="thinking", redacted=True), + ToolCallPart(name="t", arguments={"a": 1}, tool_call_id="c1"), + ], + ), + Message( + role="tool", + parts=[ToolResultPart(name="t", output="ok", tool_call_id="c1")], + ), + ] + tip = store.ensure_chain(messages) + for row in store.db["parts"].rows: + if row["payload"] is not None: + assert "type" not in json.loads(row["payload"]) + assert store.load_chain(tip) == messages + assert store.verify() == [] + + def test_fragment_referencing_text_stays_structured(self, store): + novel = "CALL ME ISHMAEL. " * 20 + ensure_fragment(store.db, novel) + store.ensure_chain([llm.user(f"{novel}\nwho?")], fragments=[novel]) + row = next(iter(store.db["parts"].rows)) + assert row["text"] is None + assert json.loads(row["payload"]) == { + "text_ref": [{"fragment": 1}, {"literal": "\nwho?"}] + } + + +class TestFragmentReferences: + """The point of fragments is that a novel is stored once and pointed + at from every prompt about it, so the text must not be expanded into + each message that uses it.""" + + NOVEL = "CALL ME ISHMAEL. " * 500 + + def messages_using(self, fragment, question): + # What Prompt.prompt builds: fragments joined to the prompt text. + return [llm.user(f"{fragment}\n{question}")] + + def test_fragment_text_is_not_duplicated_into_the_part(self, store): + ensure_fragment(store.db, self.NOVEL) + store.ensure_chain( + self.messages_using(self.NOVEL, "who is the narrator?"), + fragments=[self.NOVEL], + ) + payload = next(iter(store.db["parts"].rows))["payload"] + assert self.NOVEL not in payload + assert len(payload) < 200 + + def test_many_prompts_about_one_fragment_store_it_once(self, store): + ensure_fragment(store.db, self.NOVEL) + for question in ("who?", "where?", "when?", "why?"): + store.ensure_chain( + self.messages_using(self.NOVEL, question), fragments=[self.NOVEL] + ) + assert store.db["fragments"].count == 1 + assert store.db["parts"].count == 4 + total = sum(len(row["payload"]) for row in store.db["parts"].rows) + assert total < len(self.NOVEL) + + def test_referenced_text_round_trips(self, store): + ensure_fragment(store.db, self.NOVEL) + messages = self.messages_using(self.NOVEL, "who is the narrator?") + tip = store.ensure_chain(messages, fragments=[self.NOVEL]) + assert store.load_chain(tip) == messages + + def test_several_fragments_in_one_part_round_trip(self, store): + one, two = "FIRST FRAGMENT", "SECOND FRAGMENT" + for content in (one, two): + ensure_fragment(store.db, content) + messages = [llm.user(f"{one}\n{two}\ncompare them")] + tip = store.ensure_chain(messages, fragments=[one, two]) + assert store.load_chain(tip) == messages + + def test_part_fragments_records_the_link(self, store): + ensure_fragment(store.db, self.NOVEL) + store.ensure_chain( + self.messages_using(self.NOVEL, "who?"), fragments=[self.NOVEL] + ) + assert store.db["part_fragments"].count == 1 + + def test_messages_using_a_fragment_are_one_join_away(self, store): + ensure_fragment(store.db, self.NOVEL) + for question in ("who?", "where?"): + store.ensure_chain( + self.messages_using(self.NOVEL, question), fragments=[self.NOVEL] + ) + fragment_id = next(iter(store.db["fragments"].rows))["id"] + found = list( + store.db.query( + """ + select distinct parts.message_hash from part_fragments + join parts on parts.id = part_fragments.part_id + where part_fragments.fragment_id = ? + """, + [fragment_id], + ) + ) + assert len(found) == 2 + + def test_unknown_fragments_are_stored_inline(self, store): + messages = [llm.user("just some text")] + tip = store.ensure_chain(messages) + assert store.db["part_fragments"].count == 0 + assert store.load_chain(tip) == messages + + def test_hashes_do_not_depend_on_where_the_bytes_live(self, store): + # Identity is the resolved content, so storing by reference must + # produce exactly the hash that storing inline would. + messages = self.messages_using(self.NOVEL, "who?") + inline = store.ensure_chain(messages) + ensure_fragment(store.db, self.NOVEL) + by_reference = store.ensure_chain(messages, fragments=[self.NOVEL]) + assert inline == by_reference + + +# ---- verification ---------------------------------------------------- + + +class TestVerify: + """Reads resolve references, so a reconstruction bug would produce a + chain that differs from what was hashed - and would do it silently. + Re-hashing every stored message catches the whole class at once.""" + + def test_a_fresh_store_verifies(self, store): + assert store.verify() == [] + + def test_every_kind_of_part_verifies(self, store): + novel = "CALL ME ISHMAEL. " * 100 + store.ensure_chain( + [ + llm.system("be brief"), + llm.user( + f"{novel}\nwho is the narrator?", + Attachment(type="image/png", content=b"bytes"), + ), + llm.assistant( + ReasoningPart(text="thinking", provider_metadata={"a": 1}), + ReasoningPart(redacted=True), + ToolCallPart(name="s", arguments={"q": 1}, tool_call_id="tc1"), + TextPart(text="answer"), + ), + llm.tool_message( + ToolResultPart( + name="s", + output="out", + tool_call_id="tc1", + exception="ValueError: x", + attachments=[Attachment(type="image/png", content=b"one")], + ) + ), + ], + fragments=[novel], + ) + assert store.verify() == [] + + def test_a_corrupted_part_is_caught(self, store): + tip = store.ensure_chain([llm.user("Hi")]) + with store.db.conn: + store.db.execute("update parts set text = 'tampered'") + assert store.verify() == [tip] + + def test_a_missing_fragment_is_caught(self, store): + novel = "CALL ME ISHMAEL. " * 100 + tip = store.ensure_chain([llm.user(f"{novel}\nwho?")], fragments=[novel]) + with store.db.conn: + store.db.execute("delete from fragments") + assert store.verify() == [tip] + + +class TestFragmentsEndToEnd: + + def test_a_conversation_chain_includes_fragment_text(self, mock_model): + # prompt.messages is meant to be exactly what the model sees, and + # what the model sees has the fragments concatenated in. + conversation = mock_model.conversation() + mock_model.enqueue(["ok"]) + response = conversation.prompt("question", fragments=["FRAGMENT-BODY"]) + response.text() + assert response.prompt.messages[-1].parts[0].text == response.prompt.prompt + + def test_the_cli_stores_a_fragment_by_reference(self, user_path, tmpdir): + novel = "CALL ME ISHMAEL. " * 3000 + path = tmpdir / "novel.txt" + path.write_text(novel, "utf-8") + for question in ("who?", "where?", "when?"): + run("-m", "echo", question, "-f", str(path)) + + db = sqlite_utils.Database(str(user_path / "logs.db")) + assert db["fragments"].count == 1 + assert db["part_fragments"].count >= 3 + # Every question re-sends the novel; it must be stored once. + user_payloads = sum( + len(row["payload"]) + for row in db.query( + "select payload from parts join messages" + " on messages.hash = parts.message_hash" + " where messages.role = 'user'" + ) + ) + assert user_payloads < len(novel) + assert LogStore(db).verify() == [] + + +# ---- async ------------------------------------------------------------ + + +class TestAsyncLogging: + """The CLI converts an async response to a sync one before logging, + so anything the conversion drops is dropped from the log.""" + + def enqueue_reasoning(self, model): + model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", + chunk="thinking hard", + provider_metadata={"anthropic": {"signature": "SIG"}}, + ), + llm.parts.StreamEvent(type="text", chunk="the answer"), + ] + ) + + async def respond(self, model, prompt="q"): + response = model.prompt(prompt) + await response.text() + return response + + @pytest.mark.asyncio + async def test_to_sync_response_keeps_the_parts(self, async_mock_model): + self.enqueue_reasoning(async_mock_model) + response = await self.respond(async_mock_model) + before = response._messages_now() + after = (await response.to_sync_response())._messages_now() + assert after == before + + @pytest.mark.asyncio + async def test_logging_an_async_response_keeps_reasoning( + self, store, async_mock_model + ): + self.enqueue_reasoning(async_mock_model) + response = await self.respond(async_mock_model) + store.log(await response.to_sync_response()) + + parts = store.load_chain( + next(iter(store.db["turns"].rows))["tip_message_hash"] + )[-1].parts + assert [type(part).__name__ for part in parts] == [ + "ReasoningPart", + "TextPart", + ] + assert parts[0].provider_metadata == {"anthropic": {"signature": "SIG"}} + assert store.verify() == [] + + +# ---- llm logs against the new tables --------------------------------- + + +def forget_legacy(user_path): + """Empty the table the old `llm logs` reads from. + + Every test below runs this first, so a passing assertion can only + have been served by the content-addressed tables. Without it these + tests pass against the legacy path and prove nothing. + """ + db = sqlite_utils.Database(str(user_path / "logs.db")) + with db.conn: + db.execute("delete from responses") + db.close() + + +class TestLogsCommand: + """`llm logs` reads the content-addressed tables only. Conversations + logged before this schema existed are deliberately not shown yet.""" + + def test_shows_a_logged_prompt(self, user_path): + run("-m", "echo", "hello there") + forget_legacy(user_path) + assert "hello there" in run("logs", "-n", "1").output + + def test_json_output_carries_the_turn(self, user_path): + run("-m", "echo", "hello there") + forget_legacy(user_path) + rows = json.loads(run("logs", "-n", "1", "--json").output) + assert len(rows) == 1 + assert rows[0]["model"] == "echo" + assert rows[0]["prompt"] == "hello there" + assert "hello there" in rows[0]["response"] + + def test_count_limits_results(self, user_path): + for word in ("one", "two", "three"): + run("-m", "echo", word) + forget_legacy(user_path) + assert len(json.loads(run("logs", "-n", "2", "--json").output)) == 2 + + def test_results_are_chronological(self, user_path): + for word in ("one", "two", "three"): + run("-m", "echo", word) + forget_legacy(user_path) + rows = json.loads(run("logs", "-n", "0", "--json").output) + assert [r["prompt"] for r in rows] == ["one", "two", "three"] + + def test_filters_by_model(self, user_path): + run("-m", "echo", "hello") + forget_legacy(user_path) + assert json.loads(run("logs", "-m", "echo", "--json").output) + assert json.loads(run("logs", "-m", "gpt-4o", "--json").output) == [] + + def test_filters_to_a_conversation(self, user_path): + run("-m", "echo", "first") + run("-c", "second") + run("-m", "echo", "unrelated") + forget_legacy(user_path) + db = sqlite_utils.Database(str(user_path / "logs.db")) + thread_id = next( + iter(db.query("select thread_id from turns order by id limit 1")) + )["thread_id"] + rows = json.loads(run("logs", "--cid", thread_id, "--json").output) + assert [r["prompt"] for r in rows] == ["first", "second"] + + def test_filters_by_fragment(self, user_path, tmpdir): + path = tmpdir / "frag.txt" + path.write_text("FRAGMENT BODY", "utf-8") + run("-m", "echo", "with fragment", "-f", str(path)) + run("-m", "echo", "without fragment") + forget_legacy(user_path) + + db = sqlite_utils.Database(str(user_path / "logs.db")) + fragment_hash = next(iter(db["fragments"].rows))["hash"] + rows = json.loads(run("logs", "-f", fragment_hash, "--json").output) + # The stored prompt is the resolved text the model was sent. + assert [r["prompt"] for r in rows] == ["FRAGMENT BODY\nwith fragment"] + + def test_filters_by_tool(self, user_path): + run( + "-m", + "echo", + '{"tool_calls": [{"name": "llm_version"}]}', + "-T", + "llm_version", + ) + run("-m", "echo", "no tools here") + forget_legacy(user_path) + assert len(json.loads(run("logs", "-T", "llm_version", "--json").output)) == 1 + + def test_usage_is_reported(self, user_path): + run("-m", "echo", "hello") + forget_legacy(user_path) + rows = json.loads(run("logs", "--json").output) + assert "input_tokens" in rows[0] + assert "datetime_utc" in rows[0] + + +class TestPayloadOrdering: + def test_tool_call_argument_order_is_preserved(self, store): + # canonical_json sorts keys - that is for hashing, not storage. + # Arguments come back in the order the model produced them. + messages = [ + llm.assistant( + ToolCallPart( + name="demo", + arguments={"timeout": 120, "options": ["tick"]}, + tool_call_id="tc1", + ) + ) + ] + tip = store.ensure_chain(messages) + assert list(store.load_chain(tip)[0].parts[0].arguments) == [ + "timeout", + "options", + ] + + def test_hashing_still_ignores_key_order(self, store): + one = store.ensure_chain( + [llm.assistant(ToolCallPart(name="d", arguments={"a": 1, "b": 2}))] + ) + two = store.ensure_chain( + [llm.assistant(ToolCallPart(name="d", arguments={"b": 2, "a": 1}))] + ) + assert one == two + assert store.db["messages"].count == 1 + + +# ---- the message_tree view ------------------------------------------- + + +class TestMessageTreeView: + def rows(self, store): + return list(store.db.query("select * from message_tree order by path")) + + def test_forks_render_as_indented_siblings(self, store): + thread = store.create_thread() + fork_point = store.append( + thread, [llm.user("Question"), llm.assistant("Answer")] + ) + store.append(thread, [llm.user("Follow-up A")]) + forked = store.fork(fork_point) + store.append(forked, [llm.user("Follow-up B")]) + + assert [row["message"] for row in self.rows(store)] == [ + "Question", + " Answer", + " Follow-up A", + " Follow-up B", + ] + + def test_every_message_carries_its_tree_root_hash(self, store): + store.append(store.create_thread(), [llm.user("One"), llm.assistant("1")]) + store.append(store.create_thread(), [llm.user("Two")]) + + rows = self.rows(store) + assert [row["message"] for row in rows] == ["One", " 1", "Two"] + one, reply, two = rows + # A root is its own root; descendants inherit it; separate + # conversations get separate roots. + assert one["root_hash"] == one["message_hash"] + assert reply["root_hash"] == one["message_hash"] + assert two["root_hash"] == two["message_hash"] + + def test_tool_results_show_their_tool_names(self, store): + store.append( + store.create_thread(), + [ + llm.user("Search"), + Message( + role="tool", + parts=[ + ToolResultPart(name="lookup", output="42", tool_call_id="c1"), + ToolResultPart(name="fetch", output="x", tool_call_id="c2"), + ], + ), + ], + ) + prompt, tool = self.rows(store) + assert prompt["tools"] == "" + assert tool["message"].strip() == "[tool_result]" + assert sorted(tool["tools"].split(", ")) == ["fetch", "lookup"] + + def test_fragment_referenced_text_is_resolved(self, store): + novel = "Call me Ishmael, but keep this fragment out of the parts table." + store.ensure_chain([llm.user(f"{novel}\nwho?")], fragments=[novel]) + (row,) = self.rows(store) + # The stored part holds a text_ref, not the text itself; the view + # displays the fragment's content in its place. + assert row["message"] == novel[:60] + + def test_text_is_flattened_and_truncated(self, store): + store.append( + store.create_thread(), [llm.user("line one\nline two " + "x" * 100)] + ) + (row,) = self.rows(store) + assert row["message"] == ("line one line two " + "x" * 100)[:60] + + def test_datetime_comes_from_the_turn_that_logged_the_message( + self, store, mock_model + ): + mock_model.enqueue(["Hello"]) + response = mock_model.prompt("Hi") + response.text() + store.log(response) + rows = self.rows(store) + # Both messages were recorded by the same turn, so they share + # its timestamp. + assert len({row["datetime"] for row in rows}) == 1 + assert rows[0]["datetime"] is not None + + def test_messages_no_turn_has_recorded_have_no_datetime(self, store): + store.append(store.create_thread(), [llm.user("Hi")]) + (row,) = self.rows(store) + assert row["datetime"] is None + + +# ---- condensed provider payloads ------------------------------------- + + +class TestResponseJsonPayload: + """The raw response.json() payload is stored on the turn, condensed + against the strings the turn's own messages already hold, and + resolved again on the way out.""" + + LONG = "SQLite stores the whole database in a single file on disk. " * 3 + + def logged(self, store, mock_model, payload, text=None): + mock_model.enqueue([text if text is not None else self.LONG]) + response = mock_model.prompt("Tell me about SQLite") + response.text() + response.response_json = payload + return store.log(response) + + def test_payload_is_stored_condensed(self, store, mock_model): + turn_id = self.logged( + store, + mock_model, + {"content": self.LONG, "id": "chatcmpl-1", "usage": {"total_tokens": 9}}, + ) + stored = store.db["turns"].get(turn_id)["response_json"] + # The response text is a reference, not a second copy + assert json.loads(stored)["content"] == {"$": "0.0.text"} + assert self.LONG not in stored + + def test_turn_response_json_resolves_the_payload(self, store, mock_model): + payload = {"content": self.LONG, "id": "chatcmpl-1"} + turn_id = self.logged(store, mock_model, payload) + assert store.turn_response_json(turn_id) == payload + + def test_no_payload_stores_null(self, store, mock_model): + turn_id = self.logged(store, mock_model, None) + assert store.db["turns"].get(turn_id)["response_json"] is None + assert store.turn_response_json(turn_id) is None + + def test_unknown_turn_resolves_to_none(self, store): + assert store.turn_response_json("nope") is None + + def test_short_strings_are_stored_verbatim(self, store, mock_model): + # Below the length threshold a reference would cost as much as + # the string it replaces. + turn_id = self.logged( + store, mock_model, {"content": "short answer"}, text="short answer" + ) + stored = store.db["turns"].get(turn_id)["response_json"] + assert json.loads(stored) == {"content": "short answer"} + + def test_marker_shaped_payload_round_trips(self, store, mock_model): + # A payload that already contains {"$": ...} shapes must come + # back exactly - condense-json escapes them with $raw. + payload = {"tricky": {"$": "not-a-marker"}, "also": {"$r": ["x"]}} + turn_id = self.logged(store, mock_model, payload) + assert store.turn_response_json(turn_id) == payload + + def test_merged_log_rows_carry_the_resolved_payload(self, store, mock_model): + payload = {"content": self.LONG, "id": "chatcmpl-1"} + self.logged(store, mock_model, payload) + (row,) = merged_log_rows(store) + assert json.loads(row["response_json"]) == payload + + def test_a_payload_that_no_longer_resolves_is_absent_from_rows( + self, store, mock_model + ): + turn_id = self.logged(store, mock_model, {"content": self.LONG}) + store.db["turns"].update( + turn_id, {"response_json": json.dumps({"content": {"$": "9.9.text"}})} + ) + (row,) = merged_log_rows(store) + assert row["response_json"] is None + # The API surfaces the failure instead of guessing + from condense_json import UncondenseError + + with pytest.raises(UncondenseError): + store.turn_response_json(turn_id) + + +class TestPayloadReplacements: + """_payload_replacements builds the same dict at write and read time + from the turn's chain segment - these pin down what it offers.""" + + def test_part_strings_are_keyed_by_position(self): + from llm.logs import _payload_replacements + + blob = "b" * 80 + messages = [ + llm.assistant( + llm.parts.ReasoningPart( + text="r" * 70, + provider_metadata={"openai": {"encrypted_content": blob}}, + ), + "t" * 64, + ) + ] + replacements = _payload_replacements(messages) + # Leaf strings by path, plus each provider_metadata container + # offered structurally so a payload embedding the whole object + # condenses to one reference. + assert replacements == { + "0.0.text": "r" * 70, + "0.0.pm": {"openai": {"encrypted_content": blob}}, + "0.0.pm.openai": {"encrypted_content": blob}, + "0.0.pm.openai.encrypted_content": blob, + "0.1.text": "t" * 64, + } + + def test_tool_arguments_offer_both_serializations(self): + from llm.logs import _payload_replacements + + arguments = {"query": "x" * 64} + messages = [ + llm.assistant(llm.parts.ToolCallPart(name="search", arguments=arguments)) + ] + replacements = _payload_replacements(messages) + assert replacements["0.0.args"] == json.dumps(arguments, separators=(",", ":")) + assert replacements["0.0.args2"] == json.dumps(arguments) + + def test_tool_result_output_is_offered(self): + from llm.logs import _payload_replacements + + output = "o" * 64 + messages = [ + llm.tool_message(llm.parts.ToolResultPart(name="search", output=output)) + ] + assert _payload_replacements(messages) == {"0.0.output": output} + + def test_short_strings_are_excluded(self): + from llm.logs import _payload_replacements + + assert _payload_replacements([llm.assistant("short")]) == {} + + def test_tool_descriptions_are_offered(self): + from llm.logs import _payload_replacements + + description = "d" * 64 + replacements = _payload_replacements([], tools=[("search", description)]) + assert replacements == {"tool.search.description": description} + + def test_conflicting_tool_descriptions_are_dropped(self): + from llm.logs import _payload_replacements + + # Whichever order the pairs arrive in, a name that carries two + # different long descriptions is excluded on both sides. + pairs = [("search", "a" * 64), ("search", "b" * 64), ("other", "c" * 64)] + for ordering in (pairs, list(reversed(pairs))): + assert _payload_replacements([], tools=ordering) == { + "tool.other.description": "c" * 64 + } + + def test_short_or_missing_tool_descriptions_are_excluded(self): + from llm.logs import _payload_replacements + + assert _payload_replacements([], tools=[("a", "short"), ("b", None)]) == {} + + +class TestResponseJsonToolEcho: + """Providers echo the turn's tool definitions back in the payload; + long descriptions condense against the tools table.""" + + DESCRIPTION = ( + "Execute JavaScript code using a persistent context. State is " + "maintained between calls, allowing variables to persist." + ) + + def logged(self, store, mock_model): + def execute_javascript(javascript: str) -> str: + return "ran" + + tool = llm.Tool.function(execute_javascript, description=self.DESCRIPTION) + mock_model.enqueue(["ok"]) + response = mock_model.prompt("run it", tools=[tool]) + response.text() + response.response_json = { + "content": "ok", + "tools": [ + { + "name": "execute_javascript", + "description": self.DESCRIPTION, + "parameters": {"type": "object"}, + } + ], + } + return store.log(response), response.response_json + + def test_the_echoed_description_is_a_reference(self, store, mock_model): + turn_id, _ = self.logged(store, mock_model) + stored = json.loads(store.db["turns"].get(turn_id)["response_json"]) + assert stored["tools"][0]["description"] == { + "$": "tool.execute_javascript.description" + } + + def test_the_payload_resolves_via_the_turn_tools_join(self, store, mock_model): + turn_id, payload = self.logged(store, mock_model) + assert store.turn_response_json(turn_id) == payload + (row,) = merged_log_rows(store) + assert json.loads(row["response_json"]) == payload + + +def _structural_schema(): + return { + "type": "object", + "properties": {"name": {"type": "string", "description": "d" * 40}}, + "required": ["name"], + } + + +class TestResponseJsonStructural: + """Structural replacement values (condense-json 1.1): payload + subtrees that equal stored content condense whole, regardless of + key order or serialization.""" + + def test_schema_echo_condenses_against_the_schemas_table(self, store, mock_model): + schema = _structural_schema() + mock_model.enqueue(['{"name": "Cleo"}']) + response = mock_model.prompt("Extract", schema=schema) + response.text() + # The echo reorders keys, as providers do - structural equality + # still matches + echoed = { + "required": ["name"], + "properties": {"name": {"type": "string", "description": "d" * 40}}, + "type": "object", + } + payload = {"text": {"format": {"type": "json_schema", "schema": echoed}}} + response.response_json = payload + turn_id = store.log(response) + stored = json.loads(store.db["turns"].get(turn_id)["response_json"]) + assert stored["text"]["format"]["schema"] == {"$": "schema"} + # Resolves via the schemas table; the substituted form is the + # stored schema's key order, structurally equal to the echo + resolved = store.turn_response_json(turn_id) + assert resolved["text"]["format"]["schema"] == schema + (row,) = merged_log_rows(store) + row_payload = json.loads(row["response_json"]) + assert row_payload["text"]["format"]["schema"] == schema + + def test_object_form_tool_arguments_condense(self, store, mock_model): + # Anthropic and Gemini embed tool arguments as objects rather + # than JSON-encoded strings + arguments = {"javascript": "x" * 64} + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "run", + tool_results=[ + llm.ToolResult(name="execute", output="42", tool_call_id="t1") + ], + messages=[ + llm.assistant( + llm.parts.ToolCallPart( + name="execute", arguments=arguments, tool_call_id="t1" + ) + ) + ], + ) + response.text() + payload = {"content": [{"type": "tool_use", "input": dict(arguments)}]} + response.response_json = payload + turn_id = store.log(response) + assert store.turn_response_json(turn_id) == payload + + def test_provider_metadata_container_condenses_whole(self, store, mock_model): + summary = [{"type": "summary_text", "text": "s" * 60}] + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", + chunk="thinking", + provider_metadata={"openai": {"summary": summary}}, + ), + "ok", + ] + ) + response = mock_model.prompt("hi") + response.text() + payload = {"output": [{"type": "reasoning", "summary": summary}]} + response.response_json = payload + turn_id = store.log(response) + stored = store.db["turns"].get(turn_id)["response_json"] + # One reference for the whole list, not one per inner string + assert '"$": "0.0.pm.openai.summary"' in stored + assert store.turn_response_json(turn_id) == payload + + +class TestModelJsonReplacements: + """Model classes can declare a json_replacements dictionary of + recurring payload boilerplate - like a zstd custom dictionary - + resolved by looking the model up again at read time.""" + + @staticmethod + def boiler(): + return { + "image_gen": {"input_tokens": 0, "output_tokens": 0}, + "web_search": {"num_requests": 0}, + } + + def test_model_boilerplate_condenses_and_resolves(self, store, mock_model): + mock_model.json_replacements = {"tool_usage_0": self.boiler()} + try: + mock_model.enqueue(["ok"]) + response = mock_model.prompt("hi") + response.text() + payload = { + "content": "ok", + # Key order differs from the declared entry - structural + # matching still applies + "tool_usage": { + "web_search": {"num_requests": 0}, + "image_gen": {"output_tokens": 0, "input_tokens": 0}, + }, + } + response.response_json = payload + turn_id = store.log(response) + stored = store.db["turns"].get(turn_id)["response_json"] + assert json.loads(stored)["tool_usage"] == {"$": "m.tool_usage_0"} + resolved = store.turn_response_json(turn_id) + assert resolved["tool_usage"] == self.boiler() + (row,) = merged_log_rows(store) + assert json.loads(row["response_json"])["tool_usage"] == self.boiler() + finally: + del mock_model.json_replacements + + def test_unknown_model_fails_closed(self, store, mock_model): + mock_model.json_replacements = {"tool_usage_0": self.boiler()} + try: + mock_model.enqueue(["ok"]) + response = mock_model.prompt("hi") + response.text() + response.response_json = {"tool_usage": self.boiler()} + turn_id = store.log(response) + finally: + del mock_model.json_replacements + # The model is gone (or its plugin uninstalled) at read time + store.db.execute( + "update turns set model = 'gone-model' where id = ?", [turn_id] + ) + from condense_json import UncondenseError + + with pytest.raises(UncondenseError): + store.turn_response_json(turn_id) + (row,) = merged_log_rows(store) + assert row["response_json"] is None diff --git a/tests/test_migrate.py b/tests/test_migrate.py index e7f70bc3e..330f51a12 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -1,9 +1,9 @@ -import llm -from llm.migrations import migrate -from llm.embeddings_migrations import embeddings_migrations import pytest import sqlite_utils +import llm +from llm.embeddings_migrations import embeddings_migrations +from llm.migrations import migrate EXPECTED = { "id": str, @@ -22,6 +22,7 @@ "output_tokens": int, "token_details": str, "schema_id": str, + "reasoning": str, } @@ -49,6 +50,9 @@ def test_migrate_blank(): "responses_ai", "responses_ad", "responses_au", + "turn_search_ai", + "turn_search_ad", + "turn_search_au", } @@ -89,6 +93,9 @@ def test_migrate_from_original_schema(has_record): "responses_ai", "responses_ad", "responses_au", + "turn_search_ai", + "turn_search_ad", + "turn_search_au", } diff --git a/tests/test_openai_endpoint.py b/tests/test_openai_endpoint.py new file mode 100644 index 000000000..d263edf10 --- /dev/null +++ b/tests/test_openai_endpoint.py @@ -0,0 +1,1134 @@ +import base64 +import json + +import pytest +import sqlite_utils +from click.testing import CliRunner +from pytest_httpx import IteratorStream + +from llm.cli import cli +from llm.migrations import migrate + + +def _add_chat_response(httpx_mock, url, text): + httpx_mock.add_response( + method="POST", + url=f"{url}/chat/completions", + json={ + "id": "chatcmpl_test", + "object": "chat.completion", + "created": 1, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": text}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 3, + "completion_tokens": 2, + "total_tokens": 5, + }, + }, + headers={"Content-Type": "application/json"}, + ) + + +def _add_chat_tool_call_response(httpx_mock, url, name, arguments): + httpx_mock.add_response( + method="POST", + url=f"{url}/chat/completions", + json={ + "id": "chatcmpl_tool_test", + "object": "chat.completion", + "created": 1, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_test", + "type": "function", + "function": { + "name": name, + "arguments": json.dumps(arguments), + }, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + "usage": { + "prompt_tokens": 3, + "completion_tokens": 2, + "total_tokens": 5, + }, + }, + headers={"Content-Type": "application/json"}, + ) + + +def _responses_payload(text): + return { + "id": "resp_test", + "object": "response", + "created_at": 1, + "model": "test-model", + "output": [ + { + "type": "message", + "id": "msg_test", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 3, + "output_tokens": 2, + "total_tokens": 5, + }, + "status": "completed", + } + + +def _responses_tool_call_payload(name, arguments): + return { + "id": "resp_tool_test", + "object": "response", + "created_at": 1, + "model": "test-model", + "output": [ + { + "type": "function_call", + "id": "fc_test", + "call_id": "call_test", + "name": name, + "arguments": json.dumps(arguments), + "status": "completed", + } + ], + "usage": { + "input_tokens": 3, + "output_tokens": 2, + "total_tokens": 5, + }, + "status": "completed", + } + + +def _chat_stream_events(): + for delta, finish_reason in ( + ({"role": "assistant", "content": ""}, None), + ({"content": "Hello"}, None), + ({"content": " streamed"}, None), + ({}, "stop"), + ): + yield "data: {}\n\n".format( + json.dumps( + { + "id": "chatcmpl_stream_test", + "object": "chat.completion.chunk", + "created": 1, + "model": "test-model", + "choices": [ + { + "index": 0, + "delta": delta, + "finish_reason": finish_reason, + } + ], + } + ) + ).encode("utf-8") + yield b"data: [DONE]\n\n" + + +def test_endpoint_chat_completions_does_not_log_or_leak_default_key( + httpx_mock, user_path, monkeypatch +): + base_url = "https://example.test/v1" + _add_chat_response(httpx_mock, base_url, "Hello from the endpoint") + monkeypatch.setenv("OPENAI_API_KEY", "real-default-openai-key") + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Hello", + "-m", + "test-model", + "--no-stream", + "-H", + "X-Test", + "one", + "-o", + "reasoning_effort", + "low", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Hello from the endpoint\n" + assert not (user_path / "logs.db").exists() + + request = httpx_mock.get_requests()[0] + assert request.headers["Authorization"] == "Bearer DUMMY_KEY" + assert request.headers["X-Test"] == "one" + assert json.loads(request.content) == { + "messages": [{"role": "user", "content": "Hello"}], + "model": "test-model", + "reasoning_effort": "low", + "stream": False, + } + + +def test_endpoint_chat_completions_attachment(httpx_mock, user_path, tmp_path): + base_url = "https://attachments.example.test/v1" + _add_chat_response(httpx_mock, base_url, "A test image") + image_bytes = b"\x89PNG\r\n\x1a\nendpoint attachment" + image_path = tmp_path / "image.png" + image_path.write_bytes(image_bytes) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Describe this", + "-m", + "test-model", + "--no-stream", + "-a", + str(image_path), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "A test image\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content)["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,{}".format( + base64.b64encode(image_bytes).decode("ascii") + ) + }, + }, + ], + } + ] + + +def test_endpoint_template(httpx_mock, user_path, templates_path): + base_url = "https://templates.example.test/v1" + _add_chat_response(httpx_mock, base_url, "Template response") + (templates_path / "endpoint.yaml").write_text( + """ +model: template-model +system: You are $persona +prompt: "Question: $input" +options: + temperature: 0.4 +schema_object: + type: object + properties: + answer: + type: string + required: + - answer +attachment_types: +- type: image/jpeg + value: https://images.example.test/template.jpg +""".strip(), + "utf-8", + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Where?", + "--template", + "endpoint", + "--schema", + '{"type": "object"}', + "--param", + "persona", + "concise", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Template response\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content) == { + "messages": [ + {"role": "system", "content": "You are concise"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Question: Where?"}, + { + "type": "image_url", + "image_url": { + "url": "https://images.example.test/template.jpg" + }, + }, + ], + }, + ], + "model": "template-model", + # CLI --schema takes precedence over template schema_object + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "output", + "schema": {"type": "object"}, + }, + }, + "stream": False, + "temperature": 0.4, + } + + +def test_endpoint_template_schema_object_used_when_no_cli_schema( + httpx_mock, user_path, templates_path +): + base_url = "https://templates2.example.test/v1" + _add_chat_response(httpx_mock, base_url, "Template response 2") + (templates_path / "schemaonly.yaml").write_text( + """ +model: schema-model +schema_object: + type: object + properties: + answer: + type: string + required: + - answer +""".strip(), + "utf-8", + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "test question", + "--template", + "schemaonly", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["response_format"] == { + "type": "json_schema", + "json_schema": { + "name": "output", + "schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + }, + }, + } + + +def test_endpoint_schema(httpx_mock, user_path): + base_url = "https://schema.example.test/v1" + _add_chat_response(httpx_mock, base_url, '{"name": "Cleo", "age": 10}') + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Invent a dog", + "-m", + "test-model", + "--schema", + "name, age int", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert not (user_path / "logs.db").exists() + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["response_format"] == { + "type": "json_schema", + "json_schema": { + "name": "output", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, + "required": ["name", "age"], + }, + }, + } + + +def test_endpoint_schema_by_id_from_existing_logs_database(httpx_mock, user_path): + base_url = "https://schema-id.example.test/v1" + _add_chat_response(httpx_mock, base_url, '{"name": "Cleo"}') + schema = { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + } + db = sqlite_utils.Database(str(user_path / "logs.db")) + migrate(db) + db["schemas"].insert({"id": "dog-schema", "content": json.dumps(schema)}) + assert (user_path / "logs.db").exists() + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Invent a dog", + "-m", + "test-model", + "--schema", + "dog-schema", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["response_format"]["json_schema"]["schema"] == schema + assert db["responses"].count == 0 + + +def test_endpoint_invalid_schema_id_does_not_create_logs_database(user_path): + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + "https://schema-id.example.test/v1", + "Invent a dog", + "-m", + "test-model", + "--schema", + "missing-schema", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 2 + assert "Invalid schema" in result.output + assert not (user_path / "logs.db").exists() + + +def test_endpoint_static_template_runs_once_on_terminal( + httpx_mock, user_path, templates_path, monkeypatch +): + base_url = "https://terminal-template.example.test/v1" + _add_chat_response(httpx_mock, base_url, "Five pelicans") + (templates_path / "pelican.yaml").write_text( + "prompt: List five pelican names\n", "utf-8" + ) + monkeypatch.setattr("click.testing._NamedTextIOWrapper.isatty", lambda self: True) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "-m", + "test-model", + "-t", + "pelican", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Five pelicans\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content)["messages"] == [ + {"role": "user", "content": "List five pelican names"} + ] + + +def test_endpoint_tools_and_functions(httpx_mock, user_path): + base_url = "https://tools.example.test/v1" + _add_chat_tool_call_response(httpx_mock, base_url, "multiply", {"a": 6, "b": 7}) + _add_chat_response(httpx_mock, base_url, "The answer is 42") + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "What is 6 * 7?", + "-m", + "test-model", + "--no-stream", + "-T", + "llm_version", + "--functions", + ( + "def multiply(a: int, b: int) -> int:\n" + ' "Multiply two numbers."\n' + " return a * b\n" + ), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "The answer is 42\n" + assert not (user_path / "logs.db").exists() + + requests = [json.loads(request.content) for request in httpx_mock.get_requests()] + assert len(requests) == 2 + assert {tool["function"]["name"] for tool in requests[0]["tools"]} == { + "llm_version", + "multiply", + } + assert requests[1]["messages"] == [ + {"role": "user", "content": "What is 6 * 7?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_test", + "type": "function", + "function": { + "name": "multiply", + "arguments": '{"a": 6, "b": 7}', + }, + } + ], + }, + { + "role": "tool", + "content": "42", + "tool_call_id": "call_test", + }, + ] + + +def test_endpoint_streams_by_default(httpx_mock, user_path): + base_url = "https://stream.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/chat/completions", + stream=IteratorStream(_chat_stream_events()), + headers={"Content-Type": "text/event-stream"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Hello", + "-m", + "test-model", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Hello streamed\n" + assert not (user_path / "logs.db").exists() + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["stream"] is True + assert request_body["stream_options"] == {"include_usage": True} + + +def test_endpoint_uses_explicit_key(httpx_mock, user_path): + base_url = "https://example.test/v1" + _add_chat_response(httpx_mock, base_url, "Authenticated") + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Hello", + "-m", + "test-model", + "--no-stream", + "--key", + "endpoint-key", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert httpx_mock.get_requests()[0].headers["Authorization"] == ( + "Bearer endpoint-key" + ) + assert not (user_path / "logs.db").exists() + + +def test_endpoint_lists_models_without_model_or_logging( + httpx_mock, user_path, monkeypatch +): + base_url = "https://models.example.test/v1" + monkeypatch.setenv("OPENAI_API_KEY", "real-default-openai-key") + httpx_mock.add_response( + method="GET", + url=f"{base_url}/models", + json={ + "object": "list", + "data": [ + { + "id": "first-model", + "object": "model", + "created": 1, + "owned_by": "example", + }, + { + "id": "second-model", + "object": "model", + "created": 2, + "owned_by": "example", + }, + ], + }, + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "--models", + "-H", + "X-Test", + "one", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "first-model\nsecond-model\n" + assert not (user_path / "logs.db").exists() + request = httpx_mock.get_requests()[0] + assert request.headers["Authorization"] == "Bearer DUMMY_KEY" + assert request.headers["X-Test"] == "one" + + +def test_endpoint_models_surfaces_error_from_successful_response(httpx_mock, user_path): + base_url = "https://models-error.example.test" + httpx_mock.add_response( + method="GET", + url=f"{base_url}/models", + json={"error": "Unexpected endpoint or method. (GET /models)"}, + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + ["openai", "endpoint", base_url, "--models"], + catch_exceptions=False, + ) + + assert result.exit_code == 1 + assert result.output == "Error: Unexpected endpoint or method. (GET /models)\n" + assert not (user_path / "logs.db").exists() + + +def test_endpoint_responses_api(httpx_mock, user_path): + base_url = "https://responses.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload("Hello from Responses"), + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Hello", + "-m", + "test-model", + "--responses", + "--no-stream", + "-o", + "verbosity", + "low", + "-o", + "reasoning_effort", + "low", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Hello from Responses\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content) == { + "input": [{"role": "user", "content": "Hello"}], + "include": ["reasoning.encrypted_content"], + "model": "test-model", + "reasoning": {"effort": "low"}, + "store": False, + "stream": False, + "text": {"verbosity": "low"}, + } + + +@pytest.mark.parametrize("reasoning_summary", ("auto", "concise", "detailed")) +def test_endpoint_responses_reasoning_summary_option( + httpx_mock, user_path, reasoning_summary +): + base_url = "https://responses-summary.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload("Hello from Responses"), + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Hello", + "-m", + "test-model", + "--responses", + "--no-stream", + "-o", + "reasoning_summary", + reasoning_summary, + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Hello from Responses\n" + assert not (user_path / "logs.db").exists() + assert json.loads(httpx_mock.get_requests()[0].content) == { + "input": [{"role": "user", "content": "Hello"}], + "include": ["reasoning.encrypted_content"], + "model": "test-model", + "reasoning": {"summary": reasoning_summary}, + "store": False, + "stream": False, + } + + +def test_endpoint_responses_reasoning_summary_option_rejects_invalid_value( + httpx_mock, user_path +): + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + "https://responses-summary.example.test/v1", + "Hello", + "-m", + "test-model", + "--responses", + "--no-stream", + "-o", + "reasoning_summary", + "verbose", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 1 + assert result.output == ( + "Error: reasoning_summary\n" + " Input should be 'auto', 'concise' or 'detailed'\n" + ) + assert not httpx_mock.get_requests() + assert not (user_path / "logs.db").exists() + + +def test_endpoint_responses_api_attachment(httpx_mock, user_path): + base_url = "https://responses-attachments.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload("A remote image"), + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Describe this", + "-m", + "test-model", + "--responses", + "--no-stream", + "--at", + "https://images.example.test/test.jpg", + "image/jpeg", + "-o", + "image_detail", + "original", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "A remote image\n" + assert not (user_path / "logs.db").exists() + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert "include" not in request_body + assert "reasoning" not in request_body + assert request_body["input"] == [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Describe this"}, + { + "type": "input_image", + "image_url": "https://images.example.test/test.jpg", + "detail": "original", + }, + ], + } + ] + + +def test_endpoint_responses_api_schema_multi(httpx_mock, user_path): + base_url = "https://responses-schema.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload('{"items": [{"name": "Cleo", "age": 10}]}'), + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Invent a dog", + "-m", + "test-model", + "--responses", + "--schema-multi", + "name, age int", + "--no-stream", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert not (user_path / "logs.db").exists() + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["text"]["format"] == { + "type": "json_schema", + "name": "output", + "schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, + "required": ["name", "age"], + }, + } + }, + "required": ["items"], + }, + "strict": False, + } + + +def test_endpoint_responses_api_tools(httpx_mock, user_path): + base_url = "https://responses-tools.example.test/v1" + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_tool_call_payload("multiply", {"a": 6, "b": 7}), + headers={"Content-Type": "application/json"}, + ) + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=_responses_payload("The answer is 42"), + headers={"Content-Type": "application/json"}, + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "What is 6 * 7?", + "-m", + "test-model", + "--responses", + "--no-stream", + "--functions", + ( + "def multiply(a: int, b: int) -> int:\n" + ' "Multiply two numbers."\n' + " return a * b\n" + ), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "The answer is 42\n" + assert not (user_path / "logs.db").exists() + + requests = [json.loads(request.content) for request in httpx_mock.get_requests()] + assert requests[0]["tools"][0]["name"] == "multiply" + assert requests[1]["input"] == [ + {"role": "user", "content": "What is 6 * 7?"}, + { + "type": "function_call", + "call_id": "call_test", + "name": "multiply", + "arguments": '{"a": 6, "b": 7}', + }, + { + "type": "function_call_output", + "call_id": "call_test", + "output": "42", + }, + ] + + +def test_endpoint_responses_api_raw_server_side_tool(httpx_mock, user_path): + base_url = "https://raw-tools.example.test/v1" + response_payload = _responses_payload("Search complete") + response_payload["output"].insert( + 0, + { + "id": "st_test", + "type": "openrouter:web_search", + "status": "completed", + "action": { + "type": "search", + "query": "latest news", + "sources": [{"type": "url", "url": "https://example.test/news"}], + }, + }, + ) + httpx_mock.add_response( + method="POST", + url=f"{base_url}/responses", + json=response_payload, + headers={"Content-Type": "application/json"}, + ) + tool_spec = { + "type": "openrouter:web_search", + "parameters": {"engine": "exa", "max_results": 2, "max_uses": 1}, + } + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "Search for the latest news", + "-m", + "test-model", + "--responses", + "--no-stream", + "-T", + f"ServerSideTool(spec={json.dumps(tool_spec)})", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "Search complete\n" + assert not (user_path / "logs.db").exists() + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["tools"] == [tool_spec] + + +def test_endpoint_reads_one_off_prompt_from_stdin(httpx_mock, user_path): + base_url = "https://stdin.example.test/v1" + _add_chat_response(httpx_mock, base_url, "From stdin") + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "-m", + "test-model", + "--no-stream", + ], + input="Hello from stdin", + catch_exceptions=False, + ) + + assert result.exit_code == 0 + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["messages"] == [{"role": "user", "content": "Hello from stdin"}] + assert not (user_path / "logs.db").exists() + + +def test_endpoint_without_prompt_waits_for_stdin(httpx_mock, user_path, monkeypatch): + base_url = "https://terminal-stdin.example.test/v1" + _add_chat_response(httpx_mock, base_url, "From awaited stdin") + monkeypatch.setattr("click.testing._NamedTextIOWrapper.isatty", lambda self: True) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "-m", + "test-model", + "--no-stream", + ], + input="Hello after waiting for stdin", + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output == "From awaited stdin\n" + request_body = json.loads(httpx_mock.get_requests()[0].content) + assert request_body["messages"] == [ + {"role": "user", "content": "Hello after waiting for stdin"} + ] + assert not (user_path / "logs.db").exists() + + +def test_endpoint_interactive_chat_preserves_history( + httpx_mock, user_path, templates_path +): + base_url = "https://chat.example.test/v1" + _add_chat_response(httpx_mock, base_url, "First answer") + _add_chat_response(httpx_mock, base_url, "Second answer") + (templates_path / "endpoint-chat.yaml").write_text( + 'prompt: "Question: $input"\n', "utf-8" + ) + + result = CliRunner().invoke( + cli, + [ + "openai", + "endpoint", + base_url, + "-m", + "test-model", + "--chat", + "--no-stream", + "--system", + "Be brief", + "--template", + "endpoint-chat", + "--functions", + ( + "def lookup(value: str) -> str:\n" + ' "Look up a value."\n' + " return value\n" + ), + "--at", + "https://images.example.test/context.jpg", + "image/jpeg", + ], + input="First question\nSecond question\nquit\n", + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert "First answer" in result.output + assert "Second answer" in result.output + assert not (user_path / "logs.db").exists() + + requests = httpx_mock.get_requests() + assert len(requests) == 2 + request_bodies = [json.loads(request.content) for request in requests] + assert all( + body["tools"][0]["function"]["name"] == "lookup" for body in request_bodies + ) + assert request_bodies[1]["messages"] == [ + {"role": "system", "content": "Be brief"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Question: First question"}, + { + "type": "image_url", + "image_url": { + "url": "https://images.example.test/context.jpg", + }, + }, + ], + }, + {"role": "assistant", "content": "First answer"}, + {"role": "user", "content": "Question: Second question"}, + ] diff --git a/tests/test_openai_messages.py b/tests/test_openai_messages.py new file mode 100644 index 000000000..65db37d46 --- /dev/null +++ b/tests/test_openai_messages.py @@ -0,0 +1,613 @@ +import json + +import pytest +from pytest_httpx import IteratorStream + +import llm +from llm.default_plugins.openai_models import Chat +from llm.models import Prompt + +API_KEY = "badkey" + + +def _sse(delta, finish_reason=None, usage=None, tool_calls=None): + chunk = { + "id": "c1", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + if tool_calls is not None: + chunk["choices"][0]["delta"]["tool_calls"] = tool_calls + if usage is not None: + chunk["usage"] = usage + return f"data: {json.dumps(chunk)}\n\n".encode() + + +def _text_stream(): + yield _sse({"role": "assistant", "content": ""}) + yield _sse({"content": "Hel"}) + yield _sse({"content": "lo"}) + yield _sse({}, finish_reason="stop") + yield b"data: [DONE]\n\n" + + +def _tool_call_stream(): + """Mimic an OpenAI stream with a tool call (no preceding text).""" + yield _sse({"role": "assistant", "content": None}) + yield _sse( + {}, + tool_calls=[ + { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": ""}, + } + ], + ) + yield _sse( + {}, + tool_calls=[ + { + "index": 0, + "function": {"arguments": '{"city":'}, + } + ], + ) + yield _sse( + {}, + tool_calls=[ + { + "index": 0, + "function": {"arguments": '"Paris"}'}, + } + ], + ) + yield _sse({}, finish_reason="tool_calls") + yield b"data: [DONE]\n\n" + + +def _text_then_tool_call_stream(): + """Text arrives first, then a tool call — the tool call must get + a part_index past the text so assembly doesn't mix families.""" + yield _sse({"role": "assistant", "content": ""}) + yield _sse({"content": "Looking up"}) + yield _sse( + {}, + tool_calls=[ + { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"c":1}'}, + } + ], + ) + yield _sse({}, finish_reason="tool_calls") + yield b"data: [DONE]\n\n" + + +@pytest.fixture +def chat_model(): + # A plain Chat instance with vision and tools enabled — enough + # capabilities for the Part subtypes we translate. + return Chat("gpt-4o-mini", vision=True, supports_tools=True) + + +class TestBuildMessagesFromExplicitMessages: + def test_single_user_message(self, chat_model): + prompt = Prompt(None, model=chat_model, messages=[llm.user("hi")]) + result = chat_model.build_messages(prompt, None) + assert result == [{"role": "user", "content": "hi"}] + + def test_system_plus_user(self, chat_model): + prompt = Prompt( + None, + model=chat_model, + messages=[llm.system("be brief"), llm.user("hi")], + ) + result = chat_model.build_messages(prompt, None) + assert result == [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": "hi"}, + ] + + def test_user_with_attachment(self, chat_model): + att = llm.Attachment(type="image/jpeg", url="http://example.com/cat.jpg") + prompt = Prompt( + None, + model=chat_model, + messages=[llm.user("describe", att)], + ) + result = chat_model.build_messages(prompt, None) + assert result == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + { + "type": "image_url", + "image_url": {"url": "http://example.com/cat.jpg"}, + }, + ], + } + ] + + def test_assistant_with_tool_call(self, chat_model): + tool_call = llm.parts.ToolCallPart( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + prompt = Prompt( + None, + model=chat_model, + messages=[ + llm.user("search weather"), + llm.assistant("on it", tool_call), + ], + ) + result = chat_model.build_messages(prompt, None) + assert result == [ + {"role": "user", "content": "search weather"}, + { + "role": "assistant", + "content": "on it", + "tool_calls": [ + { + "type": "function", + "id": "c1", + "function": { + "name": "search", + "arguments": json.dumps({"q": "weather"}), + }, + } + ], + }, + ] + + def test_assistant_tool_call_only_no_text(self, chat_model): + """When an assistant message has tool_calls but no text, OpenAI + expects content=null.""" + tool_call = llm.parts.ToolCallPart( + name="search", arguments={"q": "x"}, tool_call_id="c1" + ) + prompt = Prompt( + None, + model=chat_model, + messages=[llm.user("q"), llm.assistant(tool_call)], + ) + result = chat_model.build_messages(prompt, None) + assert result[1] == { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "type": "function", + "id": "c1", + "function": { + "name": "search", + "arguments": json.dumps({"q": "x"}), + }, + } + ], + } + + def test_tool_role_message_with_tool_result(self, chat_model): + tr = llm.parts.ToolResultPart(name="search", output="sunny", tool_call_id="c1") + prompt = Prompt( + None, + model=chat_model, + messages=[ + llm.user("q"), + llm.tool_message(tr), + ], + ) + result = chat_model.build_messages(prompt, None) + assert result == [ + {"role": "user", "content": "q"}, + {"role": "tool", "tool_call_id": "c1", "content": "sunny"}, + ] + + def test_multiple_tool_results_emit_multiple_messages(self, chat_model): + """Parallel tool results: one OpenAI 'tool' message per result.""" + a = llm.parts.ToolResultPart(name="t", output="A", tool_call_id="c1") + b = llm.parts.ToolResultPart(name="t", output="B", tool_call_id="c2") + prompt = Prompt( + None, + model=chat_model, + messages=[llm.user("q"), llm.tool_message(a, b)], + ) + result = chat_model.build_messages(prompt, None) + assert result == [ + {"role": "user", "content": "q"}, + {"role": "tool", "tool_call_id": "c1", "content": "A"}, + {"role": "tool", "tool_call_id": "c2", "content": "B"}, + ] + + +class TestBuildMessagesLegacyFieldsStillWork: + """prompt=, system=, attachments= keep working — they synthesize + messages via Prompt.messages before build_messages sees them.""" + + def test_prompt_only(self, chat_model): + prompt = Prompt("hi", model=chat_model) + result = chat_model.build_messages(prompt, None) + assert result == [{"role": "user", "content": "hi"}] + + def test_system_and_prompt(self, chat_model): + prompt = Prompt("hi", model=chat_model, system="be brief") + result = chat_model.build_messages(prompt, None) + assert result == [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": "hi"}, + ] + + def test_attachments(self, chat_model): + att = llm.Attachment(type="image/jpeg", url="http://example.com/a.jpg") + prompt = Prompt("look", model=chat_model, attachments=[att]) + result = chat_model.build_messages(prompt, None) + assert result == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + { + "type": "image_url", + "image_url": {"url": "http://example.com/a.jpg"}, + }, + ], + } + ] + + +class TestBuildMessagesSystemDedup: + """Explicit messages with repeated system messages dedupe + repeated unchanged systems; OpenAI accepts one.""" + + def test_same_system_not_repeated(self, chat_model): + prompt = Prompt( + None, + model=chat_model, + messages=[ + llm.system("be brief"), + llm.user("q1"), + llm.assistant("a1"), + llm.system("be brief"), + llm.user("q2"), + ], + ) + result = chat_model.build_messages(prompt, None) + system_msgs = [m for m in result if m["role"] == "system"] + assert len(system_msgs) == 1 + assert system_msgs[0]["content"] == "be brief" + + def test_system_change_emitted(self, chat_model): + prompt = Prompt( + None, + model=chat_model, + messages=[ + llm.system("be brief"), + llm.user("q1"), + llm.assistant("a1"), + llm.system("be expansive"), + llm.user("q2"), + ], + ) + result = chat_model.build_messages(prompt, None) + system_msgs = [m for m in result if m["role"] == "system"] + assert [m["content"] for m in system_msgs] == [ + "be brief", + "be expansive", + ] + + +class TestBuildMessagesConversationHistory: + def test_prior_turn_text_plus_current_user(self, chat_model): + new_prompt = Prompt( + None, + model=chat_model, + messages=[ + llm.user("what's 1+1?"), + llm.assistant("2"), + llm.user("what about 2+2?"), + ], + ) + result = chat_model.build_messages(new_prompt, None) + assert result == [ + {"role": "user", "content": "what's 1+1?"}, + {"role": "assistant", "content": "2"}, + {"role": "user", "content": "what about 2+2?"}, + ] + + def test_no_double_emission_from_conversation_prompt_flow( + self, chat_model, httpx_mock + ): + # Two staged responses so conv.prompt twice can complete. + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + "choices": [ + { + "message": {"role": "assistant", "content": "A1"}, + "finish_reason": "stop", + } + ], + }, + headers={"Content-Type": "application/json"}, + ) + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + "choices": [ + { + "message": {"role": "assistant", "content": "A2"}, + "finish_reason": "stop", + } + ], + }, + headers={"Content-Type": "application/json"}, + ) + + model = llm.get_model("gpt-4o-mini") + conv = model.conversation() + r1 = conv.prompt("Q1", key=API_KEY, stream=False) + r1.text() + r2 = conv.prompt("Q2", key=API_KEY, stream=False) + r2.text() + + # Inspect what was sent on the SECOND turn. + sent_body = json.loads(httpx_mock.get_requests()[-1].content) + sent_messages = sent_body["messages"] + # Exactly three: user(Q1), assistant(A1), user(Q2). + assert sent_messages == [ + {"role": "user", "content": "Q1"}, + {"role": "assistant", "content": "A1"}, + {"role": "user", "content": "Q2"}, + ] + + +class TestStreamingExecuteYieldsStreamEvents: + def test_text_stream_yields_text_events(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + events = list(response.stream_events()) + # At least one StreamEvent, all text, all at part_index=0. + assert events, "expected stream events" + assert all(isinstance(e, llm.parts.StreamEvent) for e in events) + assert all(e.type == "text" for e in events) + assert all(e.part_index == 0 for e in events) + # Text chunks concatenate to the expected full text. + assert "".join(e.chunk for e in events) == "Hello" + + def test_text_stream_plain_iteration_still_returns_strings(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + chunks = list(response) + assert all(isinstance(c, str) for c in chunks) + assert "".join(chunks) == "Hello" + + def test_text_stream_messages_assembled(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + response.text() + assert response.messages() == [ + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="Hello")]) + ] + + def test_tool_call_stream_yields_name_and_args_events(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_tool_call_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + + def get_weather(city: str) -> str: + "Look up the weather." + return "sunny" + + model = llm.get_model("gpt-4o-mini") + response = model.prompt("weather?", tools=[get_weather], key=API_KEY) + events = list(response.stream_events()) + types = [e.type for e in events] + assert "tool_call_name" in types + assert "tool_call_args" in types + # Name event carries the tool_call_id and name. + name_ev = next(e for e in events if e.type == "tool_call_name") + assert name_ev.tool_call_id == "call_1" + assert name_ev.chunk == "get_weather" + # Args events share the same part_index and concatenate to + # valid JSON. + args_events = [e for e in events if e.type == "tool_call_args"] + assert all(e.part_index == name_ev.part_index for e in args_events) + assert json.loads("".join(e.chunk for e in args_events)) == {"city": "Paris"} + + def test_tool_call_registered_via_add_tool_call(self, httpx_mock): + """response.tool_calls() still works — chain/execute relies on it.""" + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_tool_call_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + + def get_weather(city: str) -> str: + "Look up the weather." + return "sunny" + + model = llm.get_model("gpt-4o-mini") + response = model.prompt("weather?", tools=[get_weather], key=API_KEY) + response.text() + tcs = response.tool_calls() + assert len(tcs) == 1 + assert tcs[0].name == "get_weather" + assert tcs[0].arguments == {"city": "Paris"} + assert tcs[0].tool_call_id == "call_1" + + def test_text_then_tool_call_part_index_advances(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_then_tool_call_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + + def get_weather(c: int) -> str: + "Weather." + return "sunny" + + model = llm.get_model("gpt-4o-mini") + response = model.prompt("q", tools=[get_weather], key=API_KEY) + response.text() + # After streaming, messages has both a TextPart and a ToolCallPart. + parts = response.messages()[0].parts + assert any(isinstance(p, llm.parts.TextPart) for p in parts) + assert any(isinstance(p, llm.parts.ToolCallPart) for p in parts) + text_part = next(p for p in parts if isinstance(p, llm.parts.TextPart)) + tc_part = next(p for p in parts if isinstance(p, llm.parts.ToolCallPart)) + assert text_part.text == "Looking up" + assert tc_part.name == "get_weather" + assert tc_part.arguments == {"c": 1} + + +class TestAsyncStreamingExecuteYieldsStreamEvents: + @pytest.mark.asyncio + async def test_text_stream_yields_text_events(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_async_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + events = [] + async for event in response.astream_events(): + events.append(event) + assert all(isinstance(e, llm.parts.StreamEvent) for e in events) + assert [e.type for e in events] == ["text"] * len(events) + assert "".join(e.chunk for e in events) == "Hello" + + +def _text_stream_with_reasoning_usage(reasoning_tokens): + """Stream with usage in the final chunk reporting reasoning_tokens.""" + yield _sse({"role": "assistant", "content": ""}) + yield _sse({"content": "Hel"}) + yield _sse({"content": "lo"}) + yield _sse({}, finish_reason="stop") + # Final chunk with usage — OpenAI streams usage once at the end + # when stream_options.include_usage=True. + yield _sse( + {}, + usage={ + "prompt_tokens": 5, + "completion_tokens": 2, + "total_tokens": 7, + "completion_tokens_details": {"reasoning_tokens": reasoning_tokens}, + }, + ) + yield b"data: [DONE]\n\n" + + +class TestReasoningTokenCount: + def test_redacted_reasoning_part_emitted_when_count_present(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream_with_reasoning_usage(150)), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + response.text() + assert response.messages() == [ + llm.Message( + role="assistant", + parts=[ + llm.parts.ReasoningPart(text="", redacted=True), + llm.parts.TextPart(text="Hello"), + ], + ) + ] + + def test_no_reasoning_part_when_zero_or_absent(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + stream=IteratorStream(_text_stream_with_reasoning_usage(0)), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY) + response.text() + parts = response.messages()[0].parts + assert not any( + isinstance(p, llm.parts.ReasoningPart) for p in parts + ), "should not add a redacted reasoning part when count=0" + + +class TestNonStreamingExecuteYieldsStreamEvents: + def test_non_streaming_text_yields_single_event(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + "choices": [ + { + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-4o-mini") + response = model.prompt("hi", key=API_KEY, stream=False) + events = list(response.stream_events()) + assert events == [ + llm.parts.StreamEvent(type="text", chunk="Hello", part_index=0) + ] + assert response.messages() == [ + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="Hello")]) + ] diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py new file mode 100644 index 000000000..bff2909cd --- /dev/null +++ b/tests/test_openai_responses.py @@ -0,0 +1,1614 @@ +"""Tests for the /v1/responses code path in the default OpenAI plugin.""" + +import json +import os + +import pytest +from pytest_httpx import IteratorStream + +import llm +from llm.default_plugins.openai_models import CodeInterpreter, Responses, WebSearch + +API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" + + +def _text_response_json(model="gpt-5.6-luna", text="ok"): + return { + "id": "resp_server_tool", + "object": "response", + "created_at": 1, + "model": model, + "output": [ + { + "type": "message", + "id": "msg_server_tool", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + "status": "completed", + } + + +@pytest.mark.parametrize( + ("tool", "expected"), + ( + ( + CodeInterpreter(), + {"type": "code_interpreter", "container": {"type": "auto"}}, + ), + ( + CodeInterpreter(memory_limit="4g", file_ids=["file-1", "file-2"]), + { + "type": "code_interpreter", + "container": { + "type": "auto", + "memory_limit": "4g", + "file_ids": ["file-1", "file-2"], + }, + }, + ), + ( + CodeInterpreter(container="cntr_123"), + {"type": "code_interpreter", "container": "cntr_123"}, + ), + ), +) +def test_code_interpreter_tool_spec(tool, expected): + assert tool.tool_spec(llm.get_model("gpt-5.6-luna")) == expected + + +def test_code_interpreter_validates_container_configuration(): + with pytest.raises(ValueError, match="memory_limit"): + CodeInterpreter(memory_limit="8g") + with pytest.raises(ValueError, match="cannot be combined"): + CodeInterpreter(container="cntr_123", memory_limit="4g") + with pytest.raises(ValueError, match="cannot be combined"): + CodeInterpreter(container="cntr_123", file_ids=["file-1"]) + with pytest.raises(TypeError, match="container must be a string"): + CodeInterpreter(container={"type": "auto"}) + + +def test_code_interpreter_prepare_request_is_additive_and_idempotent(): + tool = CodeInterpreter() + kwargs = {"include": ["reasoning.encrypted_content"]} + + tool.prepare_request(llm.get_model("gpt-5.6-luna"), kwargs) + tool.prepare_request(llm.get_model("gpt-5.6-luna"), kwargs) + + assert kwargs == { + "include": [ + "reasoning.encrypted_content", + "code_interpreter_call.outputs", + ] + } + + +@pytest.mark.parametrize( + ("tool", "expected"), + ( + (WebSearch(), {"type": "web_search"}), + ( + WebSearch( + allowed_domains=["openai.com"], + blocked_domains=["example.com"], + user_location={"country": "GB", "city": "London"}, + search_context_size="high", + external_web_access=False, + return_token_budget="unlimited", + search_content_types=["image", "text"], + image_settings={"max_results": 3, "caption": True}, + ), + { + "type": "web_search", + "filters": { + "allowed_domains": ["openai.com"], + "blocked_domains": ["example.com"], + }, + "user_location": { + "type": "approximate", + "country": "GB", + "city": "London", + }, + "search_context_size": "high", + "external_web_access": False, + "return_token_budget": "unlimited", + "search_content_types": ["image", "text"], + "image_settings": {"max_results": 3, "caption": True}, + }, + ), + ), +) +def test_web_search_tool_spec(tool, expected): + assert tool.tool_spec(llm.get_model("gpt-5.6-luna")) == expected + + +@pytest.mark.parametrize( + ("kwargs", "error"), + ( + ({"search_context_size": "huge"}, "search_context_size"), + ({"return_token_budget": "lots"}, "return_token_budget"), + ({"search_content_types": ["video"]}, "search_content_types"), + ({"allowed_domains": ["https://openai.com"]}, "scheme"), + ({"allowed_domains": [f"example{i}.com" for i in range(101)]}, "100"), + ({"user_location": {"type": "exact"}}, "approximate"), + ({"external_web_access": "no"}, "external_web_access"), + ({"image_settings": {"max_results": 0}}, "max_results"), + ({"image_settings": {"caption": "yes"}}, "caption"), + ), +) +def test_web_search_validates_configuration(kwargs, error): + with pytest.raises((TypeError, ValueError), match=error): + WebSearch(**kwargs) + + +def test_web_search_prepare_request_is_additive_and_idempotent(): + tool = WebSearch(include_sources=True, include_results=True) + kwargs = {"include": ["reasoning.encrypted_content"]} + + tool.prepare_request(llm.get_model("gpt-5.6-luna"), kwargs) + tool.prepare_request(llm.get_model("gpt-5.6-luna"), kwargs) + + assert kwargs == { + "include": [ + "reasoning.encrypted_content", + "web_search_call.action.sources", + "web_search_call.results", + ] + } + default_kwargs = {} + WebSearch().prepare_request(llm.get_model("gpt-5.6-luna"), default_kwargs) + assert default_kwargs == {} + + +def test_responses_web_search_request_and_result_capture(httpx_mock): + sources = [ + {"type": "url", "url": "https://openai.com/news/"}, + {"type": "url", "url": "https://example.com/report"}, + ] + results = [ + { + "type": "image_result", + "image_url": "https://example.com/image.jpg", + "source_website_url": "https://example.com/report", + "thumbnail_url": "https://example.com/thumb.jpg", + "caption": "An example image", + } + ] + response_json = _text_response_json(text="A cited answer") + response_json["output"].insert( + 0, + { + "type": "web_search_call", + "id": "ws_123", + "status": "completed", + "action": { + "type": "search", + "query": "OpenAI news", + "queries": ["OpenAI news"], + "sources": sources, + }, + "results": results, + }, + ) + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=response_json, + headers={"Content-Type": "application/json"}, + ) + + response = llm.get_model("gpt-5.6-luna").prompt( + "Search for OpenAI news", + tools=[ + WebSearch( + allowed_domains=["openai.com"], + include_sources=True, + include_results=True, + ) + ], + stream=False, + key="test", + ) + + assert response.text() == "A cited answer" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [ + { + "type": "web_search", + "filters": {"allowed_domains": ["openai.com"]}, + } + ] + assert request_body["include"] == [ + "reasoning.encrypted_content", + "web_search_call.action.sources", + "web_search_call.results", + ] + message = response.messages()[0] + assert [type(part).__name__ for part in message.parts] == [ + "ToolCallPart", + "ToolResultPart", + "TextPart", + ] + assert message.parts[0].server_executed + assert message.parts[0].arguments["sources"] == sources + assert message.parts[1].server_executed + assert json.loads(message.parts[1].output) == results + + +def test_server_side_prepare_request_runs_in_list_order_after_baseline(): + class IncludeMarker(CodeInterpreter): + def __init__(self, marker): + super().__init__() + self.marker = marker + + def tool_spec(self, model): + return {"type": f"marker_{self.marker}"} + + def prepare_request(self, model, kwargs): + assert kwargs["store"] is False + assert len(kwargs["tools"]) == 2 + kwargs.setdefault("include", []).append(self.marker) + + model = llm.get_model("gpt-5.6-luna") + + class FakePrompt: + pass + + prompt = FakePrompt() + prompt.options = model.Options() + prompt.tools = [IncludeMarker("first"), IncludeMarker("second")] + prompt.schema = None + prompt.hide_reasoning = False + + kwargs = model._finalize_responses_kwargs( + prompt, stream=False, instructions="Be useful" + ) + assert kwargs["instructions"] == "Be useful" + assert kwargs["include"] == [ + "reasoning.encrypted_content", + "first", + "second", + ] + + +def test_responses_mixes_function_and_code_interpreter_tools(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=_text_response_json(), + headers={"Content-Type": "application/json"}, + ) + + def multiply(a: int, b: int) -> int: + return a * b + + model = llm.get_model("gpt-5.6-luna") + response = model.prompt( + "Calculate 111 * 333 using the python tool", + tools=[multiply, CodeInterpreter(memory_limit="4g")], + stream=False, + key="test", + ) + assert response.text() == "ok" + + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"][0]["type"] == "function" + assert request_body["tools"][0]["name"] == "multiply" + assert request_body["tools"][1] == { + "type": "code_interpreter", + "container": {"type": "auto", "memory_limit": "4g"}, + } + assert request_body["include"] == [ + "reasoning.encrypted_content", + "code_interpreter_call.outputs", + ] + + +def test_responses_raw_server_tool_passthrough_on_custom_endpoint(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://example.test/v1/responses", + json=_text_response_json(model="custom-model"), + headers={"Content-Type": "application/json"}, + ) + model = Responses( + "custom-model", + api_base="https://example.test/v1", + supports_tools=False, + ) + raw_spec = {"type": "browser_search", "depth": "deep"} + response = model.prompt( + "Search", tools=[llm.ServerSideTool(raw_spec)], stream=False, key="test" + ) + + assert response.text() == "ok" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [raw_spec] + + +@pytest.mark.parametrize("tool", (CodeInterpreter(), WebSearch())) +def test_server_side_tool_rejected_by_chat_and_chat_fallback(tool): + from llm.default_plugins.openai_models import Chat + + chat = Chat("chat-model", supports_tools=True) + with pytest.raises(ValueError, match="llm tools -m chat-model"): + chat.prompt("Use a server-side tool", tools=[tool]) + + responses_model = llm.get_model("gpt-5.6-luna") + response = responses_model.prompt( + "Use a server-side tool", + tools=[tool], + chat_completions=True, + key="test", + ) + with pytest.raises(ValueError, match="llm tools -m gpt-5.6-luna"): + response.text() + + +@pytest.mark.asyncio +async def test_async_responses_code_interpreter_request(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=_text_response_json(), + headers={"Content-Type": "application/json"}, + ) + model = llm.get_async_model("gpt-5.6-luna") + response = model.prompt( + "Calculate", + tools=[CodeInterpreter(file_ids=["file-1"])], + stream=False, + key="test", + ) + + assert await response.text() == "ok" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [ + { + "type": "code_interpreter", + "container": {"type": "auto", "file_ids": ["file-1"]}, + } + ] + assert request_body["include"] == [ + "reasoning.encrypted_content", + "code_interpreter_call.outputs", + ] + + +@pytest.mark.asyncio +async def test_async_responses_web_search_request(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=_text_response_json(), + headers={"Content-Type": "application/json"}, + ) + response = llm.get_async_model("gpt-5.6-luna").prompt( + "Search", + tools=[WebSearch(external_web_access=False, include_sources=True)], + stream=False, + key="test", + ) + + assert await response.text() == "ok" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["tools"] == [ + {"type": "web_search", "external_web_access": False} + ] + assert request_body["include"] == [ + "reasoning.encrypted_content", + "web_search_call.action.sources", + ] + + +def _responses_sse(event_type, data): + data = {"type": event_type, **data} + return f"event: {event_type}\ndata: {json.dumps(data)}\n\n".encode() + + +def _code_interpreter_stream(): + yield _responses_sse( + "response.output_item.added", + { + "output_index": 0, + "item": { + "id": "ci_stream", + "type": "code_interpreter_call", + "status": "in_progress", + "container_id": "cntr_stream", + "code": "", + "outputs": [], + }, + }, + ) + yield _responses_sse( + "response.output_item.done", + { + "output_index": 0, + "item": { + "id": "ci_stream", + "type": "code_interpreter_call", + "status": "completed", + "container_id": "cntr_stream", + "code": "print(6 * 7)", + "outputs": [{"type": "logs", "logs": "42\n"}], + }, + }, + ) + yield _responses_sse( + "response.output_item.added", + { + "output_index": 1, + "item": { + "id": "msg_stream", + "type": "message", + "status": "in_progress", + "role": "assistant", + "content": [], + }, + }, + ) + yield _responses_sse( + "response.output_text.delta", + { + "item_id": "msg_stream", + "output_index": 1, + "content_index": 0, + "delta": "42", + }, + ) + + +def _web_search_refresh_stream(): + def image_result(name): + return { + "type": "image_result", + "image_url": f"https://example.com/{name}.jpg", + "source_website_url": f"https://example.com/{name}", + "thumbnail_url": f"https://example.com/{name}-thumb.jpg", + "caption": f"{name.title()} image", + } + + def web_search_item(sources, results): + return { + "id": "ws_stream", + "type": "web_search_call", + "status": "completed", + "action": { + "type": "search", + "query": "example images", + "queries": ["example images"], + "sources": sources, + }, + "results": results, + } + + partial_sources = [{"type": "url", "url": "https://example.com/first"}] + final_sources = partial_sources + [ + {"type": "url", "url": "https://example.com/second"}, + ] + partial_results = [image_result("first")] + final_results = partial_results + [image_result("second")] + yield _responses_sse( + "response.output_item.done", + { + "output_index": 0, + "item": web_search_item(partial_sources, partial_results), + }, + ) + yield _responses_sse( + "response.output_text.delta", + { + "item_id": "msg_stream", + "output_index": 1, + "content_index": 0, + "delta": "done", + }, + ) + response_json = _text_response_json(text="done") + response_json["output"].insert(0, web_search_item(final_sources, final_results)) + yield _responses_sse( + "response.completed", + {"response": response_json}, + ) + + +def _responses_reasoning_summary_stream(): + yield _responses_sse( + "response.reasoning_summary_text.delta", + { + "item_id": "rs_1", + "output_index": 0, + "summary_index": 0, + "delta": "Thinking", + "sequence_number": 1, + }, + ) + yield _responses_sse( + "response.reasoning_summary_text.delta", + { + "item_id": "rs_1", + "output_index": 0, + "summary_index": 0, + "delta": " aloud", + "sequence_number": 2, + }, + ) + yield _responses_sse( + "response.output_item.done", + { + "item": { + "id": "rs_1", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Thinking aloud"}], + "encrypted_content": "encrypted", + "status": "completed", + }, + "output_index": 0, + "sequence_number": 3, + }, + ) + yield _responses_sse( + "response.output_text.delta", + { + "item_id": "msg_1", + "output_index": 1, + "content_index": 0, + "delta": "done", + "logprobs": [], + "sequence_number": 4, + }, + ) + + +def test_responses_model_is_registered(): + from llm.default_plugins.openai_models import Chat + + model = llm.get_model("gpt-5.5") + assert "Responses" in type(model).__name__ + # The chat_completions opt-out option must be exposed. + assert "chat_completions" in model.Options.model_fields + assert "reasoning_summary" in model.Options.model_fields + assert "reasoning_summary" in llm.get_async_model("gpt-5.5").Options.model_fields + assert ( + "reasoning_summary" + not in Chat("reasoning-chat-model", reasoning=True).Options.model_fields + ) + + +def test_chat_completions_opt_out_dispatches_to_chat(httpx_mock): + """When chat_completions=1 is passed, the request must hit + /v1/chat/completions, not /v1/responses.""" + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "id": "chatcmpl-x", + "object": "chat.completion", + "model": "gpt-5.5", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi from chat"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.5") + response = model.prompt( + "hello", + stream=False, + chat_completions=True, + reasoning_summary="detailed", + key="test", + ) + assert response.text() == "hi from chat" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert "reasoning_summary" not in request_body + + +def test_default_routes_to_responses_endpoint(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-5.5", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "hi from responses", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.5") + response = model.prompt("hello", stream=False, key="test") + assert response.text() == "hi from responses" + # Ensure we sent to the right endpoint + requests = [r for r in httpx_mock.get_requests()] + assert any("/v1/responses" in str(r.url) for r in requests) + request_body = json.loads(requests[-1].content) + assert request_body["include"] == ["reasoning.encrypted_content"] + assert request_body["reasoning"] == {"summary": "auto"} + + +def test_hide_reasoning_omits_reasoning_summary_from_responses_request(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-5.5", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "hidden", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.5") + response = model.prompt("hello", stream=False, key="test", hide_reasoning=True) + assert response.text() == "hidden" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["include"] == ["reasoning.encrypted_content"] + assert "reasoning" not in request_body + + +def test_non_reasoning_responses_model_omits_encrypted_reasoning_include(httpx_mock): + from llm.default_plugins.openai_models import Responses + + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-4.1", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "hi from gpt-4.1", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + + model = Responses("gpt-4.1", vision=True, supports_schema=True, supports_tools=True) + response = model.prompt("hello", stream=False, key="test") + + assert response.text() == "hi from gpt-4.1" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["model"] == "gpt-4.1" + assert "include" not in request_body + assert "reasoning" not in request_body + + +def test_responses_input_translation(): + """Unit-test the message-to-input translator without hitting the API.""" + from llm.parts import ( + Message, + TextPart, + ToolCallPart, + ToolResultPart, + ) + + model = llm.get_model("gpt-5.5") + + class FakePrompt: + messages = ( + Message(role="system", parts=[TextPart(text="be brief")]), + Message(role="user", parts=[TextPart(text="2 + 2?")]), + Message( + role="assistant", + parts=[ + ToolCallPart( + name="add", + arguments={"a": 2, "b": 2}, + tool_call_id="call_abc", + ) + ], + ), + Message( + role="tool", + parts=[ToolResultPart(name="add", output="4", tool_call_id="call_abc")], + ), + ) + + items, instructions = model._build_responses_input(FakePrompt()) + assert instructions == "be brief" + # First user message is a plain string content + assert items[0] == {"role": "user", "content": "2 + 2?"} + # function_call from assistant + assert items[1]["type"] == "function_call" + assert items[1]["call_id"] == "call_abc" + assert items[1]["name"] == "add" + assert json.loads(items[1]["arguments"]) == {"a": 2, "b": 2} + # tool result + assert items[2] == { + "type": "function_call_output", + "call_id": "call_abc", + "output": "4", + } + + +def test_responses_input_translation_assistant_text_uses_easy_input_message(): + """Plain prior assistant text should match OpenAI's EasyInputMessage shape.""" + from llm.parts import Message, TextPart + + model = llm.get_model("gpt-5.5") + + class FakePrompt: + messages = ( + Message(role="user", parts=[TextPart(text="hello")]), + Message(role="assistant", parts=[TextPart(text="first-ok")]), + Message(role="user", parts=[TextPart(text="what next?")]), + ) + + items, instructions = model._build_responses_input(FakePrompt()) + + assert instructions is None + assert items == [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "first-ok"}, + {"role": "user", "content": "what next?"}, + ] + + +def test_responses_reply_sends_prior_assistant_text_as_string(httpx_mock): + """response.reply() should send the same simple history shape a direct + openai-python Responses call would use for a text-only assistant turn.""" + + def response_json(response_id, message_id, text): + return { + "id": response_id, + "object": "response", + "created_at": 1, + "model": "gpt-5.5", + "output": [ + { + "type": "message", + "id": message_id, + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + } + + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=response_json("resp_1", "msg_1", "first-ok"), + headers={"Content-Type": "application/json"}, + ) + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json=response_json("resp_2", "msg_2", "followup-ok"), + headers={"Content-Type": "application/json"}, + ) + + model = llm.get_model("gpt-5.5") + first = model.prompt("Say exactly: first-ok", stream=False, key="test") + second = first.reply("Say exactly: followup-ok", stream=False, key="test") + + assert first.text() == "first-ok" + assert second.text() == "followup-ok" + requests = httpx_mock.get_requests() + second_body = json.loads(requests[-1].content) + assert second_body["input"] == [ + {"role": "user", "content": "Say exactly: first-ok"}, + {"role": "assistant", "content": "first-ok"}, + {"role": "user", "content": "Say exactly: followup-ok"}, + ] + + +def test_responses_kwargs_packs_reasoning_and_verbosity(): + model = llm.get_model("gpt-5.5") + options = model.Options(reasoning_effort="low", verbosity="low") + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["reasoning"] == {"summary": "auto", "effort": "low"} + assert kwargs["text"]["verbosity"] == "low" + + +def test_responses_kwargs_sets_reasoning_summary_without_effort(): + model = llm.get_model("gpt-5.5") + options = model.Options() + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["reasoning"] == {"summary": "auto"} + + +@pytest.mark.parametrize("reasoning_summary", ("auto", "concise", "detailed")) +def test_responses_kwargs_explicit_reasoning_summary(reasoning_summary): + model = llm.get_model("gpt-5.5") + options = model.Options(reasoning_summary=reasoning_summary) + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["reasoning"] == {"summary": reasoning_summary} + + +def test_async_responses_kwargs_explicit_reasoning_summary(): + model = llm.get_async_model("gpt-5.5") + options = model.Options(reasoning_summary="concise") + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["reasoning"] == {"summary": "concise"} + + +def test_responses_kwargs_omits_reasoning_summary_when_hide_reasoning(): + model = llm.get_model("gpt-5.5") + options = model.Options(reasoning_effort="low") + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + p.hide_reasoning = True + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["reasoning"] == {"effort": "low"} + + +def test_responses_kwargs_omits_explicit_reasoning_summary_when_hide_reasoning(): + model = llm.get_model("gpt-5.5") + options = model.Options(reasoning_summary="detailed") + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + p.hide_reasoning = True + kwargs = model._finalize_responses_kwargs(p, stream=False) + assert "reasoning" not in kwargs + assert kwargs["include"] == ["reasoning.encrypted_content"] + + +def test_responses_kwargs_omits_empty_reasoning_when_hide_reasoning(): + model = llm.get_model("gpt-5.5") + options = model.Options() + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + p.hide_reasoning = True + kwargs = model._build_responses_kwargs(p, stream=False) + assert "reasoning" not in kwargs + + +@pytest.mark.parametrize( + "model_id,expected", + [ + ("gpt-5.6-sol", True), + ("gpt-5.6-terra", True), + ("gpt-5.6-luna", True), + ("gpt-5.5", True), + ("gpt-4o", True), + # The legacy /v1/completions endpoint does not accept service_tier + ("gpt-3.5-turbo-instruct", False), + ], +) +def test_service_tier_option_on_models(model_id, expected): + model = llm.get_model(model_id) + assert ("service_tier" in model.Options.model_fields) == expected + + +def test_responses_kwargs_includes_service_tier(): + model = llm.get_model("gpt-5.6-sol") + options = model.Options(service_tier="fast") + + class FakePrompt: + pass + + p = FakePrompt() + p.options = options + p.tools = [] + p.schema = None + kwargs = model._build_responses_kwargs(p, stream=False) + assert kwargs["service_tier"] == "fast" + + +def test_service_tier_sent_to_responses_endpoint(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_test_1", + "object": "response", + "created_at": 1, + "model": "gpt-5.6-sol", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "fast reply", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + }, + "status": "completed", + "service_tier": "priority", + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.6-sol") + response = model.prompt("hello", stream=False, service_tier="fast", key="test") + assert response.text() == "fast reply" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["service_tier"] == "fast" + # The response body reports the tier that actually processed the request + assert response.json()["service_tier"] == "priority" + + +def test_service_tier_sent_to_chat_completions_fallback(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/chat/completions", + json={ + "id": "chatcmpl-x", + "object": "chat.completion", + "model": "gpt-5.6-sol", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "fast chat"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.6-sol") + response = model.prompt( + "hello", stream=False, chat_completions=True, service_tier="fast", key="test" + ) + assert response.text() == "fast chat" + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert request_body["service_tier"] == "fast" + + +def test_responses_streams_reasoning_summary_text(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + stream=IteratorStream(_responses_reasoning_summary_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + + model = llm.get_model("gpt-5.5") + response = model.prompt("hello", key="test") + events = list(response.stream_events()) + + assert [(e.type, e.chunk) for e in events] == [ + ("reasoning", "Thinking"), + ("reasoning", " aloud"), + ("reasoning", ""), + ("text", "done"), + ] + messages = response.messages() + reasoning_parts = [ + p for m in messages for p in m.parts if isinstance(p, llm.parts.ReasoningPart) + ] + assert reasoning_parts == [ + llm.parts.ReasoningPart( + text="Thinking aloud", + provider_metadata={ + "openai": { + "id": "rs_1", + "encrypted_content": "encrypted", + "summary": [{"type": "summary_text", "text": "Thinking aloud"}], + } + }, + ) + ] + assert response.text() == "done" + + +@pytest.mark.vcr +def test_responses_basic_non_streaming(vcr): + model = llm.get_model("gpt-5.5") + response = model.prompt( + "Reply with exactly: pong", + stream=False, + reasoning_effort="low", + key=API_KEY, + ) + text = response.text() + assert "pong" in text.lower() + # response_json should reflect the Responses API shape + assert response.response_json["object"] == "response" + + +@pytest.mark.vcr +def test_responses_basic_streaming(vcr): + model = llm.get_model("gpt-5.5") + response = model.prompt( + "Reply with exactly: pong", + reasoning_effort="low", + key=API_KEY, + ) + chunks = list(response) + text = "".join(chunks) + assert "pong" in text.lower() + + +@pytest.mark.vcr +def test_responses_tool_use(vcr): + model = llm.get_model("gpt-5.5") + + def multiply(a: int, b: int) -> int: + "Multiply two numbers." + return a * b + + chain = model.chain( + "What is 1231 * 2331? Use the multiply tool.", + tools=[multiply], + stream=False, + options={"reasoning_effort": "low"}, + key=API_KEY, + ) + output = chain.text() + assert "2869461" in output.replace(",", "") + first, second = chain._responses + assert first.tool_calls()[0].name == "multiply" + assert first.tool_calls()[0].arguments == {"a": 1231, "b": 2331} + assert second.prompt.tool_results[0].output == "2869461" + + +@pytest.mark.vcr +def test_responses_tool_use_streaming(vcr): + model = llm.get_model("gpt-5.5") + + def multiply(a: int, b: int) -> int: + "Multiply two numbers." + return a * b + + chain = model.chain( + "What is 1231 * 2331? Use the multiply tool.", + tools=[multiply], + options={"reasoning_effort": "low"}, + key=API_KEY, + ) + output = "".join(chain) + assert "2869461" in output.replace(",", "") + first, _second = chain._responses + assert first.tool_calls()[0].arguments == {"a": 1231, "b": 2331} + + +@pytest.mark.vcr +def test_responses_round_trips_encrypted_reasoning(vcr): + """Reasoning items returned by the API in the first turn must be + echoed back verbatim on the second turn so the model can pick up + its hidden chain of thought after the tool result arrives.""" + from llm.parts import ReasoningPart + + model = llm.get_model("gpt-5.5") + + def lookup_population(country: str) -> int: + "Returns the current population of the specified fictional country." + return 123124 + + def can_have_dragons(population: int) -> bool: + "Returns True if the specified population can have dragons." + return population > 10000 + + chain = model.chain( + "Pick a clever country name, look up its population, then check " + "whether it can have dragons. Be brief.", + tools=[lookup_population, can_have_dragons], + stream=False, + options={"reasoning_effort": "high"}, + key=API_KEY, + ) + chain.text() # drain the chain + + first = chain._responses[0] + + # The first response must produce at least one ReasoningPart carrying + # the opaque encrypted_content + id. + reasoning_parts = [ + p for m in first.messages() for p in m.parts if isinstance(p, ReasoningPart) + ] + assert reasoning_parts, "first turn should expose at least one ReasoningPart" + pm = reasoning_parts[0].provider_metadata or {} + assert "openai" in pm + assert pm["openai"].get("encrypted_content"), "encrypted_content must be captured" + assert pm["openai"].get("id"), "reasoning id must be captured" + + # The second turn's outgoing input must echo back that reasoning + # item, otherwise the model loses its chain of thought. + second = chain._responses[1] + second_input = (second._prompt_json or {}).get("input") or [] + reasoning_inputs = [it for it in second_input if it.get("type") == "reasoning"] + assert reasoning_inputs, "second turn must echo a reasoning input item" + assert reasoning_inputs[0]["encrypted_content"] == pm["openai"]["encrypted_content"] + assert reasoning_inputs[0]["id"] == pm["openai"]["id"] + + +@pytest.mark.vcr +def test_responses_interleaved_reasoning_between_tool_calls(vcr): + """Tool calls during reasoning: each turn produces fresh reasoning AND + every prior reasoning block is round-tripped on every subsequent turn + so the model's hidden chain of thought accumulates across the whole + chain. This is the GPT-5-class capability that the Chat Completions + API can't deliver because it discards reasoning between turns.""" + from llm.parts import ReasoningPart + + model = llm.get_model("gpt-5.5") + + # Tool whose results force the model to re-plan between calls: each + # lookup hands the model a NEW key to use next, so the model has to + # think to figure out the next argument. Parallel tool calls would + # short-circuit this, so we need the model to reason in series. + def db_lookup(key: str) -> str: + "Look up a value by key in the puzzle database." + table = { + "start": "Begin with the value 7.", + "step1_7": "Multiply by 13. Now lookup with key step2_.", + "step2_91": "Subtract 11. Now lookup with key step3_.", + "step3_80": ("The answer is the value modulo 9. State only the integer."), + } + return table.get(key, "unknown key") + + conversation = model.conversation(tools=[db_lookup]) + conversation.chain_limit = 4 + chain = conversation.chain( + "Solve this puzzle: call db_lookup('start'), then follow each " + "instruction step by step. Each lookup tells you the next key " + "to use. Compute each step in your head. State only the final " + "integer.", + stream=False, + options={"reasoning_effort": "high"}, + key=API_KEY, + ) + # The chain may exceed the limit - we just want enough turns to + # observe interleaved reasoning, then we stop. + try: + chain.text() + except ValueError as e: + if "Chain limit" not in str(e): + raise + + responses = chain._responses + assert ( + len(responses) >= 3 + ), f"expected at least 3 chained turns, got {len(responses)}" + + # 1) Fresh reasoning happens on more than just the first turn. This is + # the actual interleaved-reasoning capability, not just round-trip. + reasoning_token_counts = [] + for r in responses: + u = r.usage() + details = (u.details if u else None) or {} + reasoning_token_counts.append( + (details.get("output_tokens_details") or {}).get("reasoning_tokens") or 0 + ) + turns_with_fresh_reasoning = sum(1 for n in reasoning_token_counts if n > 0) + assert turns_with_fresh_reasoning >= 2, ( + f"expected >=2 turns to produce fresh reasoning, got " + f"{turns_with_fresh_reasoning} (counts: {reasoning_token_counts})" + ) + + # 2) Every reasoning block produced earlier in the chain is round- + # tripped on every subsequent turn. The Nth turn's outgoing input + # must contain at least N-1 reasoning items. + for i in range(1, len(responses)): + outgoing = (responses[i]._prompt_json or {}).get("input") or [] + reasoning_count = sum(1 for it in outgoing if it.get("type") == "reasoning") + # encrypted_content + id are non-empty on each one + for it in outgoing: + if it.get("type") == "reasoning": + assert it.get("encrypted_content"), "encrypted_content lost" + assert it.get("id"), "reasoning id lost" + assert ( + reasoning_count >= i + ), f"turn {i} must echo >= {i} reasoning items, got {reasoning_count}" + + # 3) The captured ReasoningParts on the assistant messages carry the + # opaque metadata that was actually echoed back on the wire. + for i, r in enumerate(responses[:-1]): + rparts = [ + p for m in r.messages() for p in m.parts if isinstance(p, ReasoningPart) + ] + if reasoning_token_counts[i] > 0: + assert rparts, ( + f"turn {i} produced reasoning_tokens={reasoning_token_counts[i]} " + "but no ReasoningPart was persisted" + ) + for rp in rparts: + pm = (rp.provider_metadata or {}).get("openai") or {} + assert pm.get( + "encrypted_content" + ), "ReasoningPart missing encrypted_content" + + +def _responses_reasoning_refresh_stream(): + """Reasoning stream whose final payload carries a different ciphertext. + + OpenAI encrypts reasoning per event, so ``output_item.done`` and the + ``response.completed`` payload hold different encrypted_content for + the same reasoning item. + """ + yield from _responses_reasoning_summary_stream() + yield _responses_sse( + "response.completed", + { + "response": { + "id": "resp_1", + "object": "response", + "created_at": 1700000000, + "model": "gpt-5.5", + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "output": [ + { + "id": "rs_1", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Thinking aloud"}], + "encrypted_content": "encrypted-final", + "status": "completed", + }, + { + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "done", "annotations": []} + ], + }, + ], + "usage": { + "input_tokens": 1, + "output_tokens": 2, + "total_tokens": 3, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 1}, + }, + }, + "sequence_number": 5, + }, + ) + + +def test_responses_reasoning_metadata_refreshed_from_final_payload(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + stream=IteratorStream(_responses_reasoning_refresh_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + + model = llm.get_model("gpt-5.5") + response = model.prompt("hello", key="test") + assert response.text() == "done" + + reasoning_parts = [ + p + for m in response.messages() + for p in m.parts + if isinstance(p, llm.parts.ReasoningPart) + ] + # One reasoning part - the refresh merged into it rather than + # growing a second one - and it carries the final payload's + # ciphertext, the same string response_json holds. + assert len(reasoning_parts) == 1 + metadata = reasoning_parts[0].provider_metadata["openai"] + assert metadata["encrypted_content"] == "encrypted-final" + payload_item = response.response_json["output"][0] + assert payload_item["encrypted_content"] == "encrypted-final" + assert reasoning_parts[0].text == "Thinking aloud" + + +def test_code_interpreter_multi_message_response(httpx_mock): + """Server-side tool execution interleaving multiple message output + items must assemble into multiple assistant Messages.""" + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + json={ + "id": "resp_multi", + "object": "response", + "created_at": 1, + "model": "gpt-5.5", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "STEP ONE", "annotations": []} + ], + }, + { + "type": "code_interpreter_call", + "id": "ci_1", + "status": "completed", + "container_id": "cntr_1", + "code": "print(111*111)", + "outputs": [{"type": "logs", "logs": "12321\n"}], + }, + { + "type": "message", + "id": "msg_2", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "DONE 12321", "annotations": []} + ], + }, + ], + "usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8}, + "status": "completed", + }, + headers={"Content-Type": "application/json"}, + ) + model = llm.get_model("gpt-5.5") + response = model.prompt("count", stream=False, key="test") + response.text() + + messages = response.messages() + assert len(messages) == 2 + first, second = messages + assert [type(p).__name__ for p in first.parts] == [ + "TextPart", + "ToolCallPart", + "ToolResultPart", + ] + assert first.parts[0].text == "STEP ONE" + assert first.parts[1].name == "code_interpreter" + assert first.parts[1].arguments == {"code": "print(111*111)"} + assert first.parts[1].server_executed + assert first.parts[2].output == "12321\n" + assert first.parts[2].server_executed + assert [type(p).__name__ for p in second.parts] == ["TextPart"] + assert second.parts[0].text == "DONE 12321" + + # Server-executed calls are not locally executable + assert response.tool_calls() == [] + + +def test_code_interpreter_streaming_output_and_request(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + stream=IteratorStream(_code_interpreter_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + model = llm.get_model("gpt-5.6-luna") + response = model.prompt("Calculate", tools=[CodeInterpreter()], key="test") + + events = list(response.stream_events()) + assert [(event.type, event.chunk) for event in events] == [ + ("tool_call_name", "code_interpreter"), + ("tool_call_args", json.dumps({"code": "print(6 * 7)"})), + ("tool_result", "42\n"), + ("text", "42"), + ] + assert all(event.server_executed for event in events[:3]) + assert response.tool_calls() == [] + + request_body = json.loads(httpx_mock.get_requests()[-1].content) + assert "code_interpreter_call.outputs" in request_body["include"] + + +def _assert_web_search_streaming_uses_final_payload(response, messages): + parts = [part for message in messages for part in message.parts] + server_parts = [ + part + for part in parts + if isinstance(part, (llm.parts.ToolCallPart, llm.parts.ToolResultPart)) + ] + assert [type(part).__name__ for part in server_parts] == [ + "ToolCallPart", + "ToolResultPart", + ] + tool_call, tool_result = server_parts + final_item = next( + item + for item in response.response_json["output"] + if item["type"] == "web_search_call" + ) + assert len(final_item["action"]["sources"]) == 2 + assert len(final_item["results"]) == 2 + assert tool_call.arguments == final_item["action"] + assert json.loads(tool_result.output) == final_item["results"] + + +def test_web_search_streaming_refreshes_from_final_payload(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + stream=IteratorStream(_web_search_refresh_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + response = llm.get_model("gpt-5.6-luna").prompt( + "Search", + tools=[WebSearch(include_sources=True, include_results=True)], + key="test", + ) + + assert response.text() == "done" + _assert_web_search_streaming_uses_final_payload(response, response.messages()) + + +@pytest.mark.asyncio +async def test_async_web_search_streaming_refreshes_from_final_payload(httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://api.openai.com/v1/responses", + stream=IteratorStream(_web_search_refresh_stream()), + headers={"Content-Type": "text/event-stream"}, + ) + response = llm.get_async_model("gpt-5.6-luna").prompt( + "Search", + tools=[WebSearch(include_sources=True, include_results=True)], + key="test", + ) + + assert await response.text() == "done" + _assert_web_search_streaming_uses_final_payload(response, await response.messages()) + + +def test_server_tool_parts_not_replayed_as_function_calls(): + from llm.parts import Message, TextPart, ToolCallPart, ToolResultPart + + model = llm.get_model("gpt-5.5") + + class FakePrompt: + messages = ( + Message(role="user", parts=[TextPart(text="count")]), + Message( + role="assistant", + parts=[ + TextPart(text="STEP ONE"), + ToolCallPart( + name="code_interpreter", + arguments={"code": "print(1)"}, + tool_call_id="ci_1", + server_executed=True, + ), + ToolResultPart( + name="code_interpreter", + output="1\n", + tool_call_id="ci_1", + server_executed=True, + ), + ], + ), + Message(role="assistant", parts=[TextPart(text="DONE")]), + Message(role="user", parts=[TextPart(text="thanks")]), + ) + + items, _instructions = model._build_responses_input(FakePrompt()) + assert items == [ + {"role": "user", "content": "count"}, + {"role": "assistant", "content": "STEP ONE"}, + {"role": "assistant", "content": "DONE"}, + {"role": "user", "content": "thanks"}, + ] diff --git a/tests/test_options_parameter.py b/tests/test_options_parameter.py new file mode 100644 index 000000000..deb00f544 --- /dev/null +++ b/tests/test_options_parameter.py @@ -0,0 +1,100 @@ +"""Tests for the `options=` parameter on `.prompt()` and `.reply()`. + +The `options={...}` dict form is the documented API; the `**kwargs` form +continues to work undocumented for backwards compatibility. +""" + +import pytest + + +def test_prompt_with_options_dict(mock_model): + mock_model.enqueue(["ok"]) + r = mock_model.prompt("q", options={"max_tokens": 42}) + r.text() + assert r.prompt.options.max_tokens == 42 + assert r.to_dict()["prompt"].get("options") == {"max_tokens": 42} + + +def test_prompt_kwargs_still_work(mock_model): + mock_model.enqueue(["ok"]) + r = mock_model.prompt("q", max_tokens=42) + r.text() + assert r.prompt.options.max_tokens == 42 + + +def test_prompt_options_and_kwargs_merge(mock_model): + # Non-overlapping keys merge cleanly — options= and kwargs both contribute + mock_model.Options.model_rebuild() + mock_model.enqueue(["ok"]) + # Only max_tokens exists on MockModel.Options — use it via options=. + # Pass an empty options dict alongside a kwarg to confirm both paths coexist. + r = mock_model.prompt("q", options={}, max_tokens=7) + r.text() + assert r.prompt.options.max_tokens == 7 + + +def test_prompt_options_and_kwargs_conflict_raises(mock_model): + mock_model.enqueue(["ok"]) + with pytest.raises(TypeError, match="both in options="): + mock_model.prompt("q", options={"max_tokens": 1}, max_tokens=2) + + +def test_conversation_prompt_with_options_dict(mock_model): + mock_model.enqueue(["ok"]) + convo = mock_model.conversation() + r = convo.prompt("q", options={"max_tokens": 99}) + r.text() + assert r.prompt.options.max_tokens == 99 + + +def test_response_reply_with_options_dict(mock_model): + mock_model.enqueue(["first"]) + mock_model.enqueue(["second"]) + r1 = mock_model.prompt("q1", options={"max_tokens": 5}) + r1.text() + r2 = r1.reply("q2", options={"max_tokens": 17}) + r2.text() + assert r2.prompt.options.max_tokens == 17 + + +def test_response_reply_kwargs_still_work(mock_model): + mock_model.enqueue(["first"]) + mock_model.enqueue(["second"]) + r1 = mock_model.prompt("q1", max_tokens=5) + r1.text() + r2 = r1.reply("q2", max_tokens=17) + r2.text() + assert r2.prompt.options.max_tokens == 17 + + +@pytest.mark.asyncio +async def test_async_prompt_with_options_dict(async_mock_model): + # AsyncMockModel inherits the empty base Options (extra="forbid"), + # so pass an empty options dict — this verifies the parameter is + # accepted and the empty-dict path works. + async_mock_model.enqueue(["ok"]) + r = await async_mock_model.prompt("q", options={}).text() + assert r == "ok" + + +@pytest.mark.asyncio +async def test_async_prompt_options_and_kwargs_conflict_raises(async_mock_model): + import llm + + # Build an async model with a real Option field so we can collide them. + class AsyncModelWithOption(llm.AsyncModel): + model_id = "async-with-option" + + class Options(llm.Options): + from typing import Optional as _Opt + + from pydantic import Field as _Field + + max_tokens: int | None = _Field(default=None) + + async def execute(self, prompt, stream, response, conversation): + yield "ok" + + m = AsyncModelWithOption() + with pytest.raises(TypeError, match="both in options="): + await m.prompt("q", options={"max_tokens": 1}, max_tokens=2).text() diff --git a/tests/test_parts.py b/tests/test_parts.py new file mode 100644 index 000000000..ffbf51295 --- /dev/null +++ b/tests/test_parts.py @@ -0,0 +1,2477 @@ +import json + +import pytest + +import llm + + +class TestTextPart: + def test_roundtrip(self): + part = llm.parts.TextPart(text="Hello world") + restored = llm.parts.Part.from_dict(part.to_dict()) + assert restored == part + assert isinstance(restored, llm.parts.TextPart) + assert restored.text == "Hello world" + + def test_to_dict_shape(self): + assert llm.parts.TextPart(text="hi").to_dict() == {"type": "text", "text": "hi"} + + def test_with_provider_metadata(self): + part = llm.parts.TextPart( + text="hi", provider_metadata={"openai": {"flag": True}} + ) + restored = llm.parts.Part.from_dict(part.to_dict()) + assert restored == part + + +class TestReasoningPart: + def test_roundtrip_with_text(self): + part = llm.parts.ReasoningPart(text="Let me think...") + restored = llm.parts.Part.from_dict(part.to_dict()) + assert restored == part + assert restored.text == "Let me think..." + assert restored.redacted is False + + def test_roundtrip_redacted(self): + part = llm.parts.ReasoningPart(text="", redacted=True) + d = part.to_dict() + assert d["redacted"] is True + assert "token_count" not in d + restored = llm.parts.Part.from_dict(d) + assert restored == part + + def test_no_token_count_field(self): + # token_count was removed: opaque token totals live on + # response.token_details, not on the Part. + with pytest.raises(TypeError): + llm.parts.ReasoningPart(text="", redacted=True, token_count=150) + + +class TestToolCallPart: + def test_roundtrip(self): + part = llm.parts.ToolCallPart( + name="search", + arguments={"query": "weather"}, + tool_call_id="call_123", + ) + restored = llm.parts.Part.from_dict(part.to_dict()) + assert restored == part + assert restored.server_executed is False + + def test_server_executed_flag_roundtrips(self): + part = llm.parts.ToolCallPart( + name="web_search", + arguments={"q": "x"}, + tool_call_id="c1", + server_executed=True, + ) + d = part.to_dict() + assert d["server_executed"] is True + restored = llm.parts.Part.from_dict(d) + assert restored.server_executed is True + + +class TestToolResultPart: + def test_roundtrip(self): + part = llm.parts.ToolResultPart( + name="search", output="72F sunny", tool_call_id="c1" + ) + restored = llm.parts.Part.from_dict(part.to_dict()) + assert restored == part + assert restored.exception is None + assert restored.attachments == [] + + def test_with_exception(self): + part = llm.parts.ToolResultPart( + name="t", output="", tool_call_id="c1", exception="boom" + ) + restored = llm.parts.Part.from_dict(part.to_dict()) + assert restored.exception == "boom" + + +class TestAttachmentPart: + def test_roundtrip_with_url(self): + att = llm.Attachment(url="http://example.com/cat.jpg") + part = llm.parts.AttachmentPart(attachment=att) + restored = llm.parts.Part.from_dict(part.to_dict()) + assert isinstance(restored, llm.parts.AttachmentPart) + assert restored.attachment.url == "http://example.com/cat.jpg" + + def test_roundtrip_with_path(self): + att = llm.Attachment(type="image/jpeg", path="/tmp/x.jpg") + part = llm.parts.AttachmentPart(attachment=att) + restored = llm.parts.Part.from_dict(part.to_dict()) + assert restored.attachment.path == "/tmp/x.jpg" + assert restored.attachment.type == "image/jpeg" + + def test_roundtrip_with_bytes_uses_base64(self): + raw = b"\x89PNG fake bytes" + att = llm.Attachment(type="image/png", content=raw) + part = llm.parts.AttachmentPart(attachment=att) + d = part.to_dict() + # Content must be a base64-encoded string in the dict form + assert isinstance(d["attachment"]["content"], str) + import base64 + + assert base64.b64decode(d["attachment"]["content"]) == raw + # And round-trip back to the original bytes + restored = llm.parts.Part.from_dict(d) + assert restored.attachment.content == raw + + def test_json_serializable(self): + att = llm.Attachment(type="image/png", content=b"\x00\x01\x02") + part = llm.parts.AttachmentPart(attachment=att) + # Must survive json dumps/loads + restored = llm.parts.Part.from_dict(json.loads(json.dumps(part.to_dict()))) + assert restored.attachment.content == b"\x00\x01\x02" + + +class TestUnknownPart: + def test_from_dict_unknown_type_raises(self): + with pytest.raises(ValueError): + llm.parts.Part.from_dict({"type": "nonsense"}) + + +class TestRoleNotOnPart: + def test_text_part_has_no_role_attribute(self): + # Role lives on Message. Parts are content-only. + part = llm.parts.TextPart(text="hi") + assert not hasattr(part, "role") + + def test_reasoning_part_has_no_role_attribute(self): + assert not hasattr(llm.parts.ReasoningPart(text=""), "role") + + def test_tool_call_part_has_no_role_attribute(self): + assert not hasattr( + llm.parts.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), + "role", + ) + + +class TestMessage: + def test_roundtrip_simple_user_message(self): + m = llm.Message(role="user", parts=[llm.parts.TextPart(text="hi")]) + restored = llm.Message.from_dict(m.to_dict()) + assert restored == m + + def test_roundtrip_with_provider_metadata(self): + m = llm.Message( + role="assistant", + parts=[llm.parts.TextPart(text="hi")], + provider_metadata={"anthropic": {"signature": "abc"}}, + ) + restored = llm.Message.from_dict(m.to_dict()) + assert restored == m + + def test_roundtrip_mixed_parts(self): + m = llm.Message( + role="assistant", + parts=[ + llm.parts.ReasoningPart(text="Thinking"), + llm.parts.TextPart(text="Result"), + llm.parts.ToolCallPart( + name="search", + arguments={"q": "x"}, + tool_call_id="c1", + ), + ], + ) + restored = llm.Message.from_dict(m.to_dict()) + assert restored == m + + def test_empty_provider_metadata_omitted(self): + m = llm.Message(role="user", parts=[llm.parts.TextPart(text="x")]) + d = m.to_dict() + assert "provider_metadata" not in d + + def test_none_and_empty_provider_metadata_equivalent(self): + m_none = llm.Message(role="user", parts=[llm.parts.TextPart(text="x")]) + m_empty = llm.Message( + role="user", + parts=[llm.parts.TextPart(text="x")], + provider_metadata={}, + ) + # Both serialize the same (empty metadata is omitted) + assert m_none.to_dict() == m_empty.to_dict() + + +class TestHelpers: + def test_user_with_string(self): + m = llm.user("hi") + assert m.role == "user" + assert m.parts == [llm.parts.TextPart(text="hi")] + + def test_assistant_with_string(self): + m = llm.assistant("there") + assert m.role == "assistant" + assert m.parts == [llm.parts.TextPart(text="there")] + + def test_system_with_string(self): + m = llm.system("be brief") + assert m.role == "system" + assert m.parts == [llm.parts.TextPart(text="be brief")] + + def test_tool_message_with_part(self): + tr = llm.parts.ToolResultPart(name="t", output="r", tool_call_id="c1") + m = llm.tool_message(tr) + assert m.role == "tool" + assert m.parts == [tr] + + def test_helper_accepts_attachment(self): + att = llm.Attachment(url="http://example.com/x.jpg") + m = llm.user("describe this", att) + assert m.parts == [ + llm.parts.TextPart(text="describe this"), + llm.parts.AttachmentPart(attachment=att), + ] + + def test_helper_accepts_existing_part(self): + tp = llm.parts.TextPart(text="pre-built") + m = llm.user(tp) + assert m.parts == [tp] + + def test_helper_flattens_one_level(self): + # Nested list gets flattened one level. + m = llm.user(["one", "two"], "three") + assert m.parts == [ + llm.parts.TextPart(text="one"), + llm.parts.TextPart(text="two"), + llm.parts.TextPart(text="three"), + ] + + def test_helper_rejects_unknown_types(self): + with pytest.raises(TypeError): + llm.user(42) + + def test_helper_with_provider_metadata(self): + m = llm.assistant("hi", provider_metadata={"openai": {"id": "x"}}) + assert m.provider_metadata == {"openai": {"id": "x"}} + + +class TestStreamEvent: + def test_dataclass_defaults(self): + ev = llm.parts.StreamEvent(type="text", chunk="hi", part_index=0) + assert ev.type == "text" + assert ev.chunk == "hi" + assert ev.part_index == 0 + assert ev.tool_call_id is None + assert ev.server_executed is False + assert ev.tool_name is None + assert ev.provider_metadata is None + assert ev.message_index == 0 + + def test_all_fields_accepted(self): + ev = llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"q":', + part_index=2, + tool_call_id="c1", + server_executed=True, + tool_name="search", + provider_metadata={"openai": {"x": 1}}, + message_index=1, + ) + assert ev.tool_call_id == "c1" + assert ev.server_executed is True + assert ev.tool_name == "search" + assert ev.provider_metadata == {"openai": {"x": 1}} + assert ev.message_index == 1 + + +# Backward compat for plain-str plugins: iterating a Response still +# yields text strings, response.text() still works, self._chunks is +# still populated. + + +class TestPlainStrPluginCompat: + """A plugin that yields plain str must still work unchanged.""" + + def test_iter_yields_strings(self, mock_model): + mock_model.enqueue(["hello", " ", "world"]) + response = mock_model.prompt("hi") + chunks = list(response) + assert chunks == ["hello", " ", "world"] + + def test_text_returns_concatenation(self, mock_model): + mock_model.enqueue(["hello ", "world"]) + response = mock_model.prompt("hi") + assert response.text() == "hello world" + + def test_chunks_are_preserved(self, mock_model): + mock_model.enqueue(["a", "b", "c"]) + response = mock_model.prompt("hi") + response.text() + assert response._chunks == ["a", "b", "c"] + + +class TestStreamEventsFromPlainStrPlugin: + """When a plugin yields plain str, stream_events synthesizes text events.""" + + def test_stream_events_yields_text_events(self, mock_model): + mock_model.enqueue(["hel", "lo"]) + response = mock_model.prompt("hi") + events = list(response.stream_events()) + assert all(isinstance(e, llm.parts.StreamEvent) for e in events) + assert [e.type for e in events] == ["text", "text"] + assert [e.chunk for e in events] == ["hel", "lo"] + assert all(e.part_index == 0 for e in events) + + def test_response_messages_is_single_assistant_text(self, mock_model): + mock_model.enqueue(["hello"]) + response = mock_model.prompt("hi") + response.text() + messages = response.messages() + assert messages == [ + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hello")]) + ] + + def test_empty_response_has_empty_messages(self, mock_model): + mock_model.enqueue([]) + response = mock_model.prompt("hi") + response.text() + assert response.messages() == [] + + +class TestStreamEventsFromStreamEventPlugin: + """When a plugin yields StreamEvents, they pass through unchanged + and iteration filters to text only.""" + + def test_iter_yields_only_text_chunks(self, mock_model): + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="think ", part_index=0), + llm.parts.StreamEvent(type="text", chunk="hel", part_index=1), + llm.parts.StreamEvent(type="text", chunk="lo", part_index=1), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + chunks = list(response) + assert chunks == ["hel", "lo"] + + def test_stream_events_yields_all_events(self, mock_model): + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.parts.StreamEvent(type="text", chunk="x", part_index=1), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + got = list(response.stream_events()) + assert [e.type for e in got] == ["reasoning", "text"] + + def test_messages_assembles_reasoning_then_text(self, mock_model): + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="thinking", part_index=0), + llm.parts.StreamEvent(type="text", chunk="hello", part_index=1), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages() == [ + llm.Message( + role="assistant", + parts=[ + llm.parts.ReasoningPart(text="thinking"), + llm.parts.TextPart(text="hello"), + ], + ) + ] + + def test_tool_call_name_and_args_merge(self, mock_model): + events = [ + llm.parts.StreamEvent(type="text", chunk="calling", part_index=0), + llm.parts.StreamEvent( + type="tool_call_name", + chunk="search", + part_index=1, + tool_call_id="c1", + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"q":', + part_index=1, + tool_call_id="c1", + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk='"weather"}', + part_index=1, + tool_call_id="c1", + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + msgs = response.messages() + assert len(msgs) == 1 + parts = msgs[0].parts + assert parts == [ + llm.parts.TextPart(text="calling"), + llm.parts.ToolCallPart( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ), + ] + + def test_tool_call_args_unparseable_json_falls_back(self, mock_model): + events = [ + llm.parts.StreamEvent( + type="tool_call_name", + chunk="t", + part_index=0, + tool_call_id="c1", + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk="not json", + part_index=0, + tool_call_id="c1", + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + part = response.messages()[0].parts[0] + assert part.name == "t" + assert part.arguments == {"_raw": "not json"} + + def test_family_mismatch_at_same_part_index_raises(self, mock_model): + events = [ + llm.parts.StreamEvent(type="text", chunk="x", part_index=0), + llm.parts.StreamEvent( + type="tool_call_name", + chunk="t", + part_index=0, + tool_call_id="c1", + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + with pytest.raises(ValueError, match="part_index"): + response.messages() + + def test_provider_metadata_merges_last_wins(self, mock_model): + events = [ + llm.parts.StreamEvent( + type="reasoning", + chunk="think", + part_index=0, + provider_metadata={"anthropic": {"signature": "one"}}, + ), + llm.parts.StreamEvent( + type="reasoning", + chunk="", + part_index=0, + provider_metadata={"anthropic": {"signature": "final"}}, + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + part = response.messages()[0].parts[0] + assert part.provider_metadata == {"anthropic": {"signature": "final"}} + + def test_redacted_reasoning_event_emits_marker_part(self, mock_model): + # A reasoning StreamEvent with redacted=True yields a + # ReasoningPart(text="", redacted=True) marker — opaque token + # totals live on response.token_details, not on the Part. + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="", redacted=True), + llm.parts.StreamEvent(type="text", chunk="hi"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.ReasoningPart(text="", redacted=True), + llm.parts.TextPart(text="hi"), + ] + + def test_redacted_reasoning_hoisted_to_start_when_emitted_late(self, mock_model): + # Plugins typically learn opaque reasoning happened only when + # the final usage chunk arrives, so they emit the marker last. + # The framework hoists redacted reasoning Parts to the start of + # the assembled message so UIs can render them before content. + events = [ + llm.parts.StreamEvent(type="text", chunk="hello"), + llm.parts.StreamEvent(type="reasoning", chunk="", redacted=True), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.ReasoningPart(text="", redacted=True), + llm.parts.TextPart(text="hello"), + ] + + def test_redacted_reasoning_event_default_redacted_is_false(self): + ev = llm.parts.StreamEvent(type="reasoning", chunk="thinking") + assert ev.redacted is False + + def test_metadata_only_reasoning_event_emits_part(self, mock_model): + # Anthropic display:"omitted" thinking arrives as an empty + # thinking block whose signature carries the encrypted + # reasoning state. The signature-only event must still become + # a ReasoningPart or the signature is lost from history. + events = [ + llm.parts.StreamEvent( + type="reasoning", + chunk="", + provider_metadata={"anthropic": {"signature": "sig-omitted"}}, + ), + llm.parts.StreamEvent(type="text", chunk="hi"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.ReasoningPart( + text="", + redacted=False, + provider_metadata={"anthropic": {"signature": "sig-omitted"}}, + ), + llm.parts.TextPart(text="hi"), + ] + assert "sig-omitted" not in response.text() + + def test_metadata_only_reasoning_part_is_not_hoisted(self, mock_model): + # Unlike redacted=True markers, metadata-only reasoning keeps + # its position: Anthropic requires thinking/redacted_thinking + # blocks replayed in their original order. + events = [ + llm.parts.StreamEvent(type="text", chunk="hello", part_index=0), + llm.parts.StreamEvent( + type="reasoning", + chunk="", + part_index=1, + provider_metadata={ + "anthropic": {"type": "redacted_thinking", "data": "opaque"} + }, + ), + llm.parts.StreamEvent(type="text", chunk="bye", part_index=2), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.TextPart(text="hello"), + llm.parts.ReasoningPart( + text="", + redacted=False, + provider_metadata={ + "anthropic": {"type": "redacted_thinking", "data": "opaque"} + }, + ), + llm.parts.TextPart(text="bye"), + ] + + def test_adjacent_reasoning_blocks_with_explicit_index_stay_distinct( + self, mock_model + ): + # Explicit part_index keeps adjacent provider blocks separate: + # a visible signed thinking block followed by a redacted one + # must not merge (each keeps its own metadata and boundaries). + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="visible ", part_index=0), + llm.parts.StreamEvent( + type="reasoning", + chunk="", + part_index=0, + provider_metadata={"anthropic": {"signature": "sig-1"}}, + ), + llm.parts.StreamEvent( + type="reasoning", + chunk="", + part_index=1, + provider_metadata={ + "anthropic": {"type": "redacted_thinking", "data": "blob"} + }, + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=2), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.ReasoningPart( + text="visible ", + redacted=False, + provider_metadata={"anthropic": {"signature": "sig-1"}}, + ), + llm.parts.ReasoningPart( + text="", + redacted=False, + provider_metadata={ + "anthropic": {"type": "redacted_thinking", "data": "blob"} + }, + ), + llm.parts.TextPart(text="answer"), + ] + + def test_empty_reasoning_group_without_metadata_is_dropped(self, mock_model): + # No text, no redacted marker, no metadata: nothing worth + # preserving, so no Part is built. + events = [ + llm.parts.StreamEvent(type="reasoning", chunk=""), + llm.parts.StreamEvent(type="text", chunk="hi"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + response.text() + assert response.messages()[0].parts == [llm.parts.TextPart(text="hi")] + + +class TestPartIndexAutoAllocation: + """When part_index is None (the default), the framework groups + events into Parts using same-family adjacency for text/reasoning + and tool_call_id for tool calls.""" + + def test_streamevent_part_index_defaults_to_none(self): + ev = llm.parts.StreamEvent(type="text", chunk="hi") + assert ev.part_index is None + + def test_consecutive_text_concatenates_into_one_part(self, mock_model): + events = [ + llm.parts.StreamEvent(type="text", chunk="hello "), + llm.parts.StreamEvent(type="text", chunk="world"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages()[0].parts == [llm.parts.TextPart(text="hello world")] + + def test_text_then_reasoning_splits_into_two_parts(self, mock_model): + events = [ + llm.parts.StreamEvent(type="text", chunk="hello"), + llm.parts.StreamEvent(type="reasoning", chunk="thinking"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages()[0].parts == [ + llm.parts.TextPart(text="hello"), + llm.parts.ReasoningPart(text="thinking"), + ] + + def test_text_tool_call_text_produces_three_parts(self, mock_model): + events = [ + llm.parts.StreamEvent(type="text", chunk="before"), + llm.parts.StreamEvent( + type="tool_call_name", + chunk="search", + tool_call_id="c1", + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"q": "x"}', + tool_call_id="c1", + ), + llm.parts.StreamEvent(type="text", chunk="after"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages()[0].parts == [ + llm.parts.TextPart(text="before"), + llm.parts.ToolCallPart( + name="search", arguments={"q": "x"}, tool_call_id="c1" + ), + llm.parts.TextPart(text="after"), + ] + + def test_tool_call_groups_by_tool_call_id(self, mock_model): + events = [ + llm.parts.StreamEvent( + type="tool_call_name", + chunk="search", + tool_call_id="c1", + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"q":', + tool_call_id="c1", + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk='"weather"}', + tool_call_id="c1", + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages()[0].parts == [ + llm.parts.ToolCallPart( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + ] + + def test_parallel_tool_calls_interleaved_by_id(self, mock_model): + # Two tool calls whose args interleave on the wire — must + # still produce two distinct ToolCallParts grouped by id. + events = [ + llm.parts.StreamEvent( + type="tool_call_name", chunk="search", tool_call_id="A" + ), + llm.parts.StreamEvent( + type="tool_call_name", chunk="lookup", tool_call_id="B" + ), + llm.parts.StreamEvent( + type="tool_call_args", chunk='{"q":"a"}', tool_call_id="A" + ), + llm.parts.StreamEvent( + type="tool_call_args", chunk='{"k":"b"}', tool_call_id="B" + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.ToolCallPart( + name="search", arguments={"q": "a"}, tool_call_id="A" + ), + llm.parts.ToolCallPart( + name="lookup", arguments={"k": "b"}, tool_call_id="B" + ), + ] + + def test_tool_result_is_always_own_part(self, mock_model): + events = [ + llm.parts.StreamEvent( + type="tool_call_name", + chunk="web_search", + tool_call_id="c1", + server_executed=True, + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"q":"x"}', + tool_call_id="c1", + server_executed=True, + ), + llm.parts.StreamEvent( + type="tool_result", + chunk="results...", + tool_call_id="c1", + tool_name="web_search", + server_executed=True, + ), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.ToolCallPart( + name="web_search", + arguments={"q": "x"}, + tool_call_id="c1", + server_executed=True, + ), + llm.parts.ToolResultPart( + name="web_search", + output="results...", + tool_call_id="c1", + server_executed=True, + ), + ] + + def test_two_reasoning_blocks_split_by_tool_call(self, mock_model): + # Some providers emit two thinking blocks separated by a tool + # call — those should yield two ReasoningParts, not one. + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="first"), + llm.parts.StreamEvent(type="tool_call_name", chunk="t", tool_call_id="c1"), + llm.parts.StreamEvent(type="tool_call_args", chunk="{}", tool_call_id="c1"), + llm.parts.StreamEvent(type="reasoning", chunk="second"), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.ReasoningPart(text="first"), + llm.parts.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), + llm.parts.ReasoningPart(text="second"), + ] + + def test_parallel_tool_calls_without_id_each_get_own_part(self, mock_model): + # Gemini emits multiple functionCall parts back-to-back without + # a tool_call_id. Each tool_call_name must allocate a fresh + # part — otherwise the N tool calls collapse into one with + # concatenated names and args. + events = [ + llm.parts.StreamEvent(type="tool_call_name", chunk="store_fact"), + llm.parts.StreamEvent(type="tool_call_args", chunk='{"fact":"a"}'), + llm.parts.StreamEvent(type="tool_call_name", chunk="store_fact"), + llm.parts.StreamEvent(type="tool_call_args", chunk='{"fact":"b"}'), + llm.parts.StreamEvent(type="tool_call_name", chunk="store_fact"), + llm.parts.StreamEvent(type="tool_call_args", chunk='{"fact":"c"}'), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.ToolCallPart(name="store_fact", arguments={"fact": "a"}), + llm.parts.ToolCallPart(name="store_fact", arguments={"fact": "b"}), + llm.parts.ToolCallPart(name="store_fact", arguments={"fact": "c"}), + ] + + def test_explicit_part_index_still_works(self, mock_model): + # Back-compat: plugins that pass explicit part_index should + # behave exactly as before. + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.parts.StreamEvent(type="text", chunk="hi", part_index=1), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + assert response.messages()[0].parts == [ + llm.parts.ReasoningPart(text="t"), + llm.parts.TextPart(text="hi"), + ] + + def test_mix_explicit_zero_and_none_for_text_concatenates(self, mock_model): + # Forcing a single TextPart across non-adjacent text bursts: + # plugin pins explicit part_index=0 on the wraparound text + # events, and the tool call in between gets None (auto). + events = [ + llm.parts.StreamEvent(type="text", chunk="before ", part_index=0), + llm.parts.StreamEvent(type="tool_call_name", chunk="t", tool_call_id="c1"), + llm.parts.StreamEvent(type="tool_call_args", chunk="{}", tool_call_id="c1"), + llm.parts.StreamEvent(type="text", chunk="after", part_index=0), + ] + mock_model.enqueue(events) + response = mock_model.prompt("hi") + response.text() + parts = response.messages()[0].parts + assert parts == [ + llm.parts.TextPart(text="before after"), + llm.parts.ToolCallPart(name="t", arguments={}, tool_call_id="c1"), + ] + + +class TestStreamEventsLiveDuringStreaming: + """Client code sees events arrive before the response is done""" + + def test_events_arrive_before_done(self, mock_model): + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="t", part_index=0), + llm.parts.StreamEvent(type="text", chunk="hi", part_index=1), + ] + mock_model.enqueue(events) + response = mock_model.prompt("x") + seen = [] + for event in response.stream_events(): + # Record the _done state at the moment we receive the event. + seen.append((event.type, response._done)) + # Events arrived before _done was set. + assert [s[0] for s in seen] == ["reasoning", "text"] + assert all(not done for _type, done in seen) + # And after the generator is drained, the response is done. + assert response._done + + def test_stream_events_after_done_replays(self, mock_model): + mock_model.enqueue( + [llm.parts.StreamEvent(type="text", chunk="hi", part_index=0)] + ) + response = mock_model.prompt("x") + first = list(response.stream_events()) + # Second call replays from the stored events. + second = list(response.stream_events()) + assert len(first) == 1 + assert [e.type for e in second] == ["text"] + assert [e.chunk for e in second] == ["hi"] + + def test_plain_str_stream_events_after_done_replays(self, mock_model): + mock_model.enqueue(["hello"]) + response = mock_model.prompt("x") + response.text() + events = list(response.stream_events()) + assert len(events) == 1 + assert events[0].type == "text" + assert events[0].chunk == "hello" + + +class TestAsyncStreamEvents: + @pytest.mark.asyncio + async def test_async_stream_events_live(self, async_mock_model): + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="r", part_index=0), + llm.parts.StreamEvent(type="text", chunk="t", part_index=1), + ] + async_mock_model.enqueue(events) + response = async_mock_model.prompt("x") + seen_types = [] + async for event in response.astream_events(): + seen_types.append(event.type) + assert seen_types == ["reasoning", "text"] + + @pytest.mark.asyncio + async def test_async_iter_yields_only_text(self, async_mock_model): + events = [ + llm.parts.StreamEvent(type="reasoning", chunk="r", part_index=0), + llm.parts.StreamEvent(type="text", chunk="hi", part_index=1), + ] + async_mock_model.enqueue(events) + response = async_mock_model.prompt("x") + chunks = [] + async for chunk in response: + chunks.append(chunk) + assert chunks == ["hi"] + + @pytest.mark.asyncio + async def test_async_messages_after_await(self, async_mock_model): + async_mock_model.enqueue(["hi"]) + response = async_mock_model.prompt("x") + await response.text() + assert await response.messages() == [ + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hi")]) + ] + + +class TestMessagesIsCallable: + """response.messages() is a method (matching .text(), .json(), + .tool_calls()) — invocation forces execution if not yet done. + """ + + def test_sync_messages_is_callable_and_returns_list(self, mock_model): + mock_model.enqueue(["hi"]) + response = mock_model.prompt("x") + # No prior .text() or iteration — calling messages() forces + # execution and returns the assembled list. + assert response.messages() == [ + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hi")]) + ] + + def test_sync_messages_after_text_returns_same_list(self, mock_model): + mock_model.enqueue(["hi"]) + response = mock_model.prompt("x") + response.text() + assert response.messages() == [ + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hi")]) + ] + + @pytest.mark.asyncio + async def test_async_messages_is_awaitable(self, async_mock_model): + async_mock_model.enqueue(["hi"]) + response = async_mock_model.prompt("x") + # No prior await — `await response.messages()` forces it. + result = await response.messages() + assert result == [ + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hi")]) + ] + + @pytest.mark.asyncio + async def test_async_messages_after_text_returns_same_list(self, async_mock_model): + async_mock_model.enqueue(["hi"]) + response = async_mock_model.prompt("x") + await response.text() + result = await response.messages() + assert result == [ + llm.Message(role="assistant", parts=[llm.parts.TextPart(text="hi")]) + ] + + +class TestPromptMessagesSynthesis: + """Prompt.messages constructs a Message list from legacy inputs when + messages= wasn't passed explicitly.""" + + def test_empty_prompt_yields_empty_messages(self, mock_model): + from llm.models import Prompt + + p = Prompt(None, model=mock_model) + assert p.messages == [] + + def test_prompt_text_synthesizes_user_message(self, mock_model): + from llm.models import Prompt + + p = Prompt("hi", model=mock_model) + assert p.messages == [ + llm.Message(role="user", parts=[llm.parts.TextPart(text="hi")]) + ] + + def test_system_and_prompt_synthesizes_two_messages(self, mock_model): + from llm.models import Prompt + + p = Prompt("hi", model=mock_model, system="be brief") + assert p.messages == [ + llm.Message(role="system", parts=[llm.parts.TextPart(text="be brief")]), + llm.Message(role="user", parts=[llm.parts.TextPart(text="hi")]), + ] + + def test_attachments_join_user_message(self, mock_model): + from llm.models import Prompt + + att = llm.Attachment(url="http://example.com/a.jpg") + p = Prompt("look", model=mock_model, attachments=[att]) + assert p.messages == [ + llm.Message( + role="user", + parts=[ + llm.parts.TextPart(text="look"), + llm.parts.AttachmentPart(attachment=att), + ], + ) + ] + + def test_tool_results_become_tool_role_message(self, mock_model): + from llm import ToolResult + from llm.models import Prompt + + tr = ToolResult(name="t", output="ok", tool_call_id="c1") + p = Prompt(None, model=mock_model, tool_results=[tr]) + assert p.messages == [ + llm.Message( + role="tool", + parts=[ + llm.parts.ToolResultPart(name="t", output="ok", tool_call_id="c1") + ], + ) + ] + + +class TestPromptMessagesExplicit: + """When messages= is passed, it's authoritative.""" + + def test_explicit_messages_returned_verbatim(self, mock_model): + from llm.models import Prompt + + explicit = [ + llm.system("x"), + llm.user("y"), + ] + p = Prompt(None, model=mock_model, messages=explicit) + assert p.messages == explicit + + def test_explicit_messages_ignores_prompt_kwarg(self, mock_model): + """Prompt stores messages= verbatim. Folding prompt= into the + chain happens in model.prompt() / _build_full_chain before the + Prompt is constructed — internal callers pass chains that + already contain the prompt text, so Prompt itself must not + append it a second time.""" + from llm.models import Prompt + + explicit = [llm.system("x"), llm.user("prior"), llm.user("follow-up")] + p = Prompt("ignored text", model=mock_model, messages=explicit) + assert p.messages == explicit + + def test_explicit_messages_independent_copy(self, mock_model): + """Mutating the caller's list must not mutate Prompt.messages.""" + from llm.models import Prompt + + explicit = [llm.user("x")] + p = Prompt(None, model=mock_model, messages=explicit) + explicit.append(llm.user("later")) + assert p.messages == [llm.user("x")] + + +class TestModelPromptMessagesKwarg: + """model.prompt / conversation.prompt / async counterparts accept + messages= and the list is observable on the resulting Prompt.""" + + def test_model_prompt_accepts_messages(self, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt(messages=[llm.user("hi")]) + response.text() + assert response.prompt.messages == [llm.user("hi")] + + def test_model_prompt_messages_with_system(self, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt(messages=[llm.system("be brief"), llm.user("hi")]) + response.text() + assert response.prompt.messages == [ + llm.system("be brief"), + llm.user("hi"), + ] + + def test_conversation_prompt_accepts_messages(self, mock_model): + mock_model.enqueue(["ok"]) + conv = mock_model.conversation() + response = conv.prompt(messages=[llm.user("q")]) + response.text() + assert response.prompt.messages == [llm.user("q")] + + @pytest.mark.asyncio + async def test_async_model_prompt_accepts_messages(self, async_mock_model): + async_mock_model.enqueue(["ok"]) + response = async_mock_model.prompt(messages=[llm.user("hi")]) + await response.text() + assert response.prompt.messages == [llm.user("hi")] + + @pytest.mark.asyncio + async def test_async_conversation_prompt_accepts_messages(self, async_mock_model): + async_mock_model.enqueue(["ok"]) + conv = async_mock_model.conversation() + response = conv.prompt(messages=[llm.user("q")]) + await response.text() + assert response.prompt.messages == [llm.user("q")] + + +class TestModelPromptMessagesPlusNewInput: + """messages= is the authoritative history; prompt=, fragments=, + attachments= and tool_results= passed alongside are this turn's new + input, folded into the chain so prompt.messages still equals exactly + what the model sees. Previously the prompt text was silently absent + from the chain — and so from the log.""" + + def test_prompt_text_appends_user_message(self, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt("follow-up", messages=[llm.user("original")]) + response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.user("follow-up"), + ] + + def test_fragments_join_the_prompt_text(self, mock_model): + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "question", fragments=["context"], messages=[llm.user("original")] + ) + response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.user("context\nquestion"), + ] + + def test_attachments_join_the_user_message(self, mock_model): + att = llm.Attachment(type="image/png", url="http://example.com/a.png") + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "look", attachments=[att], messages=[llm.user("original")] + ) + response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.Message( + role="user", + parts=[ + llm.parts.TextPart(text="look"), + llm.parts.AttachmentPart(attachment=att), + ], + ), + ] + + def test_tool_results_append_tool_message_before_user(self, mock_model): + tr = llm.ToolResult(name="t", output="ok", tool_call_id="c1") + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "next", messages=[llm.user("original")], tool_results=[tr] + ) + response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.Message( + role="tool", + parts=[ + llm.parts.ToolResultPart(name="t", output="ok", tool_call_id="c1") + ], + ), + llm.user("next"), + ] + + def test_conversation_prompt_folds_too(self, mock_model): + mock_model.enqueue(["ok"]) + conv = mock_model.conversation() + response = conv.prompt("follow-up", messages=[llm.user("original")]) + response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.user("follow-up"), + ] + + @pytest.mark.asyncio + async def test_async_model_prompt_folds_too(self, async_mock_model): + async_mock_model.enqueue(["ok"]) + response = async_mock_model.prompt("follow-up", messages=[llm.user("original")]) + await response.text() + assert response.prompt.messages == [ + llm.user("original"), + llm.user("follow-up"), + ] + + +# Invariant: response.prompt.messages == exactly what the model was +# sent for this turn, regardless of whether the caller used +# model.prompt(messages=[...]), conversation.prompt("text"), or +# response.reply("text"). + + +class TestConversationFullChainInvariant: + def test_explicit_messages_plus_prompt_appends_user_message(self, mock_model): + """Explicit messages= is the authoritative history. A prompt= + passed alongside is this turn's new input and is folded in as a + trailing user message — the model sees it, so the chain (and + therefore the log) must contain it.""" + mock_model.enqueue(["ok"]) + response = mock_model.prompt( + "the new question", + messages=[llm.user("q")], + ) + response.text() + assert response.prompt.messages == [ + llm.user("q"), + llm.user("the new question"), + ] + + def test_conversation_second_turn_prompt_messages_has_full_chain(self, mock_model): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + conv = mock_model.conversation() + + r1 = conv.prompt("q1") + r1.text() + r2 = conv.prompt("q2") + r2.text() + + # r2 was sent the full chain. + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + def test_conversation_third_turn_includes_everything_before(self, mock_model): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + mock_model.enqueue(["a3"]) + conv = mock_model.conversation() + r1 = conv.prompt("q1") + r1.text() + r2 = conv.prompt("q2") + r2.text() + r3 = conv.prompt("q3") + r3.text() + + assert r3.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + llm.assistant("a2"), + llm.user("q3"), + ] + + def test_conversation_first_turn_chain_is_single_user_message(self, mock_model): + mock_model.enqueue(["a1"]) + conv = mock_model.conversation() + r1 = conv.prompt("q1") + r1.text() + assert r1.prompt.messages == [llm.user("q1")] + + def test_conversation_preserves_reasoning_and_tool_call_parts(self, mock_model): + """The chain carries reasoning and tool calls from prior turns, + not just the flat text — required for multi-turn extended + thinking (Claude) and tool-use round-trips.""" + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", chunk="thinking...", part_index=0 + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + mock_model.enqueue(["follow-up answer"]) + conv = mock_model.conversation() + r1 = conv.prompt("q1") + r1.text() + r2 = conv.prompt("q2") + r2.text() + + assert r2.prompt.messages == [ + llm.user("q1"), + llm.Message( + role="assistant", + parts=[ + llm.parts.ReasoningPart(text="thinking..."), + llm.parts.TextPart(text="answer"), + ], + ), + llm.user("q2"), + ] + + @pytest.mark.asyncio + async def test_async_conversation_full_chain(self, async_mock_model): + async_mock_model.enqueue(["a1"]) + async_mock_model.enqueue(["a2"]) + conv = async_mock_model.conversation() + r1 = conv.prompt("q1") + await r1.text() + r2 = conv.prompt("q2") + await r2.text() + + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + +class TestSqliteRehydrateMessages: + """After Response.from_row, response.messages() must still yield the + assistant turn as a TextPart (+ any tool calls). Otherwise + Conversation.prompt builds a broken chain for `llm -c`. + """ + + def test_from_row_response_messages_synthesized_from_chunks( + self, mock_model, tmp_path + ): + import sqlite_utils + + from llm.migrations import migrate + + db = sqlite_utils.Database(str(tmp_path / "logs.db")) + migrate(db) + # log_to_db no longer writes the legacy tables - seed the row + # the way an older version of llm recorded it, since from_row + # is the reader for exactly that history. + db["responses"].insert( + { + "id": "01aaaaaaaaaaaaaaaaaaaaaaaa", + "model": "mock", + "prompt": "q1", + "system": None, + "prompt_json": None, + "options_json": "{}", + "response": "answer text", + "response_json": None, + "conversation_id": None, + "duration_ms": 1, + "datetime_utc": "2025-01-01T00:00:00", + "schema_id": None, + }, + alter=True, + ) + + # Rehydrate the response + row = next(db["responses"].rows) + rehydrated = llm.Response.from_row(db, row) + # _stream_events is empty (SQLite doesn't persist those), but + # _chunks carries the text. response.messages() must fall back + # to synthesizing a TextPart. + assert rehydrated._stream_events == [] + assert rehydrated.messages() == [ + llm.Message( + role="assistant", parts=[llm.parts.TextPart(text="answer text")] + ) + ] + + def test_llm_dash_c_chain_preserves_prior_assistant_turn( + self, mock_model, tmp_path + ): + """End-to-end: a follow-up turn via load_conversation must send + [user(q1), assistant(a1), user(q2)] — not drop the assistant.""" + import sqlite_utils + + from llm.cli import load_conversation + from llm.migrations import migrate + + mock_model.enqueue(["first answer"]) + mock_model.enqueue(["second answer"]) + r1 = mock_model.prompt("q1") + r1.text() + + db_path = tmp_path / "logs.db" + db = sqlite_utils.Database(str(db_path)) + migrate(db) + r1.log_to_db(db) + + conv = load_conversation(None, database=str(db_path)) + r2 = conv.prompt("q2") + r2.text() + + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("first answer"), + llm.user("q2"), + ] + + def test_llm_dash_c_after_logged_tool_chain_preserves_full_chain( + self, mock_model, tmp_path + ): + """A loaded tool-result response must carry the preceding + assistant tool_use. Otherwise Anthropic sees an orphan + tool_result at the start of the continued request.""" + import sqlite_utils + + from llm.cli import load_conversation + from llm.migrations import migrate + + class ToolChainMock(type(mock_model)): + def __init__(self): + super().__init__() + self.calls = 0 + + def execute(self, prompt, stream, response, conversation): + self.calls += 1 + if self.calls == 1: + response.add_tool_call( + llm.ToolCall(name="tick", arguments={}, tool_call_id="c1") + ) + if False: + yield "" + else: + yield "final answer" + + def tick() -> str: + return "tock" + + m = ToolChainMock() + chain_response = m.chain("q1", tools=[tick]) + chain_response.text() + + db_path = tmp_path / "logs.db" + db = sqlite_utils.Database(str(db_path)) + migrate(db) + chain_response.log_to_db(db) + + conv = load_conversation(None, database=str(db_path)) + r3 = conv.prompt("q2") + + assert [m.role for m in r3.prompt.messages] == [ + "user", + "assistant", + "tool", + "assistant", + "user", + ] + assert isinstance(r3.prompt.messages[1].parts[0], llm.parts.ToolCallPart) + assert isinstance(r3.prompt.messages[2].parts[0], llm.parts.ToolResultPart) + assert r3.prompt.messages[2].parts[0].tool_call_id == "c1" + + +class TestAddToolCallWithStreamEvents: + """A plugin may yield StreamEvents *and* call response.add_tool_call(). + The Part list must include the tool call regardless of whether + _stream_events is empty or populated; otherwise persistence drops the + tool call and the next turn ships an orphan tool_result. + """ + + def test_text_yield_plus_add_tool_call_emits_both_parts(self, mock_model): + class TextAndAddToolCallMock(type(mock_model)): + def execute(self, prompt, stream, response, conversation): + yield "answer" + response.add_tool_call( + llm.ToolCall( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + ) + + m = TextAndAddToolCallMock() + response = m.prompt("hi") + response.text() + parts = response.messages()[0].parts + assert llm.parts.TextPart(text="answer") in parts + tool_call_parts = [p for p in parts if isinstance(p, llm.parts.ToolCallPart)] + assert tool_call_parts == [ + llm.parts.ToolCallPart( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + ] + + def test_stream_event_tool_call_plus_matching_add_tool_call_dedups( + self, mock_model + ): + class DualApiMock(type(mock_model)): + def execute(self, prompt, stream, response, conversation): + yield llm.parts.StreamEvent( + type="tool_call_name", chunk="search", tool_call_id="c1" + ) + yield llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"q":"weather"}', + tool_call_id="c1", + ) + response.add_tool_call( + llm.ToolCall( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + ) + + m = DualApiMock() + response = m.prompt("hi") + response.text() + tool_call_parts = [ + p + for p in response.messages()[0].parts + if isinstance(p, llm.parts.ToolCallPart) + ] + assert tool_call_parts == [ + llm.parts.ToolCallPart( + name="search", + arguments={"q": "weather"}, + tool_call_id="c1", + ) + ] + + +class TestMessageIndexAssembly: + def test_message_index_splits_assistant_messages(self, mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent(type="text", chunk="STEP ONE", message_index=0), + llm.parts.StreamEvent( + type="tool_call_name", + chunk="code_interpreter", + tool_call_id="ci1", + server_executed=True, + message_index=0, + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"code": "print(1)"}', + tool_call_id="ci1", + server_executed=True, + message_index=0, + ), + llm.parts.StreamEvent( + type="tool_result", + chunk="1\n", + tool_call_id="ci1", + server_executed=True, + tool_name="code_interpreter", + message_index=0, + ), + llm.parts.StreamEvent(type="text", chunk="STEP TWO", message_index=1), + ] + ) + response = mock_model.prompt("go") + response.text() + messages = response.messages() + assert len(messages) == 2 + assert all(m.role == "assistant" for m in messages) + assert messages[0].parts == [ + llm.parts.TextPart(text="STEP ONE"), + llm.parts.ToolCallPart( + name="code_interpreter", + arguments={"code": "print(1)"}, + tool_call_id="ci1", + server_executed=True, + ), + llm.parts.ToolResultPart( + name="code_interpreter", + output="1\n", + tool_call_id="ci1", + server_executed=True, + ), + ] + assert messages[1].parts == [llm.parts.TextPart(text="STEP TWO")] + + def test_adjacent_text_across_message_boundary_does_not_merge(self, mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent(type="text", chunk="first", message_index=0), + llm.parts.StreamEvent(type="text", chunk="second", message_index=1), + ] + ) + response = mock_model.prompt("go") + response.text() + messages = response.messages() + assert len(messages) == 2 + assert messages[0].parts == [llm.parts.TextPart(text="first")] + assert messages[1].parts == [llm.parts.TextPart(text="second")] + + +class TestResponseReply: + def test_reply_builds_next_turn_from_this_response(self, mock_model): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + + r2 = r1.reply("q2") + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + def test_reply_chains(self, mock_model): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + mock_model.enqueue(["a3"]) + r1 = mock_model.prompt("q1") + r1.text() + r2 = r1.reply("q2") + r2.text() + r3 = r2.reply("q3") + r3.text() + assert r3.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + llm.assistant("a2"), + llm.user("q3"), + ] + + def test_reply_no_prompt_reuses_messages_kwarg(self, mock_model): + """Passing messages= to reply() appends those onto the chain + in place of a new user string.""" + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + r2 = r1.reply(messages=[llm.user("alt")]) + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("alt"), + ] + + def test_reply_from_conversation_response_extends_chain(self, mock_model): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + conv = mock_model.conversation() + r1 = conv.prompt("q1") + r1.text() + r2 = r1.reply("q2") + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + def test_reply_tool_result_attachments_become_user_parts(self, mock_model): + """Attachments returned by tools must surface as a user-role + AttachmentPart message - adapters only emit ToolResultPart.output, + so attachments nested inside the part would be silently lost.""" + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + attachment = llm.Attachment(type="image/png", content=b"fakepng") + r2 = r1.reply( + "describe it", + tool_results=[ + llm.ToolResult( + name="take_photo", + output="took photo", + attachments=[attachment], + tool_call_id="c1", + ) + ], + ) + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.Message( + role="tool", + parts=[ + llm.parts.ToolResultPart( + name="take_photo", output="took photo", tool_call_id="c1" + ) + ], + ), + llm.Message( + role="user", + parts=[llm.parts.AttachmentPart(attachment=attachment)], + ), + llm.user("describe it"), + ] + + @pytest.mark.asyncio + async def test_async_reply_tool_result_attachments_become_user_parts( + self, async_mock_model + ): + async_mock_model.enqueue(["a1"]) + async_mock_model.enqueue(["a2"]) + r1 = async_mock_model.prompt("q1") + await r1.text() + attachment = llm.Attachment(type="image/png", content=b"fakepng") + r2 = await r1.reply( + "describe it", + tool_results=[ + llm.ToolResult( + name="take_photo", + output="took photo", + attachments=[attachment], + tool_call_id="c1", + ) + ], + ) + await r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.Message( + role="tool", + parts=[ + llm.parts.ToolResultPart( + name="take_photo", output="took photo", tool_call_id="c1" + ) + ], + ), + llm.Message( + role="user", + parts=[llm.parts.AttachmentPart(attachment=attachment)], + ), + llm.user("describe it"), + ] + + @pytest.mark.asyncio + async def test_async_reply(self, async_mock_model): + async_mock_model.enqueue(["a1"]) + async_mock_model.enqueue(["a2"]) + r1 = async_mock_model.prompt("q1") + await r1.text() + r2 = await r1.reply("q2") + await r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + def test_reply_with_tool_results_appends_tool_message(self, mock_model): + # model.prompt(...) makes tool calls, the + # caller runs them, then reply(tool_results=...) sends the + # results back in one call. The chain should grow by a + # role="tool" message containing ToolResultParts. + from llm.parts import ( + Message, + ToolCallPart, + ToolResultPart, + ) + + # First-turn assistant message has a tool call. + first_assistant = Message( + role="assistant", + parts=[ToolCallPart(name="echo", arguments={"x": 1}, tool_call_id="c1")], + ) + + class ToolCallMock(type(mock_model)): + supports_tools = True + + def execute(self, prompt, stream, response, conversation): + # Yield the assistant turn's parts as StreamEvents so + # response.messages() contains the tool call. + yield llm.parts.StreamEvent( + type="tool_call_name", + chunk="echo", + tool_call_id="c1", + ) + yield llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"x": 1}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo") + r1.text() + + tool_results = [llm.ToolResult(name="echo", output="ok", tool_call_id="c1")] + # The bug we're fixing: this previously silently dropped the + # tool_results because reply() forwards via messages= and the + # Prompt synthesis path is bypassed. + m.enqueue(["follow-up text"]) + r2 = r1.reply(tool_results=tool_results) + r2.text() + assert r2.prompt.messages == [ + llm.user("call echo"), + first_assistant, + Message( + role="tool", + parts=[ToolResultPart(name="echo", output="ok", tool_call_id="c1")], + ), + ] + + def test_reply_with_tool_results_and_prompt(self, mock_model): + from llm.parts import ToolResultPart + + class ToolCallMock(type(mock_model)): + supports_tools = True + + def execute(self, prompt, stream, response, conversation): + yield llm.parts.StreamEvent( + type="tool_call_name", + chunk="echo", + tool_call_id="c1", + ) + yield llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"x": 1}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo") + r1.text() + m.enqueue(["follow-up"]) + r2 = r1.reply( + "now summarise", + tool_results=[llm.ToolResult(name="echo", output="ok", tool_call_id="c1")], + ) + r2.text() + roles = [m.role for m in r2.prompt.messages] + assert roles == ["user", "assistant", "tool", "user"] + # tool message goes BEFORE the new user prompt. + tool_msg = r2.prompt.messages[2] + assert tool_msg.parts == [ + ToolResultPart(name="echo", output="ok", tool_call_id="c1") + ] + assert r2.prompt.messages[3] == llm.user("now summarise") + + def test_reply_auto_executes_tool_calls_when_none_passed(self, mock_model): + # Zero-arg sugar: response.reply() with tool calls present + # auto-executes them and threads results back into the chain. + from llm.parts import ToolResultPart + + executed = [] + + def echo(x: int) -> str: + executed.append(x) + return f"echo:{x}" + + class ToolCallMock(type(mock_model)): + supports_tools = True + + def execute(self, prompt, stream, response, conversation): + response.add_tool_call( + llm.ToolCall(name="echo", arguments={"x": 42}, tool_call_id="c1") + ) + yield llm.parts.StreamEvent( + type="tool_call_name", chunk="echo", tool_call_id="c1" + ) + yield llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"x": 42}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo", tools=[echo]) + r1.text() + + m.enqueue(["follow-up"]) + # No tool_results passed — sugar kicks in and auto-executes. + r2 = r1.reply() + r2.text() + + assert executed == [42] + # The tool message landed in the chain. + roles = [msg.role for msg in r2.prompt.messages] + assert roles == ["user", "assistant", "tool"] + tool_msg = r2.prompt.messages[2] + assert tool_msg.parts == [ + ToolResultPart(name="echo", output="echo:42", tool_call_id="c1") + ] + + def test_reply_auto_execute_with_prompt(self, mock_model): + # reply("more text") with tool calls present also auto-executes + # so the user prompt can land after the tool results. + executed = [] + + def echo(x: int) -> str: + executed.append(x) + return "out" + + class ToolCallMock(type(mock_model)): + supports_tools = True + + def execute(self, prompt, stream, response, conversation): + response.add_tool_call( + llm.ToolCall(name="echo", arguments={"x": 1}, tool_call_id="c1") + ) + yield llm.parts.StreamEvent( + type="tool_call_name", chunk="echo", tool_call_id="c1" + ) + yield llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"x": 1}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo", tools=[echo]) + r1.text() + m.enqueue(["follow-up"]) + r2 = r1.reply("now summarise") + r2.text() + assert executed == [1] + roles = [msg.role for msg in r2.prompt.messages] + assert roles == ["user", "assistant", "tool", "user"] + + def test_reply_explicit_tool_results_skips_auto_execute(self, mock_model): + # Passing tool_results= explicitly overrides the sugar — the + # tool function does NOT run (caller already ran it / wants + # custom results). + executed = [] + + def echo(x: int) -> str: + executed.append(x) + return "should not see" + + class ToolCallMock(type(mock_model)): + supports_tools = True + + def execute(self, prompt, stream, response, conversation): + yield llm.parts.StreamEvent( + type="tool_call_name", chunk="echo", tool_call_id="c1" + ) + yield llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"x": 1}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo", tools=[echo]) + r1.text() + m.enqueue(["follow-up"]) + r2 = r1.reply( + tool_results=[ + llm.ToolResult(name="echo", output="custom", tool_call_id="c1") + ] + ) + r2.text() + assert executed == [] # echo was NOT called + tool_msg = r2.prompt.messages[2] + assert tool_msg.parts[0].output == "custom" + + def test_reply_no_tool_calls_no_tool_message(self, mock_model): + # reply() on a response without tool calls is unchanged — no + # tool message gets injected. + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + r2 = r1.reply() + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + ] + + @pytest.mark.asyncio + async def test_async_reply_auto_executes_tool_calls(self, async_mock_model): + # Async reply() is a coroutine; with tool calls present the + # zero-arg sugar awaits execute_tool_calls() internally. + from llm.parts import ToolResultPart + + executed = [] + + async def echo(x: int) -> str: + executed.append(x) + return f"echo:{x}" + + class ToolCallMock(type(async_mock_model)): + supports_tools = True + + async def execute(self, prompt, stream, response, conversation): + response.add_tool_call( + llm.ToolCall(name="echo", arguments={"x": 7}, tool_call_id="c1") + ) + yield llm.parts.StreamEvent( + type="tool_call_name", chunk="echo", tool_call_id="c1" + ) + yield llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"x": 7}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo", tools=[echo]) + await r1.text() + m.enqueue(["follow-up"]) + r2 = await r1.reply() + await r2.text() + assert executed == [7] + tool_msg = r2.prompt.messages[2] + assert tool_msg.parts == [ + ToolResultPart(name="echo", output="echo:7", tool_call_id="c1") + ] + + @pytest.mark.asyncio + async def test_async_reply_with_tool_results(self, async_mock_model): + from llm.parts import ( + Message, + ToolCallPart, + ToolResultPart, + ) + + class ToolCallMock(type(async_mock_model)): + supports_tools = True + + async def execute(self, prompt, stream, response, conversation): + yield llm.parts.StreamEvent( + type="tool_call_name", + chunk="echo", + tool_call_id="c1", + ) + yield llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"x": 1}', + tool_call_id="c1", + ) + + m = ToolCallMock() + r1 = m.prompt("call echo") + await r1.text() + m.enqueue(["follow-up"]) + r2 = await r1.reply( + tool_results=[llm.ToolResult(name="echo", output="ok", tool_call_id="c1")] + ) + await r2.text() + assert r2.prompt.messages == [ + llm.user("call echo"), + Message( + role="assistant", + parts=[ + ToolCallPart(name="echo", arguments={"x": 1}, tool_call_id="c1") + ], + ), + Message( + role="tool", + parts=[ToolResultPart(name="echo", output="ok", tool_call_id="c1")], + ), + ] + + +# chain() propagates system across tool-result turns + + +class TestChainPropagatesSystem: + """On a tool-result turn within a chain loop, the Prompt must + carry forward the original system= and system_fragments= so + adapters that read prompt.system (OpenAI and other + stateless-per-turn providers) see it on every call.""" + + def assert_system(self, prompt, *expected): + assert prompt.messages[0].role == "system" + for e in expected: + assert e in prompt.system + assert e in prompt.messages[0].parts[0].text + + def test_sync_chain_tool_result_turn_preserves_system(self, mock_model): + # First turn: fake a tool call so the chain iterates. + tool_call = llm.ToolCall(tool_call_id="c1", name="tick", arguments={}) + + class ChainMock(type(mock_model)): + def execute(self, prompt, stream, response, conversation): + if not self._queue: + yield "done" + return + msgs = self._queue.pop(0) + yield from msgs + if not response._tool_calls: + response.add_tool_call(tool_call) + + def tick() -> str: + "Tick" + return "tock" + + m = ChainMock() + m.enqueue(["tool-turn"]) # first response; chain will loop + m.enqueue(["final"]) # second response, after tool results + + chain = m.chain("q", system="be brief", tools=[tick]) + list(chain.responses()) + # Second response was the tool-result turn. + self.assert_system(chain._responses[1].prompt, "be brief") + + def test_sync_chain_tool_result_turn_preserves_system_fragments(self, mock_model): + tool_call = llm.ToolCall(tool_call_id="c1", name="tick", arguments={}) + + class ChainMock(type(mock_model)): + def execute(self, prompt, stream, response, conversation): + if not self._queue: + yield "done" + return + msgs = self._queue.pop(0) + yield from msgs + if not response._tool_calls: + response.add_tool_call(tool_call) + + def tick() -> str: + "Tick" + return "tock" + + m = ChainMock() + m.enqueue(["tool-turn"]) + m.enqueue(["final"]) + + chain = m.chain( + "q", + system="inline sys", + system_fragments=["fragment A", "fragment B"], + tools=[tick], + ) + list(chain.responses()) + self.assert_system( + chain._responses[1].prompt, "inline sys", "fragment A", "fragment B" + ) + + @pytest.mark.asyncio + async def test_async_chain_tool_result_turn_preserves_system( + self, async_mock_model + ): + tool_call = llm.ToolCall(tool_call_id="c1", name="tick", arguments={}) + + class AsyncChainMock(type(async_mock_model)): + supports_tools = True + + async def execute(self, prompt, stream, response, conversation): + if not self._queue: + yield "done" + return + msgs = self._queue.pop(0) + for m in msgs: + yield m + if not response._tool_calls: + response.add_tool_call(tool_call) + + def tick() -> str: + "Tick" + return "tock" + + m = AsyncChainMock() + m.enqueue(["tool-turn"]) + m.enqueue(["final"]) + + chain = m.chain("q", system="be brief", tools=[tick]) + responses = [] + async for r in chain.responses(): + responses.append(r) + self.assert_system(responses[1].prompt, "be brief") + + def test_chain_includes_system_in_messages(self, mock_model): + chain = mock_model.chain("q", system="be brief") + self.assert_system(chain.prompt, "be brief") + + +# chain() accepts messages= (parity with prompt()) + + +class TestChainMessagesKwarg: + def test_conversation_chain_accepts_messages(self, mock_model): + mock_model.enqueue(["ok"]) + conv = mock_model.conversation() + chain = conv.chain(messages=[llm.user("explicit")]) + chain.text() + r1 = chain._responses[0] + assert r1.prompt.messages == [llm.user("explicit")] + + def test_model_chain_accepts_messages(self, mock_model): + mock_model.enqueue(["ok"]) + chain = mock_model.chain(messages=[llm.user("explicit")]) + chain.text() + r1 = chain._responses[0] + assert r1.prompt.messages == [llm.user("explicit")] + + def test_chain_messages_plus_prompt_folds_user_message(self, mock_model): + """Parity with prompt(): messages= is the history and a prompt= + passed alongside is appended as this turn's user message.""" + mock_model.enqueue(["ok"]) + chain = mock_model.chain( + "the new question", + messages=[llm.user("explicit")], + ) + chain.text() + r1 = chain._responses[0] + assert r1.prompt.messages == [ + llm.user("explicit"), + llm.user("the new question"), + ] + + def test_chain_with_messages_and_prior_conversation(self, mock_model): + """Explicit messages= on chain() replaces history reconstruction; + the chain starts from that exact list.""" + mock_model.enqueue(["first"]) + mock_model.enqueue(["second"]) + conv = mock_model.conversation() + r1 = conv.prompt("prior") + r1.text() + + # Now start a chain with explicit messages= — prior turn is + # ignored (consistent with prompt() behavior). + chain = conv.chain(messages=[llm.user("fresh start")]) + chain.text() + first_chain_response = chain._responses[0] + assert first_chain_response.prompt.messages == [llm.user("fresh start")] + + @pytest.mark.asyncio + async def test_async_conversation_chain_accepts_messages(self, async_mock_model): + async_mock_model.enqueue(["ok"]) + conv = async_mock_model.conversation() + chain = conv.chain(messages=[llm.user("explicit")]) + await chain.text() + r1 = chain._responses[0] + assert r1.prompt.messages == [llm.user("explicit")] + + @pytest.mark.asyncio + async def test_async_model_chain_accepts_messages(self, async_mock_model): + async_mock_model.enqueue(["ok"]) + chain = async_mock_model.chain(messages=[llm.user("explicit")]) + await chain.text() + r1 = chain._responses[0] + assert r1.prompt.messages == [llm.user("explicit")] + + +# Response.to_dict / Response.from_dict + + +class TestResponseToDictFromDict: + def test_to_dict_captures_chain_and_output(self, mock_model): + mock_model.enqueue(["hello"]) + r = mock_model.prompt("hi") + r.text() + + d = r.to_dict() + assert d["model"] == "mock" + assert d["prompt"]["messages"] == [llm.user("hi").to_dict()] + assert d["messages"] == [llm.assistant("hello").to_dict()] + + def test_from_dict_rehydrates_with_messages(self, mock_model): + mock_model.enqueue(["hello"]) + r = mock_model.prompt("hi") + r.text() + payload = json.dumps(r.to_dict()) + + restored = llm.Response.from_dict(json.loads(payload)) + assert restored._done + assert restored.text() == "hello" + assert restored.messages() == [llm.assistant("hello")] + assert restored.prompt.messages == [llm.user("hi")] + + def test_from_dict_then_reply_continues_conversation(self, mock_model): + mock_model.enqueue(["a1"]) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + + # Serialize across the process boundary + payload = json.dumps(r1.to_dict()) + restored = llm.Response.from_dict(json.loads(payload)) + + # Continue from the restored response + r2 = restored.reply("q2") + r2.text() + assert r2.prompt.messages == [ + llm.user("q1"), + llm.assistant("a1"), + llm.user("q2"), + ] + + def test_to_dict_preserves_reasoning_and_signatures(self, mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", + chunk="thinking...", + part_index=0, + provider_metadata={"anthropic": {"signature": "sig-abc"}}, + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + r = mock_model.prompt("q") + r.text() + + payload = json.dumps(r.to_dict()) + restored = llm.Response.from_dict(json.loads(payload)) + + msgs = restored.messages() + assert msgs[0].role == "assistant" + assert isinstance(msgs[0].parts[0], llm.parts.ReasoningPart) + assert msgs[0].parts[0].text == "thinking..." + assert msgs[0].parts[0].provider_metadata == { + "anthropic": {"signature": "sig-abc"} + } + + def test_from_dict_reply_includes_prior_reasoning_in_chain(self, mock_model): + """a reply() after from_dict() sends the thinking signature + back to the model for multi-turn extended thinking.""" + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", + chunk="thinking...", + part_index=0, + provider_metadata={"anthropic": {"signature": "sig-xyz"}}, + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + mock_model.enqueue(["a2"]) + r1 = mock_model.prompt("q1") + r1.text() + + payload = json.dumps(r1.to_dict()) + restored = llm.Response.from_dict(json.loads(payload)) + r2 = restored.reply("q2") + r2.text() + + # The signature must be in the chain sent to the model. + chain = r2.prompt.messages + reasoning_parts = [ + p for m in chain for p in m.parts if isinstance(p, llm.parts.ReasoningPart) + ] + assert len(reasoning_parts) == 1 + assert reasoning_parts[0].provider_metadata == { + "anthropic": {"signature": "sig-xyz"} + } + + def test_to_dict_captures_options(self, mock_model): + mock_model.enqueue(["ok"]) + r = mock_model.prompt("hi", max_tokens=42) + r.text() + + d = r.to_dict() + assert d["prompt"]["options"] == {"max_tokens": 42} + + def test_from_dict_options_restored(self, mock_model): + mock_model.enqueue(["ok"]) + r = mock_model.prompt("hi", max_tokens=42) + r.text() + + payload = json.dumps(r.to_dict()) + restored = llm.Response.from_dict(json.loads(payload)) + assert restored.prompt.options.max_tokens == 42 + + def test_message_from_dict_static_method_unchanged(self): + m = llm.assistant("hi") + assert llm.Message.from_dict(m.to_dict()) == m + + +class TestChainResponseStreamEvents: + def test_sync_chain_stream_events_yields_text_when_no_tools(self, mock_model): + # Chain with no tool calls is a single-response chain — its + # stream_events should concatenate from each underlying response. + mock_model.enqueue( + [llm.parts.StreamEvent(type="text", chunk="done", part_index=0)] + ) + chain = mock_model.conversation().chain("q") + events = list(chain.stream_events()) + assert [e.type for e in events] == ["text"] + assert [e.chunk for e in events] == ["done"] + + @pytest.mark.asyncio + async def test_async_chain_astream_events_yields(self, async_mock_model): + async_mock_model.enqueue( + [llm.parts.StreamEvent(type="text", chunk="done", part_index=0)] + ) + chain = async_mock_model.conversation().chain("q") + events = [] + async for event in chain.astream_events(): + events.append(event) + assert [e.type for e in events] == ["text"] + + +# Client-side serialization round-trip +# +# A library user can persist a conversation by serializing response.messages +# to JSON and later re-inflate it as messages=[...] on a follow-up prompt. +# No SQLite involvement. + + +class TestClientSerializationRoundTrip: + def test_response_messages_json_roundtrip(self, mock_model): + mock_model.enqueue(["hello there"]) + r = mock_model.prompt("hi") + r.text() + + # Serialize via Message.to_dict / json.dumps + payload = json.dumps([m.to_dict() for m in r.messages()]) + # Deserialize — no LLM state needed beyond the types. + restored = [llm.Message.from_dict(d) for d in json.loads(payload)] + + assert restored == r.messages() + + def test_rebuilt_messages_reach_plugin_via_prompt(self, mock_model): + """Round-trip: serialize messages from turn 1, re-inflate, send + as messages= to turn 2. The plugin sees the full chain.""" + # Turn 1 + mock_model.enqueue(["turn 1 answer"]) + r1 = mock_model.prompt("turn 1 question") + r1.text() + + # Persist everything the client cares about. + history = [llm.user("turn 1 question").to_dict()] + [ + m.to_dict() for m in r1.messages() + ] + payload = json.dumps(history) + + # Later — rebuild from the wire form and continue. + rebuilt = [llm.Message.from_dict(d) for d in json.loads(payload)] + mock_model.enqueue(["turn 2 answer"]) + r2 = mock_model.prompt(messages=rebuilt + [llm.user("turn 2 question")]) + r2.text() + + # The plugin saw the full structured history on prompt.messages. + assert r2.prompt.messages == rebuilt + [llm.user("turn 2 question")] + assert r2.messages() == [llm.assistant("turn 2 answer")] + + def test_roundtrip_preserves_tool_calls_and_results(self, mock_model): + """Assistant messages with tool calls + subsequent tool role + messages survive json round-trip intact.""" + messages = [ + llm.user("what's the weather?"), + llm.assistant( + "let me check", + llm.parts.ToolCallPart( + name="get_weather", + arguments={"city": "Paris"}, + tool_call_id="c1", + ), + ), + llm.tool_message( + llm.parts.ToolResultPart( + name="get_weather", + output="sunny", + tool_call_id="c1", + ) + ), + ] + payload = json.dumps([m.to_dict() for m in messages]) + restored = [llm.Message.from_dict(d) for d in json.loads(payload)] + assert restored == messages + + def test_roundtrip_preserves_redacted_reasoning(self, mock_model): + """The redacted=True marker on a ReasoningPart survives + round-trip — UIs use it to show that opaque reasoning happened + in this turn (the actual token count lives on response usage).""" + msg = llm.Message( + role="assistant", + parts=[ + llm.parts.ReasoningPart(text="", redacted=True), + llm.parts.TextPart(text="result"), + ], + ) + restored = llm.Message.from_dict(json.loads(json.dumps(msg.to_dict()))) + assert restored == msg + + def test_roundtrip_preserves_provider_metadata(self, mock_model): + msg = llm.Message( + role="assistant", + parts=[ + llm.parts.ReasoningPart( + text="thinking", + provider_metadata={"anthropic": {"signature": "abc"}}, + ), + llm.parts.TextPart(text="answer"), + ], + ) + restored = llm.Message.from_dict(json.loads(json.dumps(msg.to_dict()))) + assert restored == msg diff --git a/tests/test_pause_resume.py b/tests/test_pause_resume.py new file mode 100644 index 000000000..25bb37a24 --- /dev/null +++ b/tests/test_pause_resume.py @@ -0,0 +1,391 @@ +"""Tests for llm.PauseChain and chain resume from message history.""" + +import asyncio +import json + +import pytest + +import llm +from llm.parts import Message, TextPart, ToolCallPart, ToolResultPart + +# ---- PauseChain ---- + + +def test_pause_chain_sync_model(): + after_calls = [] + + def needs_input(path: str) -> str: + raise llm.PauseChain("waiting for approval") + + def before(tool, tool_call): + pass + + def after(tool, tool_call, tool_result): + after_calls.append(tool_result.name) + + model = llm.get_model("echo") + chain = model.chain( + json.dumps( + {"tool_calls": [{"name": "needs_input", "arguments": {"path": "/tmp"}}]} + ), + tools=[needs_input], + before_call=before, + after_call=after, + ) + with pytest.raises(llm.PauseChain) as exc_info: + chain.text() + + pause = exc_info.value + assert str(pause) == "waiting for approval" + assert pause.tool_call is not None + assert pause.tool_call.name == "needs_input" + assert pause.tool_call.arguments == {"path": "/tmp"} + assert pause.tool_call.tool_call_id.startswith("tc_") + assert pause.tool_results == [] + # after_call must not fire for the paused tool + assert after_calls == [] + # The response that requested the tool call completed normally + assert len(chain._responses) == 1 + + +@pytest.mark.asyncio +async def test_pause_chain_async_model_siblings_complete(): + after_calls = [] + executed = [] + + async def needs_input() -> str: + raise llm.PauseChain("hold on") + + async def sibling() -> str: + await asyncio.sleep(0.01) + executed.append("sibling") + return "done" + + async def after(tool, tool_call, tool_result): + after_calls.append(tool_result.name) + + model = llm.get_async_model("echo") + chain = model.chain( + json.dumps({"tool_calls": [{"name": "needs_input"}, {"name": "sibling"}]}), + tools=[needs_input, sibling], + after_call=after, + ) + with pytest.raises(llm.PauseChain) as exc_info: + await chain.text() + + pause = exc_info.value + assert pause.tool_call.name == "needs_input" + # The concurrent sibling ran to completion - no orphaned tasks + assert executed == ["sibling"] + assert after_calls == ["sibling"] + # Completed sibling results ride on the exception + assert [r.name for r in pause.tool_results] == ["sibling"] + assert pause.tool_results[0].output == "done" + + +def test_pause_chain_sync_model_stops_remaining_calls(): + executed = [] + + def pauser() -> str: + raise llm.PauseChain("wait") + + def later() -> str: + executed.append("later") + return "x" + + model = llm.get_model("echo") + chain = model.chain( + json.dumps({"tool_calls": [{"name": "pauser"}, {"name": "later"}]}), + tools=[pauser, later], + ) + with pytest.raises(llm.PauseChain) as exc_info: + chain.text() + # Sequential execution stops at the pause; later call never starts, + # so it can safely re-execute on resume. + assert executed == [] + assert exc_info.value.tool_results == [] + + +@pytest.mark.asyncio +async def test_pause_chain_async_first_of_two_pauses_propagates(): + async def pause_a() -> str: + raise llm.PauseChain("a") + + async def pause_b() -> str: + raise llm.PauseChain("b") + + model = llm.get_async_model("echo") + chain = model.chain( + json.dumps({"tool_calls": [{"name": "pause_a"}, {"name": "pause_b"}]}), + tools=[pause_a, pause_b], + ) + with pytest.raises(llm.PauseChain) as exc_info: + await chain.text() + assert str(exc_info.value) == "a" + assert exc_info.value.tool_call.name == "pause_a" + + +@pytest.mark.asyncio +async def test_async_hook_exception_does_not_orphan_siblings(): + """Defined failure semantics: an exception raised by an after_call + hook propagates only after all concurrent tool tasks finish.""" + executed = [] + + async def boomer() -> str: + return "boom" + + async def slow() -> str: + await asyncio.sleep(0.05) + executed.append("slow") + return "ok" + + async def after(tool, tool_call, tool_result): + if tool_result.name == "boomer": + raise ValueError("hook bug") + + model = llm.get_async_model("echo") + chain = model.chain( + json.dumps({"tool_calls": [{"name": "boomer"}, {"name": "slow"}]}), + tools=[boomer, slow], + after_call=after, + ) + with pytest.raises(ValueError, match="hook bug"): + await chain.text() + # The slow sibling was not orphaned mid-flight + assert executed == ["slow"] + + +@pytest.mark.asyncio +async def test_pause_chain_async_model_sync_tool(): + def pauser() -> str: + raise llm.PauseChain("wait") + + model = llm.get_async_model("echo") + chain = model.chain( + json.dumps({"tool_calls": [{"name": "pauser"}]}), + tools=[pauser], + ) + with pytest.raises(llm.PauseChain) as exc_info: + await chain.text() + assert exc_info.value.tool_call.name == "pauser" + + +# ---- chain resume from message history ---- + + +def _pending_history(tool_call_id="tc_resume1"): + return [ + Message(role="user", parts=[TextPart(text="Convert hello to uppercase")]), + Message( + role="assistant", + parts=[ + ToolCallPart( + name="upper", + arguments={"text": "hello"}, + tool_call_id=tool_call_id, + ) + ], + ), + ] + + +def test_chain_resumes_trailing_pending_tool_calls(): + executed = [] + hook_calls = [] + + def upper(text: str) -> str: + executed.append(text) + return text.upper() + + def before(tool, tool_call): + hook_calls.append(("before", tool_call.name, tool_call.tool_call_id)) + + def after(tool, tool_call, tool_result): + hook_calls.append(("after", tool_result.name, tool_result.tool_call_id)) + + model = llm.get_model("echo") + chain = model.chain( + None, + messages=_pending_history(), + tools=[upper], + before_call=before, + after_call=after, + ) + output = chain.text() + + # The pending call executed through the normal hook machinery + assert executed == ["hello"] + assert hook_calls == [ + ("before", "upper", "tc_resume1"), + ("after", "upper", "tc_resume1"), + ] + # The model then received the tool result (echo renders + # prompt.tool_results), correlated by the original id + data = json.loads(output) + assert data["tool_results"] == [ + {"name": "upper", "output": "HELLO", "tool_call_id": "tc_resume1"} + ] + # Exactly one provider call was made + assert len(chain._responses) == 1 + + +@pytest.mark.asyncio +async def test_chain_resumes_trailing_pending_tool_calls_async(): + executed = [] + + async def upper(text: str) -> str: + executed.append(text) + return text.upper() + + model = llm.get_async_model("echo") + chain = model.chain(None, messages=_pending_history(), tools=[upper]) + output = await chain.text() + + assert executed == ["hello"] + data = json.loads(output) + assert data["tool_results"] == [ + {"name": "upper", "output": "HELLO", "tool_call_id": "tc_resume1"} + ] + + +def test_resume_skips_calls_that_already_have_results(): + executed = [] + + def first() -> str: + executed.append("first") + return "one" + + def second() -> str: + executed.append("second") + return "two" + + history = [ + Message(role="user", parts=[TextPart(text="go")]), + Message( + role="assistant", + parts=[ + ToolCallPart(name="first", arguments={}, tool_call_id="tc_a"), + ToolCallPart(name="second", arguments={}, tool_call_id="tc_b"), + ], + ), + Message( + role="tool", + parts=[ToolResultPart(name="first", output="one", tool_call_id="tc_a")], + ), + ] + model = llm.get_model("echo") + chain = model.chain(None, messages=history, tools=[first, second]) + output = chain.text() + + assert executed == ["second"] + data = json.loads(output) + assert data["tool_results"] == [ + {"name": "second", "output": "two", "tool_call_id": "tc_b"} + ] + + +def test_no_resume_when_conversation_moved_on(): + executed = [] + + def upper(text: str) -> str: + executed.append(text) + return text.upper() + + history = _pending_history() + [ + Message(role="user", parts=[TextPart(text="never mind")]), + ] + model = llm.get_model("echo") + chain = model.chain(None, messages=history, tools=[upper]) + chain.text() + assert executed == [] + + +def test_no_resume_without_tools(): + model = llm.get_model("echo") + chain = model.chain(None, messages=_pending_history()) + # No tools provided: nothing to execute, chain proceeds normally + output = chain.text() + assert "tool_results" not in json.loads(output) + + +def test_resume_matches_idless_calls_by_name(): + # Histories persisted before guaranteed ids may have None ids + executed = [] + + def upper(text: str) -> str: + executed.append(text) + return text.upper() + + history = [ + Message(role="user", parts=[TextPart(text="go")]), + Message( + role="assistant", + parts=[ + ToolCallPart(name="upper", arguments={"text": "a"}, tool_call_id=None), + ToolCallPart(name="upper", arguments={"text": "b"}, tool_call_id=None), + ], + ), + Message( + role="tool", + parts=[ToolResultPart(name="upper", output="A", tool_call_id=None)], + ), + ] + model = llm.get_model("echo") + chain = model.chain(None, messages=history, tools=[upper]) + chain.text() + # One result already present: only one of the two calls re-executes + assert executed == ["b"] + + +def test_resume_ignores_server_executed_calls(): + executed = [] + + def upper(text: str) -> str: + executed.append(text) + return text.upper() + + history = [ + Message(role="user", parts=[TextPart(text="go")]), + Message( + role="assistant", + parts=[ + ToolCallPart( + name="upper", + arguments={"text": "x"}, + tool_call_id="tc_srv", + server_executed=True, + ) + ], + ), + ] + model = llm.get_model("echo") + chain = model.chain(None, messages=history, tools=[upper]) + chain.text() + assert executed == [] + + +def test_resumed_tool_can_pause_again(): + def needs_more(text: str) -> str: + raise llm.PauseChain("second question") + + history = [ + Message(role="user", parts=[TextPart(text="go")]), + Message( + role="assistant", + parts=[ + ToolCallPart( + name="needs_more", + arguments={"text": "x"}, + tool_call_id="tc_again", + ) + ], + ), + ] + model = llm.get_model("echo") + chain = model.chain(None, messages=history, tools=[needs_more]) + with pytest.raises(llm.PauseChain) as exc_info: + chain.text() + assert exc_info.value.tool_call.name == "needs_more" + assert exc_info.value.tool_call.tool_call_id == "tc_again" + # No provider call was made: the chain paused before reaching the model + assert len(chain._responses) == 0 diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 52203925f..7b8cec56a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,13 +1,17 @@ -from click.testing import CliRunner -import click import importlib +import inspect import json -import llm -from llm.tools import llm_version, llm_time -from llm import cli, hookimpl, plugins, get_template_loaders, get_fragment_loaders import pathlib +import re +from unittest.mock import ANY + +import click import pytest -import textwrap +from click.testing import CliRunner + +import llm +from llm import cli, get_fragment_loaders, get_template_loaders, hookimpl, plugins +from llm.tools import llm_time, llm_version def test_register_commands(): @@ -174,18 +178,20 @@ def register_fragment_loaders(self, register): cli.cli, ["-m", "echo", "-f", "mixed:x"], catch_exceptions=False ) assert result3.exit_code == 0 - result3.output.strip == textwrap.dedent( - """\ - system: - - - prompt: - one:x - - attachments: - - https://example.com/attachment.png - """ - ).strip() + assert json.loads(result3.output) == { + "prompt": "one:x", + "system": "", + "attachments": [ + { + "type": None, + "path": None, + "url": "https://example.com/attachment.png", + "id": ANY, + } + ], + "stream": True, + "previous": [], + } finally: plugins.pm.unregister(name="FragmentLoadersPlugin") @@ -383,7 +389,7 @@ def register_tools(self, register): assert '"output": "HI"' in result4.output # Now check in the database - tool_row = [row for row in logs_db["tools"].rows][0] + tool_row = next(iter(logs_db["tools"].rows)) assert tool_row["name"] == "upper" assert tool_row["plugin"] == "ToolsPlugin" @@ -452,19 +458,30 @@ def register_tools(self, register): runner.invoke(cli.cli, ["logs", "-c", "-n", "0", "--json"]).output ) results = tuple( - (log_row["prompt"], json.dumps(log_row["tool_results"])) + ( + log_row["prompt"], + re.sub( + r'"id": \d+', + '"id": ID', + re.sub( + r"tc_[0-9a-z]{26}", + "tc_TCID", + json.dumps(log_row["tool_results"]), + ), + ), + ) for log_row in log_rows ) assert results == ( ('{"tool_calls": [{"name": "upper", "arguments": {"text": "one"}}]}', "[]"), ( "", - '[{"id": 2, "tool_id": 1, "name": "upper", "output": "ONE", "tool_call_id": null, "exception": null, "attachments": []}]', + '[{"id": ID, "tool_id": 1, "name": "upper", "output": "ONE", "tool_call_id": "tc_TCID", "exception": null, "instance": null, "attachments": []}]', ), ('{"tool_calls": [{"name": "upper", "arguments": {"text": "two"}}]}', "[]"), ( "", - '[{"id": 3, "tool_id": 1, "name": "upper", "output": "TWO", "tool_call_id": null, "exception": null, "attachments": []}]', + '[{"id": ID, "tool_id": 1, "name": "upper", "output": "TWO", "tool_call_id": "tc_TCID", "exception": null, "instance": null, "attachments": []}]', ), ( '{"tool_calls": [{"name": "upper", "arguments": {"text": "three"}}]}', @@ -472,7 +489,7 @@ def register_tools(self, register): ), ( "", - '[{"id": 4, "tool_id": 1, "name": "upper", "output": "THREE", "tool_call_id": null, "exception": null, "attachments": []}]', + '[{"id": ID, "tool_id": 1, "name": "upper", "output": "THREE", "tool_call_id": "tc_TCID", "exception": null, "instance": null, "attachments": []}]', ), ) # Test the --td option @@ -639,6 +656,7 @@ def after_call(tool, tool_call, tool_result): "toolboxes": [ { "name": "Filesystem", + "dynamic": False, "tools": [ { "name": "Filesystem_list_files", @@ -649,6 +667,7 @@ def after_call(tool, tool_call, tool_result): }, { "name": "Memory", + "dynamic": False, "tools": [ { "name": "Memory_append", @@ -741,8 +760,8 @@ def after_call(tool, tool_call, tool_result): "[" + result3.output.split('"tool_results": [')[1].split("]")[0] + "]" ) assert tool_results == [ - {"name": "Memory_set", "output": "null", "tool_call_id": None}, - {"name": "Memory_get", "output": "two", "tool_call_id": None}, + {"name": "Memory_set", "output": "null", "tool_call_id": ANY}, + {"name": "Memory_get", "output": "two", "tool_call_id": ANY}, ] # Test the CLI running a configured toolbox prompt @@ -755,7 +774,7 @@ def after_call(tool, tool_call, tool_result): [ "prompt", "-T", - "Filesystem({})".format(json.dumps(str(my_dir2))), + f"Filesystem({json.dumps(str(my_dir2))})", json.dumps({"tool_calls": [{"name": "Filesystem_list_files"}]}), "-m", "echo", @@ -769,27 +788,37 @@ def after_call(tool, tool_call, tool_result): { "name": "Filesystem_list_files", "output": json.dumps([str(other_path)]), - "tool_call_id": None, + "tool_call_id": ANY, } ] - # Should show an error if you attempt to llm -c with configured toolboxes + # The stored instance configuration comes back as a -T style spec + conversation = cli.load_conversation(None) + assert conversation.loaded_tools == [ + "Filesystem({})".format(json.dumps({"path": str(my_dir2)})) + ] + + # llm -c should reconstruct the configured toolbox from the log result5 = runner.invoke( cli.cli, - ["-c", "list them again"], + ["-c", json.dumps({"tool_calls": [{"name": "Filesystem_list_files"}]})], ) - assert result5.exit_code == 1 - assert ( - "Error: Tool(s) Filesystem_list_files not found. Available tools:" - in result5.output + assert result5.exit_code == 0 + tool_results = json.loads( + "[" + + result5.output.rsplit('"tool_results": [', 1)[1].rsplit("]", 1)[0] + + "]" ) + assert tool_results == [ + { + "name": "Filesystem_list_files", + "output": json.dumps([str(other_path)]), + "tool_call_id": ANY, + } + ] # Test the logging worked - rows = list(logs_db.query(TOOL_RESULTS_SQL)) - # JSON decode things in rows - for row in rows: - row["tool_calls"] = json.loads(row["tool_calls"]) - row["tool_results"] = json.loads(row["tool_results"]) + rows = tool_activity_rows(logs_db) assert rows == [ { "model": "echo", @@ -846,12 +875,186 @@ def after_call(tool, tool_call, tool_result): } ], }, + # The llm -c continuation, using the reconstructed toolbox + { + "model": "echo", + "tool_calls": [{"name": "Filesystem_list_files", "arguments": "{}"}], + "tool_results": [], + }, + { + "model": "echo", + "tool_calls": [], + "tool_results": [ + { + "name": "Filesystem_list_files", + "output": json.dumps([str(other_path)]), + "instance": { + "name": "Filesystem", + "plugin": "ToolboxPlugin", + "arguments": json.dumps({"path": str(my_dir2)}), + }, + } + ], + }, ] finally: plugins.pm.unregister(name="ToolboxPlugin") +class Discovery(llm.Toolbox): + """ + Tools discovered at runtime from a configured source. + + Usage: + + Discovery("demo") + """ + + def __init__(self, source: str, prefix: str = ""): + self.source = source + self.prefix = prefix + + def tools(self): + def greet(name: str) -> str: + "Greet someone by name" + return f"hello {name} from {self.source}" + + yield llm.Tool.function(greet, name=self.prefix + "greet") + + +class Counter(llm.Toolbox): + """ + Registers a counting tool during prepare(). + """ + + def __init__(self, start: int = 0): + self.value = start + + def prepare(self): + def increment() -> int: + "Increment the counter" + self.value += 1 + return self.value + + self.add_tool(increment) + + +class DynamicToolboxPlugin: + __name__ = "DynamicToolboxPlugin" + + @hookimpl + def register_tools(self, register): + register(Discovery) + register(Counter) + + +def test_toolbox_constructor_signature_preserved(): + # Toolbox.__init_subclass__ wraps __init__ - that wrapper should not + # obscure the constructor signature + assert str(inspect.signature(Discovery)) == "(source: str, prefix: str = '')" + assert str(inspect.signature(Filesystem)) == "(path: str)" + + +def test_tools_list_dynamic_toolbox(): + # https://github.com/simonw/llm/issues/1580 + runner = CliRunner() + try: + plugins.pm.register(DynamicToolboxPlugin(), name="DynamicToolboxPlugin") + + # Plain listing shows constructor signature and docstring instead of + # a bare "Discovery:" header with nothing underneath it + result = runner.invoke(cli.cli, ["tools"]) + assert result.exit_code == 0 + assert "Discovery:" not in result.output + assert ( + "Discovery(source: str, prefix: str = '') (plugin: DynamicToolboxPlugin)\n" + "\n" + " Tools discovered at runtime from a configured source.\n" + "\n" + " Usage:\n" + "\n" + ' Discovery("demo")\n' + ) in result.output + + # --json marks the toolbox as dynamic + result2 = runner.invoke(cli.cli, ["tools", "--json"]) + assert result2.exit_code == 0 + toolboxes = { + toolbox["name"]: toolbox + for toolbox in json.loads(result2.output)["toolboxes"] + } + assert toolboxes["Discovery"]["dynamic"] is True + assert toolboxes["Discovery"]["tools"] == [] + + # Passing a constructor spec lists the tools the instance provides, + # with the spec itself as the heading + result3 = runner.invoke(cli.cli, ["tools", 'Discovery("demo")']) + assert result3.exit_code == 0 + assert result3.output == ( + 'Discovery("demo"):\n\n' + " greet(name: str) -> str\n\n" + " Greet someone by name\n\n" + ) + + # And --json with the spec includes the discovered tools + result4 = runner.invoke(cli.cli, ["tools", 'Discovery("demo")', "--json"]) + assert result4.exit_code == 0 + assert json.loads(result4.output)["toolboxes"] == [ + { + "name": "Discovery", + "dynamic": True, + "tools": [ + { + "name": "greet", + "description": "Greet someone by name", + "arguments": { + "properties": {"name": {"type": "string"}}, + "required": ["name"], + "type": "object", + }, + } + ], + } + ] + finally: + plugins.pm.unregister(name="DynamicToolboxPlugin") + + +def test_tools_list_prepare_toolbox(): + # Toolboxes that discover their tools in prepare() are dynamic too + runner = CliRunner() + try: + plugins.pm.register(DynamicToolboxPlugin(), name="DynamicToolboxPlugin") + + result = runner.invoke(cli.cli, ["tools"]) + assert result.exit_code == 0 + assert "Counter:" not in result.output + assert ( + "Counter(start: int = 0) (plugin: DynamicToolboxPlugin)\n" + "\n" + " Registers a counting tool during prepare().\n" + ) in result.output + + result2 = runner.invoke(cli.cli, ["tools", "--json"]) + assert result2.exit_code == 0 + toolboxes = { + toolbox["name"]: toolbox + for toolbox in json.loads(result2.output)["toolboxes"] + } + assert toolboxes["Counter"]["dynamic"] is True + assert toolboxes["Counter"]["tools"] == [] + + # A constructor spec runs prepare() and lists the registered tools + result3 = runner.invoke(cli.cli, ["tools", "Counter(5)"]) + assert result3.exit_code == 0 + assert result3.output == ( + "Counter(5):\n\n increment() -> int\n\n Increment the counter\n\n" + ) + finally: + plugins.pm.unregister(name="DynamicToolboxPlugin") + + def test_register_toolbox_fails_on_bad_class(): class BadTools: def bad(self): @@ -889,7 +1092,7 @@ def test_toolbox_logging_async(logs_db, tmpdir): "-T", "Memory", "--tool", - "Filesystem({})".format(json.dumps(str(path))), + f"Filesystem({json.dumps(str(path))})", json.dumps( { "tool_calls": [ @@ -911,19 +1114,15 @@ def test_toolbox_logging_async(logs_db, tmpdir): "[" + result.output.split('"tool_results": [')[1].rsplit("]", 1)[0] + "]" ) assert tool_results == [ - {"name": "Memory_set", "output": "null", "tool_call_id": None}, - {"name": "Memory_get", "output": "two", "tool_call_id": None}, - {"name": "Filesystem_list_files", "output": "[]", "tool_call_id": None}, + {"name": "Memory_set", "output": "null", "tool_call_id": ANY}, + {"name": "Memory_get", "output": "two", "tool_call_id": ANY}, + {"name": "Filesystem_list_files", "output": "[]", "tool_call_id": ANY}, ] finally: plugins.pm.unregister(name="ToolboxPlugin") # Check the database - rows = list(logs_db.query(TOOL_RESULTS_SQL)) - # JSON decode things in rows - for row in rows: - row["tool_calls"] = json.loads(row["tool_calls"]) - row["tool_results"] = json.loads(row["tool_results"]) + rows = tool_activity_rows(logs_db) assert rows == [ { "model": "echo", @@ -942,7 +1141,7 @@ def test_toolbox_logging_async(logs_db, tmpdir): "name": "Memory_set", "output": "null", "instance": { - "name": "Filesystem", + "name": "Memory", "plugin": "ToolboxPlugin", "arguments": "{}", }, @@ -951,7 +1150,7 @@ def test_toolbox_logging_async(logs_db, tmpdir): "name": "Memory_get", "output": "two", "instance": { - "name": "Filesystem", + "name": "Memory", "plugin": "ToolboxPlugin", "arguments": "{}", }, @@ -995,57 +1194,32 @@ def test_plugins_command(): ] -TOOL_RESULTS_SQL = """ --- First, create ordered subqueries for tool_calls and tool_results -with ordered_tool_calls as ( - select - tc.response_id, - json_group_array( - json_object( - 'name', tc.name, - 'arguments', tc.arguments - ) - ) as tool_calls_json - from ( - select * from tool_calls order by id - ) tc - where tc.id is not null - group by tc.response_id -), -ordered_tool_results as ( - select - tr.response_id, - json_group_array( - json_object( - 'name', tr.name, - 'output', tr.output, - 'instance', case - when ti.id is not null then json_object( - 'name', ti.name, - 'plugin', ti.plugin, - 'arguments', ti.arguments - ) - else null - end - ) - ) as tool_results_json - from ( - select distinct tr.*, ti.id as ti_id, ti.name as ti_name, - ti.plugin, ti.arguments as ti_arguments - from tool_results tr - left join tool_instances ti on tr.instance_id = ti.id - order by tr.id - ) tr - left join tool_instances ti on tr.instance_id = ti.id - where tr.id is not null - group by tr.response_id -) -select - r.model, - coalesce(otc.tool_calls_json, '[]') as tool_calls, - coalesce(otr.tool_results_json, '[]') as tool_results -from responses r -left join ordered_tool_calls otc on r.id = otc.response_id -left join ordered_tool_results otr on r.id = otr.response_id -group by r.id, r.model -order by r.id""" +def tool_activity_rows(db): + """Per-turn tool calls and results from the message store, in the + shape the old TOOL_RESULTS_SQL produced from the legacy tables.""" + from llm.logs import LogStore, log_row_extras, merged_log_rows + + store = LogStore(db) + rows = merged_log_rows(store) + rows.reverse() + out = [] + for row in rows: + extras = log_row_extras(store, row) + out.append( + { + "model": row["model"], + "tool_calls": [ + {"name": call["name"], "arguments": json.dumps(call["arguments"])} + for call in extras["tool_calls"] + ], + "tool_results": [ + { + "name": result["name"], + "output": result["output"], + "instance": result["instance"], + } + for result in extras["tool_results"] + ], + } + ) + return out diff --git a/tests/test_serialization.py b/tests/test_serialization.py new file mode 100644 index 000000000..afc5a4e6a --- /dev/null +++ b/tests/test_serialization.py @@ -0,0 +1,468 @@ +"""Tests for llm.serialization — the TypedDict spec for the JSON-safe +wire form of Message, Part, and Response. + +Uses pydantic.TypeAdapter to verify that actual to_dict() output +conforms to the TypedDict annotations. pydantic is already a runtime +dependency. +""" + +import json + +import pytest +from pydantic import TypeAdapter, ValidationError + +import llm +from llm.serialization import ( + AttachmentPartDict, + MessageDict, + PartDict, + ReasoningPartDict, + ResponseDict, + TextPartDict, + ToolCallPartDict, + ToolResultPartDict, +) + +# ---- required/optional keys ---------------------------------------- + + +class TestRequiredOptionalKeys: + def test_message_dict_required_keys(self): + assert MessageDict.__required_keys__ == {"role", "parts"} + assert MessageDict.__optional_keys__ == {"provider_metadata"} + + def test_text_part_dict_required_keys(self): + assert TextPartDict.__required_keys__ == {"type", "text"} + assert TextPartDict.__optional_keys__ == {"provider_metadata"} + + def test_reasoning_part_dict_required_keys(self): + assert ReasoningPartDict.__required_keys__ == {"type", "text"} + assert ReasoningPartDict.__optional_keys__ == { + "redacted", + "provider_metadata", + } + + def test_tool_call_part_dict_required_keys(self): + assert ToolCallPartDict.__required_keys__ == {"type", "name", "arguments"} + assert ToolCallPartDict.__optional_keys__ == { + "tool_call_id", + "server_executed", + "provider_metadata", + } + + def test_tool_result_part_dict_required_keys(self): + assert ToolResultPartDict.__required_keys__ == {"type", "name", "output"} + assert ToolResultPartDict.__optional_keys__ == { + "tool_call_id", + "server_executed", + "exception", + "attachments", + "provider_metadata", + } + + def test_attachment_part_dict_required_keys(self): + assert AttachmentPartDict.__required_keys__ == {"type"} + assert AttachmentPartDict.__optional_keys__ == { + "attachment", + "provider_metadata", + } + + def test_response_dict_required_keys(self): + assert ResponseDict.__required_keys__ == {"model", "prompt", "messages"} + assert ResponseDict.__optional_keys__ == {"id", "usage", "datetime_utc"} + + +# ---- to_dict output conforms to the TypedDict ---------------------- + + +class TestPartRoundTrip: + def _adapter(self, td): + return TypeAdapter(td) + + def test_text_part_matches(self): + d = llm.parts.TextPart(text="hello").to_dict() + self._adapter(TextPartDict).validate_python(d) + + def test_text_part_with_provider_metadata_matches(self): + d = llm.parts.TextPart( + text="hi", provider_metadata={"anthropic": {"cached": True}} + ).to_dict() + self._adapter(TextPartDict).validate_python(d) + + def test_reasoning_part_redacted_matches(self): + d = llm.parts.ReasoningPart(text="", redacted=True).to_dict() + self._adapter(ReasoningPartDict).validate_python(d) + + def test_reasoning_part_with_signature_matches(self): + d = llm.parts.ReasoningPart( + text="thinking...", + provider_metadata={"anthropic": {"signature": "sig-abc"}}, + ).to_dict() + self._adapter(ReasoningPartDict).validate_python(d) + + def test_tool_call_part_matches(self): + d = llm.parts.ToolCallPart( + name="search", arguments={"q": "x"}, tool_call_id="c1" + ).to_dict() + self._adapter(ToolCallPartDict).validate_python(d) + + def test_tool_result_part_matches(self): + d = llm.parts.ToolResultPart( + name="search", output="result", tool_call_id="c1" + ).to_dict() + self._adapter(ToolResultPartDict).validate_python(d) + + def test_attachment_part_with_url_matches(self): + att = llm.Attachment(type="image/jpeg", url="https://example.com/cat.jpg") + d = llm.parts.AttachmentPart(attachment=att).to_dict() + self._adapter(AttachmentPartDict).validate_python(d) + + def test_attachment_part_with_bytes_matches(self): + att = llm.Attachment(type="image/png", content=b"\x89PNG...") + d = llm.parts.AttachmentPart(attachment=att).to_dict() + self._adapter(AttachmentPartDict).validate_python(d) + + +class TestPartDiscriminatedUnion: + def test_text_part_validates_as_part_dict(self): + d = llm.parts.TextPart(text="hi").to_dict() + TypeAdapter(PartDict).validate_python(d) + + def test_reasoning_part_validates_as_part_dict(self): + d = llm.parts.ReasoningPart(text="thinking").to_dict() + TypeAdapter(PartDict).validate_python(d) + + def test_tool_call_part_validates_as_part_dict(self): + d = llm.parts.ToolCallPart(name="t", arguments={}, tool_call_id="c1").to_dict() + TypeAdapter(PartDict).validate_python(d) + + def test_tool_result_part_validates_as_part_dict(self): + d = llm.parts.ToolResultPart( + name="t", output="out", tool_call_id="c1" + ).to_dict() + TypeAdapter(PartDict).validate_python(d) + + def test_attachment_part_validates_as_part_dict(self): + att = llm.Attachment(type="image/jpeg", url="http://x") + d = llm.parts.AttachmentPart(attachment=att).to_dict() + TypeAdapter(PartDict).validate_python(d) + + def test_unknown_type_rejected(self): + with pytest.raises(ValidationError): + TypeAdapter(PartDict).validate_python({"type": "nonsense", "text": "x"}) + + +class TestMessageDictRoundTrip: + def test_user_message_matches(self): + d = llm.user("hi").to_dict() + TypeAdapter(MessageDict).validate_python(d) + + def test_assistant_with_mixed_parts_matches(self): + m = llm.Message( + role="assistant", + parts=[ + llm.parts.ReasoningPart( + text="thinking", + provider_metadata={"anthropic": {"signature": "s"}}, + ), + llm.parts.TextPart(text="answer"), + llm.parts.ToolCallPart( + name="search", + arguments={"q": "x"}, + tool_call_id="c1", + ), + ], + ) + TypeAdapter(MessageDict).validate_python(m.to_dict()) + + def test_tool_role_message_with_results_matches(self): + m = llm.tool_message( + llm.parts.ToolResultPart(name="s", output="r", tool_call_id="c1"), + ) + TypeAdapter(MessageDict).validate_python(m.to_dict()) + + +class TestResponseDictRoundTrip: + def test_mock_response_to_dict_matches(self, mock_model): + mock_model.enqueue(["answer"]) + r = mock_model.prompt("q") + r.text() + + d = r.to_dict() + TypeAdapter(ResponseDict).validate_python(d) + + def test_to_dict_forces_unconsumed_response(self, mock_model): + mock_model.enqueue(["answer"]) + r = mock_model.prompt("q") + # No text()/messages access first - to_dict() must execute the prompt + d = r.to_dict() + TypeAdapter(ResponseDict).validate_python(d) + assert d["messages"] == [ + {"role": "assistant", "parts": [{"type": "text", "text": "answer"}]} + ] + + def test_from_dict_restores_pending_tool_calls(self, mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="tool_call_name", + chunk="search", + part_index=0, + tool_call_id="c1", + ), + llm.parts.StreamEvent( + type="tool_call_args", + chunk='{"q": "weather"}', + part_index=0, + tool_call_id="c1", + ), + ] + ) + r = mock_model.prompt("find weather") + r2 = llm.Response.from_dict(r.to_dict(), model=mock_model) + assert r2.tool_calls() == [ + llm.ToolCall(name="search", arguments={"q": "weather"}, tool_call_id="c1") + ] + + def search(q: str) -> str: + return f"results for {q}" + + mock_model.enqueue(["it is sunny"]) + r3 = r2.reply(tools=[search]) + r3.text() + tool_message = next(m for m in r3.prompt.messages if m.role == "tool") + result_part = tool_message.parts[0] + assert result_part.output == "results for weather" + assert result_part.exception is None + + def test_from_dict_excludes_server_executed_tool_calls(self, mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="tool_call_name", + chunk="web_search", + part_index=0, + tool_call_id="s1", + server_executed=True, + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + r = mock_model.prompt("q") + r2 = llm.Response.from_dict(r.to_dict(), model=mock_model) + assert r2.tool_calls() == [] + + def test_response_with_reasoning_matches(self, mock_model): + mock_model.enqueue( + [ + llm.parts.StreamEvent( + type="reasoning", + chunk="thinking", + part_index=0, + provider_metadata={"anthropic": {"signature": "s"}}, + ), + llm.parts.StreamEvent(type="text", chunk="answer", part_index=1), + ] + ) + r = mock_model.prompt("q") + r.text() + + d = r.to_dict() + TypeAdapter(ResponseDict).validate_python(d) + + def test_response_with_options_matches(self, mock_model): + mock_model.enqueue(["ok"]) + r = mock_model.prompt("q", max_tokens=42) + r.text() + + d = r.to_dict() + TypeAdapter(ResponseDict).validate_python(d) + assert d["prompt"].get("options") == {"max_tokens": 42} + + +# ---- Literal discriminators ---------------------------------------- + + +class TestLiteralDiscriminators: + """The `type` field on each PartDict is a Literal — that's how + Pydantic's discriminated unions work. Verify each literal.""" + + def test_text_part_literal_is_text(self): + import typing + + hints = typing.get_type_hints(TextPartDict) + # Literal["text"] — check the args + assert typing.get_args(hints["type"]) == ("text",) + + def test_reasoning_part_literal_is_reasoning(self): + import typing + + hints = typing.get_type_hints(ReasoningPartDict) + assert typing.get_args(hints["type"]) == ("reasoning",) + + def test_tool_call_part_literal_is_tool_call(self): + import typing + + hints = typing.get_type_hints(ToolCallPartDict) + assert typing.get_args(hints["type"]) == ("tool_call",) + + def test_tool_result_part_literal_is_tool_result(self): + import typing + + hints = typing.get_type_hints(ToolResultPartDict) + assert typing.get_args(hints["type"]) == ("tool_result",) + + def test_attachment_part_literal_is_attachment(self): + import typing + + hints = typing.get_type_hints(AttachmentPartDict) + assert typing.get_args(hints["type"]) == ("attachment",) + + +# ---- to_dict / from_dict return-type annotations ------------------- + + +class TestAnnotations: + """Method signatures should advertise the specific TypedDicts.""" + + def test_text_part_to_dict_annotation(self): + import typing + + hints = typing.get_type_hints(llm.parts.TextPart.to_dict) + assert hints["return"] is TextPartDict + + def test_reasoning_part_to_dict_annotation(self): + import typing + + hints = typing.get_type_hints(llm.parts.ReasoningPart.to_dict) + assert hints["return"] is ReasoningPartDict + + def test_tool_call_part_to_dict_annotation(self): + import typing + + hints = typing.get_type_hints(llm.parts.ToolCallPart.to_dict) + assert hints["return"] is ToolCallPartDict + + def test_tool_result_part_to_dict_annotation(self): + import typing + + hints = typing.get_type_hints(llm.parts.ToolResultPart.to_dict) + assert hints["return"] is ToolResultPartDict + + def test_attachment_part_to_dict_annotation(self): + import typing + + hints = typing.get_type_hints(llm.parts.AttachmentPart.to_dict) + assert hints["return"] is AttachmentPartDict + + def test_message_to_dict_annotation(self): + import typing + + hints = typing.get_type_hints(llm.Message.to_dict) + assert hints["return"] is MessageDict + + def test_message_from_dict_annotation(self): + import typing + + hints = typing.get_type_hints(llm.Message.from_dict) + assert hints["d"] is MessageDict + + def test_response_to_dict_annotation(self): + import typing + + hints = typing.get_type_hints(llm.Response.to_dict) + assert hints["return"] is ResponseDict + + +# ---- End-to-end JSON round-trip validates against schema ----------- + + +class TestEndToEnd: + def test_json_roundtrip_validates(self, mock_model): + mock_model.enqueue(["text answer"]) + r = mock_model.prompt("q") + r.text() + + payload = json.dumps(r.to_dict()) + parsed = json.loads(payload) + # Parsed dict should still conform to ResponseDict. + TypeAdapter(ResponseDict).validate_python(parsed) + + +# ---- to_dict() must not emit keys absent from the TypedDict -------- +# +# pydantic's TypeAdapter on a TypedDict silently drops keys that aren't +# declared, so the round-trip tests above will not catch the case where +# .to_dict() starts emitting a brand-new key that nobody added to the +# TypedDict. These tests close that gap by asserting the set of keys +# .to_dict() returns is a subset of the union of required + optional +# keys declared on the corresponding TypedDict. + + +def _allowed(td): + return td.__required_keys__ | td.__optional_keys__ + + +class TestNoUndeclaredKeys: + def test_text_part_keys(self): + d = llm.parts.TextPart( + text="hi", + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(TextPartDict) + + def test_reasoning_part_keys(self): + d = llm.parts.ReasoningPart( + text="t", + redacted=True, + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(ReasoningPartDict) + + def test_tool_call_part_keys(self): + d = llm.parts.ToolCallPart( + name="t", + arguments={"q": "x"}, + tool_call_id="c1", + server_executed=True, + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(ToolCallPartDict) + + def test_tool_result_part_keys(self): + d = llm.parts.ToolResultPart( + name="t", + output="r", + tool_call_id="c1", + server_executed=True, + exception="boom", + attachments=[llm.Attachment(type="image/png", url="http://x/y.png")], + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(ToolResultPartDict) + + def test_attachment_part_keys(self): + d = llm.parts.AttachmentPart( + attachment=llm.Attachment(type="image/png", url="http://x/y.png"), + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(AttachmentPartDict) + + def test_message_keys(self): + d = llm.Message( + role="assistant", + parts=[llm.parts.TextPart(text="hi")], + provider_metadata={"k": "v"}, + ).to_dict() + assert set(d.keys()) <= _allowed(MessageDict) + + def test_response_keys(self, mock_model): + mock_model.enqueue(["answer"]) + r = mock_model.prompt("q", max_tokens=10) + r.text() + d = r.to_dict() + assert set(d.keys()) <= _allowed(ResponseDict) + # And the nested prompt sub-dict must conform too. + from llm.serialization import PromptDict + + assert set(d["prompt"].keys()) <= _allowed(PromptDict) diff --git a/tests/test_templates.py b/tests/test_templates.py index dcdabd9bc..9a01b7c5e 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -1,15 +1,17 @@ -from click.testing import CliRunner -from importlib.metadata import version import json -from llm import Template, Toolbox, hookimpl, user_dir -from llm.cli import cli -from llm.plugins import pm import os -from unittest import mock import pathlib -import pytest import textwrap +from importlib.metadata import version +from unittest import mock + +import pytest import yaml +from click.testing import CliRunner + +from llm import Template, Toolbox, hookimpl, user_dir +from llm.cli import cli +from llm.plugins import pm @pytest.mark.parametrize( @@ -199,7 +201,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "'Summarize this: $input'", "Input text", - [], + ["-m", "gpt-4o-mini"], "gpt-4o-mini", "Summarize this: Input text", None, @@ -227,7 +229,7 @@ def test_templates_error_on_missing_schema(templates_path): pytest.param( "boo", "Input text", - ["-s", "custom system"], + ["-m", "gpt-4o-mini", "-s", "custom system"], "gpt-4o-mini", [ {"role": "system", "content": "custom system"}, @@ -251,7 +253,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "prompt: 'Say $hello'", "Input text", - ["-p", "hello", "Blah"], + ["-m", "gpt-4o-mini", "-p", "hello", "Blah"], "gpt-4o-mini", "Say Blah\nInput text", None, @@ -260,7 +262,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "prompt: 'Say pelican'", "", - [], + ["-m", "gpt-4o-mini"], "gpt-4o-mini", "Say pelican", None, @@ -270,7 +272,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "system: 'Summarize this'", "Input text", - [], + ["-m", "gpt-4o-mini"], "gpt-4o-mini", [ {"content": "Summarize this", "role": "system"}, @@ -283,7 +285,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "prompt: 'Summarize this: $input'\noptions:\n temperature: 0.5", "Input text", - [], + ["-m", "gpt-4o-mini"], "gpt-4o-mini", "Summarize this: Input text", None, @@ -293,7 +295,7 @@ def test_templates_error_on_missing_schema(templates_path): ( "prompt: 'Summarize this: $input'\noptions:\n temperature: 0.5", "Input text", - ["-o", "temperature", "0.7"], + ["-m", "gpt-4o-mini", "-o", "temperature", "0.7"], "gpt-4o-mini", "Summarize this: Input text", None, @@ -413,6 +415,21 @@ def test_execute_prompt_from_template_path(): } +def test_template_respects_cli_extract_flag( + mocked_openai_chat_returning_fenced_code, templates_path +): + (templates_path / "code.yaml").write_text("prompt: Write code", "utf-8") + runner = CliRunner() + result = runner.invoke( + cli, + ["-t", "code", "-m", "gpt-4o-mini", "--key", "x", "-x"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert "```" not in result.output + assert result.output.strip() == "function foo() {\n return 'bar';\n}" + + FUNCTIONS_EXAMPLE = """ def greet(name: str) -> str: return f"Hello, {name}!" @@ -449,8 +466,7 @@ def register_tools(self, register): def test_tools_in_templates( source, expected_tool_success, expected_functions_success, httpx_mock, tmpdir ): - template_yaml = textwrap.dedent( - """ + template_yaml = textwrap.dedent(""" name: test tools: - llm_version @@ -458,8 +474,7 @@ def test_tools_in_templates( functions: | def demo(): return "Demo" - """ - ) + """) args = [] def before(): diff --git a/tests/test_tools.py b/tests/test_tools.py index c61154f22..97dc3d22b 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,20 +1,216 @@ import asyncio -from click.testing import CliRunner -from importlib.metadata import version import json -import llm -from llm import cli, CancelToolCall -from llm.migrations import migrate -from llm.tools import llm_time import os +import re +import time +from importlib.metadata import version + import pytest import sqlite_utils -import time +from click.testing import CliRunner +import llm +from llm import CancelToolCall, cli +from llm.logs import LogStore +from llm.migrations import migrate +from llm.tools import llm_time API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" +class DemoServerSideTool(llm.ServerSideTool): + "A server-side tool used by the core tests." + + name = "demo_server_tool" + + def __init__(self, value="demo"): + self.value = value + self.prepare_calls = 0 + + def tool_spec(self, model): + return {"type": self.name, "value": self.value} + + def prepare_request(self, model, kwargs): + self.prepare_calls += 1 + + +class ServerToolsOnlyModel(llm.Model): + model_id = "server-tools-only" + + @property + def supported_server_side_tools(self): + return (DemoServerSideTool,) + + def execute(self, prompt, stream, response, conversation): + yield "done" + + +class AsyncServerToolsOnlyModel(llm.AsyncModel): + model_id = "async-server-tools-only" + + @property + def supported_server_side_tools(self): + return (DemoServerSideTool,) + + async def execute(self, prompt, stream, response, conversation): + yield "done" + + +class MixedToolsModel(ServerToolsOnlyModel): + model_id = "mixed-tools" + supports_tools = True + + +class RawServerToolOnlyModel(ServerToolsOnlyModel): + model_id = "raw-server-tool-only" + + @property + def supported_server_side_tools(self): + return (llm.ServerSideTool,) + + +def test_server_side_tool_raw_spec_escape_hatch(): + tool = llm.ServerSideTool({"type": "browser_search"}) + model = ServerToolsOnlyModel() + + assert tool.tool_spec(model) == {"type": "browser_search"} + assert tool.name == "server_side_tool" + assert tool._config == {"spec": {"type": "browser_search"}} + + kwargs = {} + assert tool.prepare_request(model, kwargs) is None + assert kwargs == {} + + with pytest.raises(TypeError, match="raw provider tool spec"): + llm.ServerSideTool().tool_spec(model) + + +def test_declared_server_side_tool_does_not_require_function_tool_support(): + tool = DemoServerSideTool("one") + model = ServerToolsOnlyModel() + response = model.prompt("hello", tools=[tool]) + + assert response.text() == "done" + assert response.prompt.tools == [tool] + # Core transports and validates server-side tools but never invokes + # their provider request hook itself. + assert tool.prepare_calls == 0 + assert tool._config == {"value": "one"} + + +def test_unsupported_server_side_tool_fails_before_execution(mock_model): + with pytest.raises( + ValueError, + match=( + "Model 'mock' does not support server-side tool " + "'demo_server_tool'. Run: llm tools -m mock" + ), + ): + mock_model.prompt("hello", tools=[DemoServerSideTool()]) + + +def test_declaring_raw_escape_hatch_does_not_claim_every_subclass(): + model = RawServerToolOnlyModel() + assert ( + model.prompt("hello", tools=[llm.ServerSideTool({"type": "custom"})]).text() + == "done" + ) + with pytest.raises(ValueError, match="does not support server-side tool"): + model.prompt("hello", tools=[DemoServerSideTool()]) + + +def test_server_side_tool_support_can_vary_by_model_instance(): + class ConditionalModel(ServerToolsOnlyModel): + def __init__(self, enabled): + self.enabled = enabled + + @property + def supported_server_side_tools(self): + return (DemoServerSideTool,) if self.enabled else () + + assert ( + ConditionalModel(True).prompt("hello", tools=[DemoServerSideTool()]).text() + == "done" + ) + with pytest.raises(ValueError, match="does not support server-side tool"): + ConditionalModel(False).prompt("hello", tools=[DemoServerSideTool()]) + + +def test_server_side_tool_configuration_is_logged(): + db = sqlite_utils.Database(memory=True) + migrate(db) + response = ServerToolsOnlyModel().prompt( + "hello", tools=[DemoServerSideTool("configured")] + ) + response.text() + response.log_to_db(db) + + instance = next(iter(db["tool_instances"].rows)) + assert instance["name"] == "DemoServerSideTool" + assert instance["plugin"] is None + assert json.loads(instance["arguments"]) == {"value": "configured"} + assert next(iter(db["turn_tools"].rows))["instance_id"] == instance["id"] + + +def test_logs_expanded_server_side_tool(user_path): + db = sqlite_utils.Database(str(user_path / "logs.db")) + migrate(db) + response = ServerToolsOnlyModel().prompt( + "hello", tools=[DemoServerSideTool("configured")] + ) + response.text() + response.log_to_db(db) + + result = CliRunner().invoke(cli.cli, ["logs", "-cue"], catch_exceptions=False) + + assert result.exit_code == 0 + assert '- `DemoServerSideTool({"value": "configured"})`:' in result.output + assert "Arguments: `{}`" in result.output + + +def test_function_tools_still_require_supports_tools(): + def local_tool(): + return "local" + + with pytest.raises(ValueError, match="does not support tools"): + ServerToolsOnlyModel().prompt("hello", tools=[local_tool]) + + +def test_local_executor_ignores_server_side_tools(): + def local_tool(): + return "local" + + tool = DemoServerSideTool() + response = MixedToolsModel().prompt( + "hello", tools=[tool, llm.Tool.function(local_tool)] + ) + response.text() + + results = response.execute_tool_calls( + tool_calls_list=[ + llm.ToolCall(name="local_tool", arguments={}), + llm.ToolCall(name=tool.name, arguments={}), + ] + ) + assert [result.output for result in results] == [ + "local", + 'Error: tool "demo_server_tool" does not exist', + ] + + +@pytest.mark.asyncio +async def test_async_declared_server_side_tool_and_executor_partition(): + tool = DemoServerSideTool() + model = AsyncServerToolsOnlyModel() + response = model.prompt("hello", tools=[tool]) + + assert await response.text() == "done" + results = await response.execute_tool_calls( + tool_calls_list=[llm.ToolCall(name=tool.name, arguments={})] + ) + assert results[0].output == 'Error: tool "demo_server_tool" does not exist' + + @pytest.mark.vcr def test_tool_use_basic(vcr): model = llm.get_model("gpt-4o-mini") @@ -42,13 +238,10 @@ def multiply(a: int, b: int) -> int: db = sqlite_utils.Database(memory=True) migrate(db) chain_response.log_to_db(db) - assert set(db.table_names()).issuperset( - {"tools", "tool_responses", "tool_calls", "tool_results"} - ) - responses = list(db["responses"].rows) - assert len(responses) == 2 - first_response, second_response = responses + turns = list(db["turns"].rows) + assert len(turns) == 2 + first_turn, second_turn = turns tools = list(db["tools"].rows) assert len(tools) == 1 @@ -56,18 +249,30 @@ def multiply(a: int, b: int) -> int: assert tools[0]["description"] == "Multiply two numbers." assert tools[0]["plugin"] is None - tool_results = list(db["tool_results"].rows) - tool_calls = list(db["tool_calls"].rows) - + # The tool call is in the first turn's output parts; the result is + # among the second turn's inputs. + store = LogStore(db) + first_chain = store.load_chain(first_turn["tip_message_hash"]) + tool_calls = [ + part + for message in first_chain + for part in message.parts + if isinstance(part, llm.parts.ToolCallPart) + ] assert len(tool_calls) == 1 - assert tool_calls[0]["response_id"] == first_response["id"] - assert tool_calls[0]["name"] == "multiply" - assert tool_calls[0]["arguments"] == '{"a": 1231, "b": 2331}' - - assert len(tool_results) == 1 - assert tool_results[0]["response_id"] == second_response["id"] - assert tool_results[0]["output"] == "2869461" - assert tool_results[0]["tool_call_id"] == tool_calls[0]["tool_call_id"] + assert tool_calls[0].name == "multiply" + assert tool_calls[0].arguments == {"a": 1231, "b": 2331} + + second_inputs = store.load_chain(second_turn["parent_message_hash"]) + tool_results_parts = [ + part + for message in second_inputs + for part in message.parts + if isinstance(part, llm.parts.ToolResultPart) + ] + assert len(tool_results_parts) == 1 + assert tool_results_parts[0].output == "2869461" + assert tool_results_parts[0].tool_call_id == tool_calls[0].tool_call_id @pytest.mark.vcr @@ -102,6 +307,31 @@ def can_have_dragons(population: int) -> bool: assert third.tool_calls() == [] +def test_chain_round_separator_is_display_only(): + """The space between chain rounds is synthesized at the chain level + for display - it must never become a stored whitespace part.""" + + def hello(): + return "world" + + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "hello"}]}), tools=[hello] + ) + events = list(chain_response.stream_events()) + text = "".join(e.chunk for e in events if e.type == "text") + # The separator reached the streamed output... + assert "\n} {\n" in text + + db = sqlite_utils.Database(memory=True) + migrate(db) + chain_response.log_to_db(db) + # ...but no whitespace-only text part was stored. + for row in db["parts"].rows: + if row["type"] == "text" and row["text"] is not None: + assert row["text"].strip(), row + + def test_tool_use_async_tool_function(): async def hello(): return "world" @@ -111,10 +341,13 @@ async def hello(): json.dumps({"tool_calls": [{"name": "hello"}]}), tools=[hello] ) output = chain_response.text() - # That's two JSON objects separated by '\n}{\n' - bits = output.split("\n}{\n") + # Two JSON objects, separated by the chain's round boundary space + bits = output.split("\n} {\n") assert len(bits) == 2 objects = [json.loads(bits[0] + "}"), json.loads("{" + bits[1])] + tool_call_id = objects[1]["tool_results"][0]["tool_call_id"] + assert tool_call_id.startswith("tc_") + objects[1]["tool_results"][0]["tool_call_id"] = None assert objects == [ {"prompt": "", "system": "", "attachments": [], "stream": True, "previous": []}, { @@ -152,10 +385,15 @@ async def hello2(): tools=[hello, hello2], ) output = await chain_response.text() - # That's two JSON objects separated by '\n}{\n' - bits = output.split("\n}{\n") + # Two JSON objects, separated by the chain's round boundary space + bits = output.split("\n} {\n") assert len(bits) == 2 objects = [json.loads(bits[0] + "}"), json.loads("{" + bits[1])] + ids = [r["tool_call_id"] for r in objects[1]["tool_results"]] + assert all(i.startswith("tc_") for i in ids) + assert len(set(ids)) == 2 + for r in objects[1]["tool_results"]: + r["tool_call_id"] = None assert objects == [ {"prompt": "", "system": "", "attachments": [], "stream": True, "previous": []}, { @@ -473,6 +711,102 @@ async def after(*args): assert len(after_collected) == 2 +def test_provider_managed_tool_execution_uses_response_context(): + events = [] + implementation_call = {} + + def lookup(value: str, llm_tool_call: llm.ToolCall) -> dict: + implementation_call["tool_call"] = llm_tool_call + return {"value": value.upper()} + + def before(tool, tool_call): + events.append(("before", tool.name, tool_call.tool_call_id)) + + def after(tool, tool_call, tool_result): + events.append( + ( + "after", + tool.name, + tool_call.tool_call_id, + tool_result.output, + ) + ) + + class ProviderManagedToolModel(llm.Model): + model_id = "provider-managed-tool" + supports_tools = True + + def execute(self, prompt, stream, response, conversation=None): + tool_result = response.execute_tool_call( + llm.ToolCall(name="lookup", arguments={"value": prompt.prompt}) + ) + yield tool_result.output + + chain = ProviderManagedToolModel().chain( + "pelican", + tools=[lookup], + before_call=before, + after_call=after, + ) + + assert chain.text() == '{"value": "PELICAN"}' + assert len(chain._responses) == 1 + tool_call = implementation_call["tool_call"] + assert tool_call.tool_call_id.startswith("tc_") + assert events == [ + ("before", "lookup", tool_call.tool_call_id), + ("after", "lookup", tool_call.tool_call_id, '{"value": "PELICAN"}'), + ] + + +@pytest.mark.asyncio +async def test_async_provider_managed_tool_execution_uses_response_context(): + events = [] + + async def lookup(value: str) -> str: + await asyncio.sleep(0) + return value.upper() + + async def before(tool, tool_call): + events.append(("before", tool.name, tool_call.tool_call_id)) + + async def after(tool, tool_call, tool_result): + events.append( + ( + "after", + tool.name, + tool_call.tool_call_id, + tool_result.output, + ) + ) + + class AsyncProviderManagedToolModel(llm.AsyncModel): + model_id = "async-provider-managed-tool" + supports_tools = True + + async def execute(self, prompt, stream, response, conversation=None): + tool_result = await response.execute_tool_call( + llm.ToolCall(name="lookup", arguments={"value": prompt.prompt}) + ) + yield tool_result.output + + chain = AsyncProviderManagedToolModel().chain( + "puffin", + tools=[lookup], + before_call=before, + after_call=after, + ) + + assert await chain.text() == "PUFFIN" + assert len(chain._responses) == 1 + assert events[0][0:2] == ("before", "lookup") + assert events[0][2].startswith("tc_") + assert events == [ + ("before", "lookup", events[0][2]), + ("after", "lookup", events[0][2], "PUFFIN"), + ] + + ERROR_FUNCTION = """ def trigger_error(msg: str): raise Exception(msg) @@ -513,11 +847,14 @@ def test_tool_errors(async_): # llm logs -c output log_text_result = runner.invoke(cli.cli, ["logs", "-c"]) assert log_text_result.exit_code == 0 + normalized_log_text = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", log_text_result.output) assert ( - "- **trigger_error**: `None`
\n" - " Error: Error!
\n" + "- **trigger_error**: `tc_TCID` \n" + " ```\n" + " Error: Error!\n" + " ``` \n" " **Error**: Exception: Error!\n" - ) in log_text_result.output + ) in normalized_log_text def test_chain_sync_cancel_only_first_of_two(): @@ -533,7 +870,6 @@ def before(tool, tool_call): if tool.name == "t1": raise CancelToolCall("skip1") # allow t2 - return None calls = [ {"name": "t1"}, @@ -572,7 +908,6 @@ async def t2() -> str: async def before(tool, tool_call): if tool.name == "t1": raise CancelToolCall("skip1") - return None calls = [ {"name": "t1"}, @@ -593,3 +928,297 @@ async def before(tool, tool_call): assert results[1].name == "t2" assert results[1].output == "ran2" assert results[1].exception is None + + +def test_tool_function_receives_llm_tool_call(): + captured = {} + + def lookup(name: str, llm_tool_call) -> str: + "Look up a name" + captured["tool_call"] = llm_tool_call + return "result for " + name + + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps( + {"tool_calls": [{"name": "lookup", "arguments": {"name": "simon"}}]} + ), + tools=[lookup], + ) + chain_response.text() + + tool_call = captured["tool_call"] + assert isinstance(tool_call, llm.ToolCall) + assert tool_call.name == "lookup" + assert tool_call.arguments == {"name": "simon"} + second = chain_response._responses[1] + assert second.prompt.tool_results[0].output == "result for simon" + + +def test_async_tool_function_receives_llm_tool_call_with_sync_model(): + captured = {} + + async def lookup(name: str, llm_tool_call: llm.ToolCall) -> str: + "Look up a name" + captured["tool_call"] = llm_tool_call + return "result for " + name + + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps( + {"tool_calls": [{"name": "lookup", "arguments": {"name": "simon"}}]} + ), + tools=[lookup], + ) + chain_response.text() + + tool_call = captured["tool_call"] + assert isinstance(tool_call, llm.ToolCall) + assert tool_call.name == "lookup" + assert tool_call.arguments == {"name": "simon"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_tool", (False, True)) +async def test_tool_function_receives_llm_tool_call_async_model(async_tool): + captured = {} + + def lookup(name: str, llm_tool_call) -> str: + "Look up a name" + captured["tool_call"] = llm_tool_call + return "result for " + name + + async def async_lookup(name: str, llm_tool_call) -> str: + "Look up a name" + captured["tool_call"] = llm_tool_call + return "result for " + name + + fn = async_lookup if async_tool else lookup + model = llm.get_async_model("echo") + chain_response = model.chain( + json.dumps( + {"tool_calls": [{"name": fn.__name__, "arguments": {"name": "simon"}}]} + ), + tools=[fn], + ) + output = await chain_response.text() + assert '"output": "result for simon"' in output + + tool_call = captured["tool_call"] + assert isinstance(tool_call, llm.ToolCall) + assert tool_call.name == fn.__name__ + assert tool_call.arguments == {"name": "simon"} + + +def test_llm_tool_call_excluded_from_input_schema(): + def lookup(name: str, llm_tool_call) -> str: + "Look up a name" + return name + + tool = llm.Tool.function(lookup) + assert "llm_tool_call" not in tool.input_schema.get("properties", {}) + assert "llm_tool_call" not in tool.input_schema.get("required", []) + assert "name" in tool.input_schema["properties"] + + +def test_kwargs_only_function_does_not_receive_llm_tool_call(): + # A tool that accepts **kwargs but does not name llm_tool_call + # explicitly should NOT have it injected. + captured = {} + + async def impl(**kwargs): + captured.update(kwargs) + return "ok" + + tool = llm.Tool( + name="t", + description="A tool", + input_schema={"type": "object", "properties": {"name": {"type": "string"}}}, + implementation=impl, + ) + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "t", "arguments": {"name": "x"}}]}), + tools=[tool], + ) + chain_response.text() + assert captured == {"name": "x"} + + +def test_toolbox_method_receives_llm_tool_call(): + captured = {} + + class Tools(llm.Toolbox): + def lookup(self, name: str, llm_tool_call) -> str: + captured["tool_call"] = llm_tool_call + return "hi " + name + + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps( + {"tool_calls": [{"name": "Tools_lookup", "arguments": {"name": "simon"}}]} + ), + tools=[Tools()], + ) + output = chain_response.text() + assert '"output": "hi simon"' in output + + tool_call = captured["tool_call"] + assert isinstance(tool_call, llm.ToolCall) + assert tool_call.arguments == {"name": "simon"} + + +def test_add_tool_call_synthesizes_missing_tool_call_id(): + model = llm.get_model("echo") + response = model.prompt("hello") + response.add_tool_call(llm.ToolCall(name="a", arguments={})) + response.add_tool_call(llm.ToolCall(name="b", arguments={}, tool_call_id="given")) + response.add_tool_call(llm.ToolCall(name="c", arguments={})) + ids = [tc.tool_call_id for tc in response._tool_calls] + assert ids[0] is not None and ids[0].startswith("tc_") + assert ids[1] == "given" + assert ids[2] is not None and ids[2].startswith("tc_") + assert ids[0] != ids[2] + + +def test_tool_call_ids_guaranteed_through_chain(): + seen_before_call = [] + captured = {} + + def first(llm_tool_call) -> str: + captured["first_id"] = llm_tool_call.tool_call_id + return "one" + + def second() -> str: + return "two" + + def before(tool, tool_call): + seen_before_call.append(tool_call.tool_call_id) + + model = llm.get_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "first"}, {"name": "second"}]}), + tools=[first, second], + before_call=before, + ) + chain_response.text() + + assert len(seen_before_call) == 2 + assert all(i is not None and i.startswith("tc_") for i in seen_before_call) + assert seen_before_call[0] != seen_before_call[1] + # The implementation saw the same id via llm_tool_call + assert captured["first_id"] == seen_before_call[0] + + # ToolResults and the next prompt's tool message carry the same ids + second_response = chain_response._responses[1] + result_ids = [r.tool_call_id for r in second_response.prompt.tool_results] + assert result_ids == seen_before_call + + # The assistant message parts carry the synthesized ids too, so a + # persisted-and-replayed history stays correlated + from llm.parts import ToolCallPart + + first_response = chain_response._responses[0] + part_ids = [ + p.tool_call_id + for p in first_response._messages_now()[0].parts + if isinstance(p, ToolCallPart) + ] + assert part_ids == seen_before_call + + +@pytest.mark.asyncio +async def test_tool_call_ids_guaranteed_async_model(): + seen = [] + + async def hello() -> str: + return "world" + + async def before(tool, tool_call): + seen.append(tool_call.tool_call_id) + + model = llm.get_async_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "hello"}]}), + tools=[hello], + before_call=before, + ) + await chain_response.text() + assert len(seen) == 1 + assert seen[0] is not None and seen[0].startswith("tc_") + + +@pytest.mark.asyncio +async def test_async_missing_tool_produces_error_result(): + # Async executor parity with sync: a call to a tool that is not in + # tools= must produce an error ToolResult, not silently vanish - + # otherwise the next provider call has a tool_call with no result. + before_calls = [] + + async def real_tool() -> str: + return "ok" + + async def before(tool, tool_call): + # before_call fires even when tool is None, like the sync path + before_calls.append((tool.name if tool else None, tool_call.name)) + + model = llm.get_async_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "missing_tool"}, {"name": "real_tool"}]}), + tools=[real_tool], + before_call=before, + ) + await chain_response.text() + + second = chain_response._responses[1] + results = [(r.name, r.output) for r in second.prompt.tool_results] + assert results == [ + ("missing_tool", 'Error: tool "missing_tool" does not exist'), + ("real_tool", "ok"), + ] + assert isinstance(second.prompt.tool_results[0].exception, KeyError) + assert (None, "missing_tool") in before_calls + + +@pytest.mark.asyncio +async def test_async_missing_tool_can_be_cancelled_by_before_call(): + async def real_tool() -> str: + return "ok" + + async def before(tool, tool_call): + if tool is None: + raise CancelToolCall("no such tool") + + model = llm.get_async_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "missing_tool"}, {"name": "real_tool"}]}), + tools=[real_tool], + before_call=before, + ) + await chain_response.text() + second = chain_response._responses[1] + results = [(r.name, r.output) for r in second.prompt.tool_results] + assert results == [ + ("missing_tool", "Cancelled: no such tool"), + ("real_tool", "ok"), + ] + + +@pytest.mark.asyncio +async def test_async_tool_without_implementation_produces_error_result(): + tool = llm.Tool( + name="no_impl", + description="A tool with no implementation", + input_schema={"type": "object", "properties": {}}, + implementation=None, + ) + model = llm.get_async_model("echo") + chain_response = model.chain( + json.dumps({"tool_calls": [{"name": "no_impl"}]}), + tools=[tool], + ) + await chain_response.text() + second = chain_response._responses[1] + assert [(r.name, r.output) for r in second.prompt.tool_results] == [ + ("no_impl", 'Error: tool "no_impl" has no implementation'), + ] diff --git a/tests/test_tools_streaming.py b/tests/test_tools_streaming.py index 31ed2cb71..1a64cac9f 100644 --- a/tests/test_tools_streaming.py +++ b/tests/test_tools_streaming.py @@ -1,8 +1,9 @@ -import llm -from llm.tools import llm_version import os + import pytest +import llm +from llm.tools import llm_version API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" @@ -38,3 +39,15 @@ def test_tools_streaming_variant_c(): "".join(chain) == "The installed version of LLM on this system is 0.fixed-version." ) + + +# This response contains streaming variant "d" where a no-argument tool call +# streams arguments=null and never sends a "{}" chunk, so the accumulated +# arguments string stays empty - json.loads("") used to raise here. +@pytest.mark.vcr(record_mode="none") +def test_tools_streaming_variant_d(): + model = llm.get_model("gpt-4.1-mini") + chain = model.chain( + "What is the current llm version?", tools=[llm_version], key=API_KEY + ) + assert "".join(chain) == "The current version of *llm* is **0.fixed-version**." diff --git a/tests/test_utils.py b/tests/test_utils.py index 51fb8754f..6b8061258 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,15 +1,17 @@ import json + import pytest + +from llm import Toolbox, get_key from llm.utils import ( extract_fenced_code_block, instantiate_from_spec, maybe_fenced_code, + monotonic_ulid, schema_dsl, simplify_usage_dict, truncate_string, - monotonic_ulid, ) -from llm import get_key, Toolbox @pytest.mark.parametrize( @@ -83,22 +85,28 @@ def test_simplify_usage_dict(input_data, expected_output): None, ], [ - "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" - "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n```", + ( + "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" + "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n```" + ), False, "def foo():\n return 'bar'\n", ], [ - "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" - "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n```", + ( + "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" + "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n```" + ), True, "function foo() {\n return 'bar';\n}\n", ], [ - "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" - # This one has trailing whitespace after the second code block: - # https://github.com/simonw/llm/pull/718#issuecomment-2613177036 - "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n``` ", + ( + "First code block:\n\n```python\ndef foo():\n return 'bar'\n```\n\n" + # This one has trailing whitespace after the second code block: + # https://github.com/simonw/llm/pull/718#issuecomment-2613177036 + "Second code block:\n\n```javascript\nfunction foo() {\n return 'bar';\n}\n``` " + ), True, "function foo() {\n return 'bar';\n}\n", ], @@ -246,6 +254,21 @@ def test_schema_dsl_multi(): } +@pytest.mark.parametrize( + ("schema", "invalid_field"), + ( + (":just a description", ":just a description"), + ("name, :description", ":description"), + ), +) +def test_schema_dsl_missing_field_name(schema, invalid_field): + with pytest.raises(ValueError) as ex: + schema_dsl(schema) + assert str(ex.value) == ( + f"Invalid schema DSL: field {invalid_field!r} is missing a name before ':'" + ) + + @pytest.mark.parametrize( "text, max_length, normalize_whitespace, keep_end, expected", [ @@ -268,8 +291,6 @@ def test_schema_dsl_multi(): ("Hello \n\t world!", 12, True, True, "Hello world!"), # Edge cases ("12345", 5, False, False, "12345"), - ("123456", 5, False, False, "12..."), - ("12345", 5, False, True, "12345"), # Unchanged for exact fit ("123456", 5, False, False, "12..."), # Regular truncation for small max_length # Very long string ("A" * 200, 10, False, False, "AAAAAAA..."), @@ -287,7 +308,6 @@ def test_schema_dsl_multi(): True, "12345...", ), # Too small for keep_end, use regular - ("1234567890", 9, False, True, "12... 90"), # Just enough for keep_end ], ) def test_truncate_string(text, max_length, normalize_whitespace, keep_end, expected): @@ -328,7 +348,7 @@ def test_test_truncate_string_keep_end( assert result == expected_full # Only check prefix/suffix when we expect truncation with keep_end - if prefix_len is not None and len(text) > max_length and max_length >= 9: + if prefix_len is not None and len(text) > max_length >= 9: assert result[:prefix_len] == text[:prefix_len] assert result[-prefix_len:] == text[-prefix_len:] assert "... " in result